diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dc63b4..533a426 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: name: Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 @@ -40,7 +40,7 @@ jobs: name: Type Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 @@ -62,7 +62,7 @@ jobs: name: Unit Tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 @@ -84,7 +84,7 @@ jobs: name: Smoke Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b298458..0fbd1dd 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -25,18 +25,18 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Initialize CodeQL - uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} queries: security-extended,security-and-quality - name: Autobuild - uses: github/codeql-action/autobuild@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4 + uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 02c7578..abc3ec0 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Dependency Review uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v4 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f7731e3..c814aab 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,7 +27,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 diff --git a/.github/workflows/hero-tests.yml b/.github/workflows/hero-tests.yml index 4c62de4..03f93bf 100644 --- a/.github/workflows/hero-tests.yml +++ b/.github/workflows/hero-tests.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.head_ref || github.ref }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3f398ef..3870e9a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,7 +22,7 @@ jobs: id-token: write contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ba985a..3ded9af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: tag_exists: ${{ steps.calculate_version.outputs.tag_exists }} pypi_exists: ${{ steps.calculate_version.outputs.pypi_exists }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 # Fetch all history and tags @@ -179,7 +179,7 @@ jobs: runs-on: ubuntu-latest needs: version steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 @@ -240,7 +240,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 @@ -275,7 +275,7 @@ jobs: path: dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 github-release: name: Create GitHub Release @@ -285,7 +285,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Download build artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 diff --git a/examples/README.md b/examples/README.md index 1f2ba0d..14b826d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,75 +1,68 @@ # Examples -Working examples are in `tests/integration/` — they are the best reference for how to use pytest-skill-engineering. +Working examples live in [`tests/integration/copilot/`](../tests/integration/copilot/). They use real GitHub Copilot models and are the best reference for pytest-skill-engineering. ## Test Files | File | What it demonstrates | -|------|---------------------| -| [pydantic/test_01_basic.py](../tests/integration/pydantic/test_01_basic.py) | Natural language → tool usage with Banking and Todo servers | -| [pydantic/test_02_models.py](../tests/integration/pydantic/test_02_models.py) | Model comparison (parametrize) | -| [pydantic/test_03_prompts.py](../tests/integration/pydantic/test_03_prompts.py) | System prompt comparison | -| [pydantic/test_04_matrix.py](../tests/integration/pydantic/test_04_matrix.py) | Model × prompt 2×2 grid | -| [pydantic/test_05_skills.py](../tests/integration/pydantic/test_05_skills.py) | Skills with references and metadata | -| [pydantic/test_06_sessions.py](../tests/integration/pydantic/test_06_sessions.py) | Multi-turn session continuity | -| [pydantic/test_07_clarification.py](../tests/integration/pydantic/test_07_clarification.py) | ClarificationDetection feature | -| [pydantic/test_08_scoring.py](../tests/integration/pydantic/test_08_scoring.py) | llm_score + ScoringDimension | -| [pydantic/test_09_cli.py](../tests/integration/pydantic/test_09_cli.py) | CLIServer wrapping shell commands | -| [pydantic/test_10_ab_servers.py](../tests/integration/pydantic/test_10_ab_servers.py) | A/B server comparison | -| [pydantic/test_11_iterations.py](../tests/integration/pydantic/test_11_iterations.py) | --aitest-iterations=N reliability | -| [pydantic/test_12_custom_agents.py](../tests/integration/pydantic/test_12_custom_agents.py) | Eval.from_agent_file + load_custom_agent | +|------|----------------------| +| [`test_01_basic.py`](../tests/integration/copilot/test_01_basic.py) | Basic file creation and refactoring | +| [`test_02_models.py`](../tests/integration/copilot/test_02_models.py) | Model comparison | +| [`test_03_instructions.py`](../tests/integration/copilot/test_03_instructions.py) | System prompt comparison and tool restrictions | +| [`test_04_matrix.py`](../tests/integration/copilot/test_04_matrix.py) | Model × system prompt matrix | +| [`test_05_skills.py`](../tests/integration/copilot/test_05_skills.py) | Skill A/B comparison | +| [`test_06_sessions.py`](../tests/integration/copilot/test_06_sessions.py) | Multi-turn sessions | +| [`test_07_clarification.py`](../tests/integration/copilot/test_07_clarification.py) | Clarification detection | +| [`test_08_scoring.py`](../tests/integration/copilot/test_08_scoring.py) | LLM scoring | +| [`test_09_cli.py`](../tests/integration/copilot/test_09_cli.py) | CLI workflows | +| [`test_10_ab_servers.py`](../tests/integration/copilot/test_10_ab_servers.py) | Configuration A/B comparison | +| [`test_11_iterations.py`](../tests/integration/copilot/test_11_iterations.py) | Iteration reliability | +| [`test_12_custom_agents.py`](../tests/integration/copilot/test_12_custom_agents.py) | Custom agent dispatch | +| [`test_13_plugins.py`](../tests/integration/copilot/test_13_plugins.py) | Plugin discovery and loading | +| [`test_14_skill_evals.py`](../tests/integration/copilot/test_14_skill_evals.py) | Skill eval execution | +| [`test_15_skill_refinement.py`](../tests/integration/copilot/test_15_skill_refinement.py) | Skill refinement | +| [`test_16_skill_benchmark.py`](../tests/integration/copilot/test_16_skill_benchmark.py) | Skill benchmarking | +| [`test_17_plugin_skill_workflow.py`](../tests/integration/copilot/test_17_plugin_skill_workflow.py) | End-to-end plugin skill workflow | +| [`test_events.py`](../tests/integration/copilot/test_events.py) | Copilot SDK event capture | ## Run Examples ```bash -# Prerequisites -uv add pytest-skill-engineering -az login # For Azure OpenAI +# Authenticate once +gh auth login # Run basic usage tests -uv run python -m pytest tests/integration/pydantic/test_01_basic.py -v +uv run python -m pytest tests/integration/copilot/test_01_basic.py -v -# Run all pydantic tests -uv run python -m pytest tests/integration/pydantic/ -v +# Run all Copilot integration tests +uv run python -m pytest tests/integration/copilot/ -v -# Run with report -uv run python -m pytest tests/integration/pydantic/ -v --aitest-html=report.html +# Generate a report +uv run python -m pytest tests/integration/copilot/test_04_matrix.py -v \ + --aitest-html=report.html ``` -## Test Servers +## MCP Server Example -Two built-in test servers for natural language testing: +`CopilotEval` accepts MCP server configurations in the Copilot SDK format: -### Banking Server ```python -@pytest.fixture(scope="module") -def banking_server(): - return MCPServer( - command=["python", "-m", "pytest_skill_engineering.testing.banking_mcp"], - wait=Wait.for_tools(["get_balance", "transfer"]), - ) - - -# Test: "What's my checking balance?" +import sys + +from pytest_skill_engineering.copilot import CopilotEval + + +banking_eval = CopilotEval( + name="banking", + instructions="Use the banking tools for every account request.", + mcp_servers={ + "banking": { + "command": sys.executable, + "args": ["-m", "pytest_skill_engineering.testing.banking_mcp"], + "tools": ["*"], + } + }, +) ``` -### Todo Server -```python -@pytest.fixture(scope="module") -def todo_server(): - return MCPServer( - command=["python", "-m", "pytest_skill_engineering.testing.todo_mcp"], - wait=Wait.for_tools(["add_task", "list_tasks"]), - ) - - -# Test: "Add buy milk to my shopping list" -``` - -## Fixtures - -See [conftest.py](../tests/integration/conftest.py) for fixture patterns: - -- `banking_server` / `todo_server` — Test MCP servers -- `DEFAULT_MODEL`, `DEFAULT_RPM`, `DEFAULT_TPM`, `DEFAULT_MAX_TURNS` — Constants for eval creation -- Azure token handling +The package includes banking and todo MCP servers for integration scenarios. See [`tests/showcase/`](../tests/showcase/) for a complete banking example. diff --git a/tests/integration/README.md b/tests/integration/README.md index df9f8b0..e5863de 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -16,6 +16,7 @@ tests/integration/ ├── test_01_basic.py # Basic file creation and refactoring ├── test_02_models.py # Model comparison ├── test_03_instructions.py # System prompt and tool filtering + ├── test_04_matrix.py # Model × system prompt matrix ├── test_05_skills.py # Skill A/B comparison ├── test_06_sessions.py # Multi-turn sessions ├── test_07_clarification.py # Clarification detection diff --git a/tests/integration/copilot/test_04_matrix.py b/tests/integration/copilot/test_04_matrix.py new file mode 100644 index 0000000..90957f0 --- /dev/null +++ b/tests/integration/copilot/test_04_matrix.py @@ -0,0 +1,64 @@ +"""Level 04 — Model × system prompt matrix comparison. + +Runs the same task across every configured model and two distinct system +prompts. The report exposes both dimensions for side-by-side comparison. + +Run with: pytest tests/integration/copilot/test_04_matrix.py -v +""" + +from __future__ import annotations + +import pytest + +from pytest_skill_engineering.copilot.eval import CopilotEval + +from .conftest import MODELS + +pytestmark = [pytest.mark.copilot] + +SYSTEM_PROMPTS = { + "typed": ( + "Write fully typed Python. Every function must have parameter and return " + "type annotations plus a concise docstring." + ), + "minimal": ( + "Write minimal Python. Do not add type annotations, docstrings, comments, " + "tests, or supporting files." + ), +} + + +class TestModelSystemPromptMatrix: + """Compare each model under typed and minimal system prompts.""" + + @pytest.mark.parametrize("model", MODELS) + @pytest.mark.parametrize("system_prompt_name", SYSTEM_PROMPTS) + async def test_greeting_module( + self, + copilot_eval, + tmp_path, + model: str, + system_prompt_name: str, + ) -> None: + """Each matrix cell creates the requested module in its required style.""" + agent = CopilotEval( + name=f"{model}-{system_prompt_name}", + model=model, + instructions=SYSTEM_PROMPTS[system_prompt_name], + working_directory=str(tmp_path), + ) + + result = await copilot_eval( + agent, + "Create greeting.py with greet(name), returning 'Hello, {name}!'.", + ) + + assert result.success, result.error + content = (tmp_path / "greeting.py").read_text(encoding="utf-8") + assert "def greet" in content + if system_prompt_name == "typed": + assert "-> str" in content + assert '"""' in content or "'''" in content + else: + assert "->" not in content + assert '"""' not in content and "'''" not in content