Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Install uv
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
Expand All @@ -40,7 +40,7 @@ jobs:
name: Type Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Install uv
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
Expand All @@ -62,7 +62,7 @@ jobs:
name: Unit Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Install uv
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
Expand All @@ -84,7 +84,7 @@ jobs:
name: Smoke Test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Install uv
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,18 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Initialize CodeQL
uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
with:
languages: ${{ matrix.language }}
queries: security-extended,security-and-quality

- name: Autobuild
uses: github/codeql-action/autobuild@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
with:
category: "/language:${{ matrix.language }}"
2 changes: 1 addition & 1 deletion .github/workflows/dependency-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Dependency Review
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v4
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Install uv
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/hero-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:

steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.head_ref || github.ref }}
token: ${{ secrets.GITHUB_TOKEN }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
id-token: write
contents: read
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Install uv
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
tag_exists: ${{ steps.calculate_version.outputs.tag_exists }}
pypi_exists: ${{ steps.calculate_version.outputs.pypi_exists }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0 # Fetch all history and tags

Expand Down Expand Up @@ -179,7 +179,7 @@ jobs:
runs-on: ubuntu-latest
needs: version
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Install uv
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
Expand Down Expand Up @@ -240,7 +240,7 @@ jobs:
permissions:
contents: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0

Expand Down Expand Up @@ -275,7 +275,7 @@ jobs:
path: dist/

- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

github-release:
name: Create GitHub Release
Expand All @@ -285,7 +285,7 @@ jobs:
permissions:
contents: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Download build artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
Expand Down
101 changes: 47 additions & 54 deletions examples/README.md
Original file line number Diff line number Diff line change
@@ -1,75 +1,68 @@
# Examples

Working examples are in `tests/integration/` — they are the best reference for how to use pytest-skill-engineering.
Working examples live in [`tests/integration/copilot/`](../tests/integration/copilot/). They use real GitHub Copilot models and are the best reference for pytest-skill-engineering.

## Test Files

| File | What it demonstrates |
|------|---------------------|
| [pydantic/test_01_basic.py](../tests/integration/pydantic/test_01_basic.py) | Natural language → tool usage with Banking and Todo servers |
| [pydantic/test_02_models.py](../tests/integration/pydantic/test_02_models.py) | Model comparison (parametrize) |
| [pydantic/test_03_prompts.py](../tests/integration/pydantic/test_03_prompts.py) | System prompt comparison |
| [pydantic/test_04_matrix.py](../tests/integration/pydantic/test_04_matrix.py) | Model × prompt 2×2 grid |
| [pydantic/test_05_skills.py](../tests/integration/pydantic/test_05_skills.py) | Skills with references and metadata |
| [pydantic/test_06_sessions.py](../tests/integration/pydantic/test_06_sessions.py) | Multi-turn session continuity |
| [pydantic/test_07_clarification.py](../tests/integration/pydantic/test_07_clarification.py) | ClarificationDetection feature |
| [pydantic/test_08_scoring.py](../tests/integration/pydantic/test_08_scoring.py) | llm_score + ScoringDimension |
| [pydantic/test_09_cli.py](../tests/integration/pydantic/test_09_cli.py) | CLIServer wrapping shell commands |
| [pydantic/test_10_ab_servers.py](../tests/integration/pydantic/test_10_ab_servers.py) | A/B server comparison |
| [pydantic/test_11_iterations.py](../tests/integration/pydantic/test_11_iterations.py) | --aitest-iterations=N reliability |
| [pydantic/test_12_custom_agents.py](../tests/integration/pydantic/test_12_custom_agents.py) | Eval.from_agent_file + load_custom_agent |
|------|----------------------|
| [`test_01_basic.py`](../tests/integration/copilot/test_01_basic.py) | Basic file creation and refactoring |
| [`test_02_models.py`](../tests/integration/copilot/test_02_models.py) | Model comparison |
| [`test_03_instructions.py`](../tests/integration/copilot/test_03_instructions.py) | System prompt comparison and tool restrictions |
| [`test_04_matrix.py`](../tests/integration/copilot/test_04_matrix.py) | Model × system prompt matrix |
| [`test_05_skills.py`](../tests/integration/copilot/test_05_skills.py) | Skill A/B comparison |
| [`test_06_sessions.py`](../tests/integration/copilot/test_06_sessions.py) | Multi-turn sessions |
| [`test_07_clarification.py`](../tests/integration/copilot/test_07_clarification.py) | Clarification detection |
| [`test_08_scoring.py`](../tests/integration/copilot/test_08_scoring.py) | LLM scoring |
| [`test_09_cli.py`](../tests/integration/copilot/test_09_cli.py) | CLI workflows |
| [`test_10_ab_servers.py`](../tests/integration/copilot/test_10_ab_servers.py) | Configuration A/B comparison |
| [`test_11_iterations.py`](../tests/integration/copilot/test_11_iterations.py) | Iteration reliability |
| [`test_12_custom_agents.py`](../tests/integration/copilot/test_12_custom_agents.py) | Custom agent dispatch |
| [`test_13_plugins.py`](../tests/integration/copilot/test_13_plugins.py) | Plugin discovery and loading |
| [`test_14_skill_evals.py`](../tests/integration/copilot/test_14_skill_evals.py) | Skill eval execution |
| [`test_15_skill_refinement.py`](../tests/integration/copilot/test_15_skill_refinement.py) | Skill refinement |
| [`test_16_skill_benchmark.py`](../tests/integration/copilot/test_16_skill_benchmark.py) | Skill benchmarking |
| [`test_17_plugin_skill_workflow.py`](../tests/integration/copilot/test_17_plugin_skill_workflow.py) | End-to-end plugin skill workflow |
| [`test_events.py`](../tests/integration/copilot/test_events.py) | Copilot SDK event capture |

## Run Examples

```bash
# Prerequisites
uv add pytest-skill-engineering
az login # For Azure OpenAI
# Authenticate once
gh auth login

# Run basic usage tests
uv run python -m pytest tests/integration/pydantic/test_01_basic.py -v
uv run python -m pytest tests/integration/copilot/test_01_basic.py -v

# Run all pydantic tests
uv run python -m pytest tests/integration/pydantic/ -v
# Run all Copilot integration tests
uv run python -m pytest tests/integration/copilot/ -v

# Run with report
uv run python -m pytest tests/integration/pydantic/ -v --aitest-html=report.html
# Generate a report
uv run python -m pytest tests/integration/copilot/test_04_matrix.py -v \
--aitest-html=report.html
```

## Test Servers
## MCP Server Example

Two built-in test servers for natural language testing:
`CopilotEval` accepts MCP server configurations in the Copilot SDK format:

### Banking Server
```python
@pytest.fixture(scope="module")
def banking_server():
return MCPServer(
command=["python", "-m", "pytest_skill_engineering.testing.banking_mcp"],
wait=Wait.for_tools(["get_balance", "transfer"]),
)


# Test: "What's my checking balance?"
import sys

from pytest_skill_engineering.copilot import CopilotEval


banking_eval = CopilotEval(
name="banking",
instructions="Use the banking tools for every account request.",
mcp_servers={
"banking": {
"command": sys.executable,
"args": ["-m", "pytest_skill_engineering.testing.banking_mcp"],
"tools": ["*"],
}
},
)
```

### Todo Server
```python
@pytest.fixture(scope="module")
def todo_server():
return MCPServer(
command=["python", "-m", "pytest_skill_engineering.testing.todo_mcp"],
wait=Wait.for_tools(["add_task", "list_tasks"]),
)


# Test: "Add buy milk to my shopping list"
```

## Fixtures

See [conftest.py](../tests/integration/conftest.py) for fixture patterns:

- `banking_server` / `todo_server` — Test MCP servers
- `DEFAULT_MODEL`, `DEFAULT_RPM`, `DEFAULT_TPM`, `DEFAULT_MAX_TURNS` — Constants for eval creation
- Azure token handling
The package includes banking and todo MCP servers for integration scenarios. See [`tests/showcase/`](../tests/showcase/) for a complete banking example.
1 change: 1 addition & 0 deletions tests/integration/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ tests/integration/
├── test_01_basic.py # Basic file creation and refactoring
├── test_02_models.py # Model comparison
├── test_03_instructions.py # System prompt and tool filtering
├── test_04_matrix.py # Model × system prompt matrix
├── test_05_skills.py # Skill A/B comparison
├── test_06_sessions.py # Multi-turn sessions
├── test_07_clarification.py # Clarification detection
Expand Down
64 changes: 64 additions & 0 deletions tests/integration/copilot/test_04_matrix.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
"""Level 04 — Model × system prompt matrix comparison.

Runs the same task across every configured model and two distinct system
prompts. The report exposes both dimensions for side-by-side comparison.

Run with: pytest tests/integration/copilot/test_04_matrix.py -v
"""

from __future__ import annotations

import pytest

from pytest_skill_engineering.copilot.eval import CopilotEval

from .conftest import MODELS

pytestmark = [pytest.mark.copilot]

SYSTEM_PROMPTS = {
"typed": (
"Write fully typed Python. Every function must have parameter and return "
"type annotations plus a concise docstring."
),
"minimal": (
"Write minimal Python. Do not add type annotations, docstrings, comments, "
"tests, or supporting files."
),
}


class TestModelSystemPromptMatrix:
"""Compare each model under typed and minimal system prompts."""

@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("system_prompt_name", SYSTEM_PROMPTS)
async def test_greeting_module(
self,
copilot_eval,
tmp_path,
model: str,
system_prompt_name: str,
) -> None:
"""Each matrix cell creates the requested module in its required style."""
agent = CopilotEval(
name=f"{model}-{system_prompt_name}",
model=model,
instructions=SYSTEM_PROMPTS[system_prompt_name],
working_directory=str(tmp_path),
)

result = await copilot_eval(
agent,
"Create greeting.py with greet(name), returning 'Hello, {name}!'.",
)

assert result.success, result.error
content = (tmp_path / "greeting.py").read_text(encoding="utf-8")
assert "def greet" in content
if system_prompt_name == "typed":
assert "-> str" in content
assert '"""' in content or "'''" in content
else:
assert "->" not in content
assert '"""' not in content and "'''" not in content