Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,35 @@ print(score(
).as_dict())
```

## With vs without skill (A/B)

Authors ask: *does offering this skill actually help, and did the agent load it?*
Run the same cases twice and measure **lift** + **trigger rate**:

```python
from skill_eval import Call, Case, SkillSpec, ToolSpec, compare_skill, run_ab

tools = [ToolSpec("search", required=("q",))]
cases = [
Case("1", "find trains", "call", tool="search", required_args={"q": "trains"}),
Case("2", "write a poem", "refuse"),
]
skill = SkillSpec("web-search", text="For lookup questions, call search with q=...")

def agent(case, offered):
# Wire to your model/CLI. offered is SkillSpec or None.
if offered is None:
return Call(None, {}, refused=case.expect == "refuse")
if case.expect == "refuse":
return Call(None, {}, refused=True)
return Call("search", {"q": "trains"})

report = run_ab(tools, cases, skill, agent)
print(report.lift) # selection / schema / refusal deltas
print(report.with_skill.trigger_rate)
```

Or score pre-recorded call lists with `compare_skill(..., calls_with=..., calls_without=..., triggered=...)`.
Still zero dependencies — your agent callable owns any CLI/network I/O.

MIT. Python 3.10+.
29 changes: 29 additions & 0 deletions examples/skill_ab.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
"""Paired with/without-skill eval — measure lift and trigger rate."""

from skill_eval import Call, Case, SkillSpec, ToolSpec, run_ab

tools = [ToolSpec("search", required=("q",))]
cases = [
Case("1", "find trains to Madrid", "call", tool="search", required_args={"q": "trains Madrid"}),
Case("2", "write a poem about rain", "refuse"),
]
skill = SkillSpec(
"web-search",
text="For factual lookup questions, call search with q set to the query.",
)


def agent(case: Case, offered: SkillSpec | None) -> Call:
# Stand-in for a real model/CLI. Without the skill, selection/refusal degrade.
if offered is None:
if case.expect == "refuse":
return Call("search", {"q": "poem"})
return Call(None, {}, refused=True)
if case.expect == "refuse":
return Call(None, {}, refused=True)
return Call("search", {"q": "trains Madrid"})


if __name__ == "__main__":
report = run_ab(tools, cases, skill, agent)
print(report.as_dict())
14 changes: 13 additions & 1 deletion src/skill_eval/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
from .ab import ABReport, ArmResult, SkillSpec, compare_skill, run_ab
from .cases import Call, Case, ToolSpec
from .score import Scorecard, score

__all__ = ["ToolSpec", "Call", "Case", "score", "Scorecard"]
__all__ = [
"ABReport",
"ArmResult",
"Call",
"Case",
"SkillSpec",
"Scorecard",
"ToolSpec",
"compare_skill",
"run_ab",
"score",
]
__version__ = "0.1.0"
182 changes: 182 additions & 0 deletions src/skill_eval/ab.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
"""Paired with/without-skill A/B evaluation.

Static suites often inflate gains when the agent never loads the skill.
Run the *same* cases twice — skill offered vs withheld — and report causal
lift plus trigger rate. Pure functions; no I/O, no framework.
"""

from __future__ import annotations

from dataclasses import dataclass
from typing import Callable, Sequence

from .cases import Call, Case, ToolSpec
from .score import Scorecard, score

AgentFn = Callable[[Case, "SkillSpec | None"], Call]
TriggerFn = Callable[[Case, Call, "SkillSpec"], bool]


@dataclass(frozen=True)
class SkillSpec:
"""A skill offered to the agent (e.g. contents of a SKILL.md)."""

name: str
text: str = ""
trigger_hints: tuple[str, ...] = ()


@dataclass(frozen=True)
class ArmResult:
"""Scorecard for one arm of the A/B pair."""

label: str
scorecard: Scorecard
trigger_rate: float
n_triggered: int
n_eligible: int

def as_dict(self) -> dict[str, float | int | str | dict[str, float]]:
return {
"label": self.label,
"n_eligible": self.n_eligible,
"n_triggered": self.n_triggered,
"scorecard": self.scorecard.as_dict(),
"trigger_rate": self.trigger_rate,
}


@dataclass(frozen=True)
class ABReport:
"""Causal lift: with_skill minus without_skill, plus trigger rate."""

skill: str
with_skill: ArmResult
without_skill: ArmResult
lift: dict[str, float]

def as_dict(self) -> dict[str, object]:
return {
"lift": dict(self.lift),
"skill": self.skill,
"with_skill": self.with_skill.as_dict(),
"without_skill": self.without_skill.as_dict(),
}


def _ratio(a: int, b: int) -> float:
return round(a / b, 4) if b else 0.0


def _lift(with_card: Scorecard, without_card: Scorecard) -> dict[str, float]:
return {
"refusal": round(with_card.refusal - without_card.refusal, 4),
"schema": round(with_card.schema - without_card.schema, 4),
"selection": round(with_card.selection - without_card.selection, 4),
}


def default_trigger(case: Case, call: Call, skill: SkillSpec) -> bool:
"""Heuristic: skill "fired" if a call arm used the expected tool, or hints hit.

Prefer passing an explicit ``triggered`` sequence from your harness when you
can observe skill load events on a real CLI.
"""
if case.expect == "refuse":
return False
if call.refused or call.tool is None:
return False
if case.tool is not None and call.tool == case.tool:
return True
blob = f"{case.prompt}\n{skill.text}".lower()
return any(h.lower() in blob for h in skill.trigger_hints)


def compare_skill(
tools: Sequence[ToolSpec],
cases: Sequence[Case],
*,
skill: SkillSpec,
calls_with: Sequence[Call],
calls_without: Sequence[Call],
triggered: Sequence[bool] | None = None,
) -> ABReport:
"""Compare aligned with-skill vs without-skill call lists.

``triggered`` (optional) marks, per case on the *with* arm, whether the
skill was actually loaded/used. When omitted, :func:`default_trigger` is
used. Trigger rate is computed only over ``expect == "call"`` cases.
"""
if len(cases) != len(calls_with) or len(cases) != len(calls_without):
raise ValueError("cases, calls_with, and calls_without must align")
if triggered is not None and len(triggered) != len(cases):
raise ValueError("triggered must align with cases")

with_card = score(tools, cases, calls_with)
without_card = score(tools, cases, calls_without)

flags: list[bool]
if triggered is None:
flags = [default_trigger(c, w, skill) for c, w in zip(cases, calls_with, strict=True)]
else:
flags = list(triggered)

eligible = 0
fired = 0
for case, flag in zip(cases, flags, strict=True):
if case.expect != "call":
continue
eligible += 1
if flag:
fired += 1

rate = _ratio(fired, eligible)
with_arm = ArmResult(
label="with_skill",
scorecard=with_card,
trigger_rate=rate,
n_triggered=fired,
n_eligible=eligible,
)
without_arm = ArmResult(
label="without_skill",
scorecard=without_card,
trigger_rate=0.0,
n_triggered=0,
n_eligible=eligible,
)
return ABReport(
skill=skill.name,
with_skill=with_arm,
without_skill=without_arm,
lift=_lift(with_card, without_card),
)


def run_ab(
tools: Sequence[ToolSpec],
cases: Sequence[Case],
skill: SkillSpec,
agent: AgentFn,
*,
detect_trigger: TriggerFn | None = None,
) -> ABReport:
"""Run ``agent(case, skill|None)`` on both arms and build an :class:`ABReport`.

The agent receives the :class:`SkillSpec` on the with-arm and ``None`` on the
without-arm. No network I/O is performed here — your callable owns that.
"""
calls_with = [agent(case, skill) for case in cases]
calls_without = [agent(case, None) for case in cases]
detect = detect_trigger or default_trigger
triggered = [
detect(case, call, skill) for case, call in zip(cases, calls_with, strict=True)
]
return compare_skill(
tools,
cases,
skill=skill,
calls_with=calls_with,
calls_without=calls_without,
triggered=triggered,
)
94 changes: 94 additions & 0 deletions tests/test_ab.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
from __future__ import annotations

import pytest

from skill_eval import (
Call,
Case,
SkillSpec,
ToolSpec,
compare_skill,
run_ab,
score,
)

TOOLS = [ToolSpec("search", required=("q",), optional=("k",))]


def test_compare_skill_reports_positive_lift_and_trigger_rate() -> None:
skill = SkillSpec("web-search", text="Use search with q=...", trigger_hints=("search",))
cases = [
Case("c1", "find cafes", "call", tool="search", required_args={"q": "cafes"}),
Case("c2", "write a poem", "refuse"),
]
# Without skill: wrong tool / no refusal
without = [
Call("calendar", {"date": "x"}),
Call("search", {"q": "poem"}),
]
# With skill: correct selection + refusal
with_calls = [
Call("search", {"q": "cafes"}),
Call(None, {}, refused=True),
]
report = compare_skill(
TOOLS,
cases,
skill=skill,
calls_with=with_calls,
calls_without=without,
triggered=[True, False],
)
assert report.skill == "web-search"
assert report.with_skill.scorecard.selection == 1.0
assert report.without_skill.scorecard.selection == 0.0
assert report.lift["selection"] == 1.0
assert report.lift["refusal"] == 1.0
assert report.with_skill.trigger_rate == 1.0
assert report.with_skill.n_eligible == 1
assert report.as_dict()["lift"]["schema"] == report.lift["schema"]


def test_compare_skill_requires_alignment() -> None:
skill = SkillSpec("x")
cases = [Case("c1", "q", "call", tool="search")]
with pytest.raises(ValueError, match="align"):
compare_skill(
TOOLS,
cases,
skill=skill,
calls_with=[Call("search", {"q": "a"})],
calls_without=[],
)


def test_run_ab_invokes_agent_both_arms() -> None:
skill = SkillSpec("web-search", text="prefer search tool")
cases = [
Case("c1", "find trains", "call", tool="search", required_args={"q": "trains"}),
Case("c2", "poem please", "refuse"),
]
seen: list[str | None] = []

def agent(case: Case, offered: SkillSpec | None) -> Call:
seen.append(None if offered is None else offered.name)
if offered is None:
if case.expect == "refuse":
return Call("search", {"q": "nope"})
return Call(None, {}, refused=True)
if case.expect == "refuse":
return Call(None, {}, refused=True)
return Call("search", {"q": "trains"})

report = run_ab(TOOLS, cases, skill, agent, detect_trigger=lambda c, call, s: call.tool == "search")
assert seen == ["web-search", "web-search", None, None]
assert report.lift["selection"] == 1.0
assert report.lift["refusal"] == 1.0
assert report.with_skill.trigger_rate == 1.0


def test_existing_score_unchanged() -> None:
cases = [Case("c1", "find cafes", "call", tool="search")]
calls = [Call("search", {"q": "cafes"})]
s = score(TOOLS, cases, calls)
assert s.selection == 1.0
Loading