diff --git a/README.md b/README.md index ce1b45b..0cc49f6 100644 --- a/README.md +++ b/README.md @@ -48,4 +48,35 @@ print(score( ).as_dict()) ``` +## With vs without skill (A/B) + +Authors ask: *does offering this skill actually help, and did the agent load it?* +Run the same cases twice and measure **lift** + **trigger rate**: + +```python +from skill_eval import Call, Case, SkillSpec, ToolSpec, compare_skill, run_ab + +tools = [ToolSpec("search", required=("q",))] +cases = [ + Case("1", "find trains", "call", tool="search", required_args={"q": "trains"}), + Case("2", "write a poem", "refuse"), +] +skill = SkillSpec("web-search", text="For lookup questions, call search with q=...") + +def agent(case, offered): + # Wire to your model/CLI. offered is SkillSpec or None. + if offered is None: + return Call(None, {}, refused=case.expect == "refuse") + if case.expect == "refuse": + return Call(None, {}, refused=True) + return Call("search", {"q": "trains"}) + +report = run_ab(tools, cases, skill, agent) +print(report.lift) # selection / schema / refusal deltas +print(report.with_skill.trigger_rate) +``` + +Or score pre-recorded call lists with `compare_skill(..., calls_with=..., calls_without=..., triggered=...)`. +Still zero dependencies — your agent callable owns any CLI/network I/O. + MIT. Python 3.10+. diff --git a/examples/skill_ab.py b/examples/skill_ab.py new file mode 100644 index 0000000..512bdef --- /dev/null +++ b/examples/skill_ab.py @@ -0,0 +1,29 @@ +"""Paired with/without-skill eval — measure lift and trigger rate.""" + +from skill_eval import Call, Case, SkillSpec, ToolSpec, run_ab + +tools = [ToolSpec("search", required=("q",))] +cases = [ + Case("1", "find trains to Madrid", "call", tool="search", required_args={"q": "trains Madrid"}), + Case("2", "write a poem about rain", "refuse"), +] +skill = SkillSpec( + "web-search", + text="For factual lookup questions, call search with q set to the query.", +) + + +def agent(case: Case, offered: SkillSpec | None) -> Call: + # Stand-in for a real model/CLI. Without the skill, selection/refusal degrade. + if offered is None: + if case.expect == "refuse": + return Call("search", {"q": "poem"}) + return Call(None, {}, refused=True) + if case.expect == "refuse": + return Call(None, {}, refused=True) + return Call("search", {"q": "trains Madrid"}) + + +if __name__ == "__main__": + report = run_ab(tools, cases, skill, agent) + print(report.as_dict()) diff --git a/src/skill_eval/__init__.py b/src/skill_eval/__init__.py index 6620ecc..8daf945 100644 --- a/src/skill_eval/__init__.py +++ b/src/skill_eval/__init__.py @@ -1,5 +1,17 @@ +from .ab import ABReport, ArmResult, SkillSpec, compare_skill, run_ab from .cases import Call, Case, ToolSpec from .score import Scorecard, score -__all__ = ["ToolSpec", "Call", "Case", "score", "Scorecard"] +__all__ = [ + "ABReport", + "ArmResult", + "Call", + "Case", + "SkillSpec", + "Scorecard", + "ToolSpec", + "compare_skill", + "run_ab", + "score", +] __version__ = "0.1.0" diff --git a/src/skill_eval/ab.py b/src/skill_eval/ab.py new file mode 100644 index 0000000..17d2834 --- /dev/null +++ b/src/skill_eval/ab.py @@ -0,0 +1,182 @@ +"""Paired with/without-skill A/B evaluation. + +Static suites often inflate gains when the agent never loads the skill. +Run the *same* cases twice — skill offered vs withheld — and report causal +lift plus trigger rate. Pure functions; no I/O, no framework. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, Sequence + +from .cases import Call, Case, ToolSpec +from .score import Scorecard, score + +AgentFn = Callable[[Case, "SkillSpec | None"], Call] +TriggerFn = Callable[[Case, Call, "SkillSpec"], bool] + + +@dataclass(frozen=True) +class SkillSpec: + """A skill offered to the agent (e.g. contents of a SKILL.md).""" + + name: str + text: str = "" + trigger_hints: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ArmResult: + """Scorecard for one arm of the A/B pair.""" + + label: str + scorecard: Scorecard + trigger_rate: float + n_triggered: int + n_eligible: int + + def as_dict(self) -> dict[str, float | int | str | dict[str, float]]: + return { + "label": self.label, + "n_eligible": self.n_eligible, + "n_triggered": self.n_triggered, + "scorecard": self.scorecard.as_dict(), + "trigger_rate": self.trigger_rate, + } + + +@dataclass(frozen=True) +class ABReport: + """Causal lift: with_skill minus without_skill, plus trigger rate.""" + + skill: str + with_skill: ArmResult + without_skill: ArmResult + lift: dict[str, float] + + def as_dict(self) -> dict[str, object]: + return { + "lift": dict(self.lift), + "skill": self.skill, + "with_skill": self.with_skill.as_dict(), + "without_skill": self.without_skill.as_dict(), + } + + +def _ratio(a: int, b: int) -> float: + return round(a / b, 4) if b else 0.0 + + +def _lift(with_card: Scorecard, without_card: Scorecard) -> dict[str, float]: + return { + "refusal": round(with_card.refusal - without_card.refusal, 4), + "schema": round(with_card.schema - without_card.schema, 4), + "selection": round(with_card.selection - without_card.selection, 4), + } + + +def default_trigger(case: Case, call: Call, skill: SkillSpec) -> bool: + """Heuristic: skill "fired" if a call arm used the expected tool, or hints hit. + + Prefer passing an explicit ``triggered`` sequence from your harness when you + can observe skill load events on a real CLI. + """ + if case.expect == "refuse": + return False + if call.refused or call.tool is None: + return False + if case.tool is not None and call.tool == case.tool: + return True + blob = f"{case.prompt}\n{skill.text}".lower() + return any(h.lower() in blob for h in skill.trigger_hints) + + +def compare_skill( + tools: Sequence[ToolSpec], + cases: Sequence[Case], + *, + skill: SkillSpec, + calls_with: Sequence[Call], + calls_without: Sequence[Call], + triggered: Sequence[bool] | None = None, +) -> ABReport: + """Compare aligned with-skill vs without-skill call lists. + + ``triggered`` (optional) marks, per case on the *with* arm, whether the + skill was actually loaded/used. When omitted, :func:`default_trigger` is + used. Trigger rate is computed only over ``expect == "call"`` cases. + """ + if len(cases) != len(calls_with) or len(cases) != len(calls_without): + raise ValueError("cases, calls_with, and calls_without must align") + if triggered is not None and len(triggered) != len(cases): + raise ValueError("triggered must align with cases") + + with_card = score(tools, cases, calls_with) + without_card = score(tools, cases, calls_without) + + flags: list[bool] + if triggered is None: + flags = [default_trigger(c, w, skill) for c, w in zip(cases, calls_with, strict=True)] + else: + flags = list(triggered) + + eligible = 0 + fired = 0 + for case, flag in zip(cases, flags, strict=True): + if case.expect != "call": + continue + eligible += 1 + if flag: + fired += 1 + + rate = _ratio(fired, eligible) + with_arm = ArmResult( + label="with_skill", + scorecard=with_card, + trigger_rate=rate, + n_triggered=fired, + n_eligible=eligible, + ) + without_arm = ArmResult( + label="without_skill", + scorecard=without_card, + trigger_rate=0.0, + n_triggered=0, + n_eligible=eligible, + ) + return ABReport( + skill=skill.name, + with_skill=with_arm, + without_skill=without_arm, + lift=_lift(with_card, without_card), + ) + + +def run_ab( + tools: Sequence[ToolSpec], + cases: Sequence[Case], + skill: SkillSpec, + agent: AgentFn, + *, + detect_trigger: TriggerFn | None = None, +) -> ABReport: + """Run ``agent(case, skill|None)`` on both arms and build an :class:`ABReport`. + + The agent receives the :class:`SkillSpec` on the with-arm and ``None`` on the + without-arm. No network I/O is performed here — your callable owns that. + """ + calls_with = [agent(case, skill) for case in cases] + calls_without = [agent(case, None) for case in cases] + detect = detect_trigger or default_trigger + triggered = [ + detect(case, call, skill) for case, call in zip(cases, calls_with, strict=True) + ] + return compare_skill( + tools, + cases, + skill=skill, + calls_with=calls_with, + calls_without=calls_without, + triggered=triggered, + ) diff --git a/tests/test_ab.py b/tests/test_ab.py new file mode 100644 index 0000000..0fc02cc --- /dev/null +++ b/tests/test_ab.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import pytest + +from skill_eval import ( + Call, + Case, + SkillSpec, + ToolSpec, + compare_skill, + run_ab, + score, +) + +TOOLS = [ToolSpec("search", required=("q",), optional=("k",))] + + +def test_compare_skill_reports_positive_lift_and_trigger_rate() -> None: + skill = SkillSpec("web-search", text="Use search with q=...", trigger_hints=("search",)) + cases = [ + Case("c1", "find cafes", "call", tool="search", required_args={"q": "cafes"}), + Case("c2", "write a poem", "refuse"), + ] + # Without skill: wrong tool / no refusal + without = [ + Call("calendar", {"date": "x"}), + Call("search", {"q": "poem"}), + ] + # With skill: correct selection + refusal + with_calls = [ + Call("search", {"q": "cafes"}), + Call(None, {}, refused=True), + ] + report = compare_skill( + TOOLS, + cases, + skill=skill, + calls_with=with_calls, + calls_without=without, + triggered=[True, False], + ) + assert report.skill == "web-search" + assert report.with_skill.scorecard.selection == 1.0 + assert report.without_skill.scorecard.selection == 0.0 + assert report.lift["selection"] == 1.0 + assert report.lift["refusal"] == 1.0 + assert report.with_skill.trigger_rate == 1.0 + assert report.with_skill.n_eligible == 1 + assert report.as_dict()["lift"]["schema"] == report.lift["schema"] + + +def test_compare_skill_requires_alignment() -> None: + skill = SkillSpec("x") + cases = [Case("c1", "q", "call", tool="search")] + with pytest.raises(ValueError, match="align"): + compare_skill( + TOOLS, + cases, + skill=skill, + calls_with=[Call("search", {"q": "a"})], + calls_without=[], + ) + + +def test_run_ab_invokes_agent_both_arms() -> None: + skill = SkillSpec("web-search", text="prefer search tool") + cases = [ + Case("c1", "find trains", "call", tool="search", required_args={"q": "trains"}), + Case("c2", "poem please", "refuse"), + ] + seen: list[str | None] = [] + + def agent(case: Case, offered: SkillSpec | None) -> Call: + seen.append(None if offered is None else offered.name) + if offered is None: + if case.expect == "refuse": + return Call("search", {"q": "nope"}) + return Call(None, {}, refused=True) + if case.expect == "refuse": + return Call(None, {}, refused=True) + return Call("search", {"q": "trains"}) + + report = run_ab(TOOLS, cases, skill, agent, detect_trigger=lambda c, call, s: call.tool == "search") + assert seen == ["web-search", "web-search", None, None] + assert report.lift["selection"] == 1.0 + assert report.lift["refusal"] == 1.0 + assert report.with_skill.trigger_rate == 1.0 + + +def test_existing_score_unchanged() -> None: + cases = [Case("c1", "find cafes", "call", tool="search")] + calls = [Call("search", {"q": "cafes"})] + s = score(TOOLS, cases, calls) + assert s.selection == 1.0