From c9db31f51f2f8f46bbcfdb89262e1f23c9fda441 Mon Sep 17 00:00:00 2001 From: Vinay Kumar K M Date: Tue, 6 Oct 2026 03:29:26 +0000 Subject: [PATCH] feat(trigger): precision/recall with near-miss negatives and k repeated runs run_triggers(cases, skill, agent, k=3, threshold=0.5, competing=...) reports precision, recall, false-trigger rate, F1, run-level fire rates and flaky cases. from_observations scores recorded runs; split_cases gives a stratified train/val split for description tuning. Zero deps. --- README.md | 38 +++++ examples/trigger_eval.py | 30 ++++ src/skill_eval/__init__.py | 14 ++ src/skill_eval/trigger.py | 277 +++++++++++++++++++++++++++++++++++++ tests/test_trigger.py | 117 ++++++++++++++++ 5 files changed, 476 insertions(+) create mode 100644 examples/trigger_eval.py create mode 100644 src/skill_eval/trigger.py create mode 100644 tests/test_trigger.py diff --git a/README.md b/README.md index 0cc49f6..bd5bd91 100644 --- a/README.md +++ b/README.md @@ -79,4 +79,42 @@ print(report.with_skill.trigger_rate) Or score pre-recorded call lists with `compare_skill(..., calls_with=..., calls_without=..., triggered=...)`. Still zero dependencies — your agent callable owns any CLI/network I/O. +## Trigger precision / recall (near-miss negatives, k runs) + +`trigger_rate` above only counts cases that *should* use the skill, so it +measures recall. A description that fires on everything gets 100% there. +`run_triggers` adds **should-not-trigger near-misses**, **k repeated runs** +per case and optional **competing skills**: + +```python +from skill_eval import SkillSpec, TriggerCase, run_triggers, split_cases + +skill = SkillSpec("pdf-forms", text="Fill or read form fields in PDF files.") +cases = [ + TriggerCase("fill", "fill the form fields in invoice.pdf", True), + TriggerCase("convert", "convert report.docx to PDF", False, note="near-miss"), + TriggerCase("summary", "summarize this PDF article", False, note="near-miss"), +] + +def agent(case, offered, run): + # Call your model/CLI with `offered` skills; return the loaded skill name(s), + # None, or a bool meaning "this skill loaded". + ... + +report = run_triggers(cases, skill, agent, k=3, competing=[SkillSpec("docx")]) +print(report.precision, report.recall, report.false_trigger_rate, report.f1) +print(report.table()) # per case: fired k/k, TP/FP/FN/TN, flaky +train, val = split_cases(cases) # tune on train, report on val +``` + +- A case counts as triggered when it fired on at least `threshold` (default 0.5) + of its `k` runs. `report.run_level` gives the raw per-run fire rates, and + `report.flaky` lists the cases that fired on only some runs. +- Already have logs? Use `from_observations(cases, "pdf-forms", fired=[[True, False, True], ...])`. +- Undefined ratios (for example precision when nothing fired) are `0.0`. + The counts are always in `as_dict()["counts"]`. + +See [`examples/trigger_eval.py`](examples/trigger_eval.py), which compares a +vague and a specific description. + MIT. Python 3.10+. diff --git a/examples/trigger_eval.py b/examples/trigger_eval.py new file mode 100644 index 0000000..8cd5b21 --- /dev/null +++ b/examples/trigger_eval.py @@ -0,0 +1,30 @@ +"""Does the skill description fire when it should, and stay quiet on near-misses?""" + +from skill_eval import SkillSpec, TriggerCase, run_triggers + +skill_v1 = SkillSpec("pdf-forms", text="Use for anything involving PDFs.") +skill_v2 = SkillSpec("pdf-forms", text="Use to fill or read *form fields* in PDF files. Not for converting or summarizing PDFs.") +docx = SkillSpec("docx", text="Edit Word documents.") + +cases = [ + TriggerCase("fill", "fill the form fields in invoice.pdf", True), + TriggerCase("fields", "what fields does tax-form.pdf have?", True), + TriggerCase("convert", "convert report.docx to PDF", False, note="near-miss: mentions PDF"), + TriggerCase("summary", "summarize this PDF article", False, note="near-miss: PDF, no form"), +] + + +def agent(case: TriggerCase, offered, run: int): + # Stand-in for a real model/CLI that reports which skill it loaded. + desc = offered[0].text.lower() + prompt = case.prompt.lower() + if "docx" in prompt: + return "docx" if "not for converting" in desc else ["pdf-forms", "docx"] + if "anything involving pdfs" in desc: + return "pdf-forms" if "pdf" in prompt else None + return "pdf-forms" if ("field" in prompt or "form" in prompt) else None + + +if __name__ == "__main__": + for skill in (skill_v1, skill_v2): + print(run_triggers(cases, skill, agent, k=3, competing=[docx]).table()) diff --git a/src/skill_eval/__init__.py b/src/skill_eval/__init__.py index 8daf945..dc87dcc 100644 --- a/src/skill_eval/__init__.py +++ b/src/skill_eval/__init__.py @@ -1,17 +1,31 @@ from .ab import ABReport, ArmResult, SkillSpec, compare_skill, run_ab from .cases import Call, Case, ToolSpec from .score import Scorecard, score +from .trigger import ( + CaseOutcome, + TriggerCase, + TriggerReport, + from_observations, + run_triggers, + split_cases, +) __all__ = [ "ABReport", "ArmResult", "Call", "Case", + "CaseOutcome", "SkillSpec", "Scorecard", "ToolSpec", + "TriggerCase", + "TriggerReport", "compare_skill", + "from_observations", "run_ab", + "run_triggers", "score", + "split_cases", ] __version__ = "0.1.0" diff --git a/src/skill_eval/trigger.py b/src/skill_eval/trigger.py new file mode 100644 index 0000000..a2609ac --- /dev/null +++ b/src/skill_eval/trigger.py @@ -0,0 +1,277 @@ +"""Trigger precision/recall for skill descriptions. + +The A/B harness measures trigger *rate* over cases that should use the skill, +which is recall only. A description that fires on everything scores 100% +there. This module adds the other half: **near-miss negatives** (prompts that +look related but should not load the skill), **k repeated runs** per case +(skill loading is stochastic), and optional **competing skills** offered at +the same time. + +Pure functions; your ``agent`` callable owns any model/CLI I/O. +""" + +from __future__ import annotations + +import random +from collections.abc import Collection +from dataclasses import dataclass +from typing import Callable, Sequence, Union + +from .ab import SkillSpec + +AgentResult = Union[bool, str, None, Collection[str]] +TriggerAgent = Callable[["TriggerCase", Sequence[SkillSpec], int], AgentResult] + + +@dataclass(frozen=True) +class TriggerCase: + """A prompt plus whether the skill under test *should* load for it. + + Use ``should_trigger=False`` for near-misses: prompts that share words or + a domain with the skill but belong to another skill or to no skill. + """ + + tid: str + prompt: str + should_trigger: bool + note: str = "" + + +@dataclass(frozen=True) +class CaseOutcome: + """Per-case result over ``runs`` repetitions.""" + + case: TriggerCase + runs: int + fired: int + threshold: float + + @property + def rate(self) -> float: + return round(self.fired / self.runs, 4) if self.runs else 0.0 + + @property + def triggered(self) -> bool: + """Counted as triggered when the run rate is at least ``threshold``.""" + return self.runs > 0 and self.fired / self.runs >= self.threshold + + @property + def correct(self) -> bool: + return self.triggered == self.case.should_trigger + + @property + def flaky(self) -> bool: + """Fired on some runs but not all.""" + return 0 < self.fired < self.runs + + @property + def kind(self) -> str: + if self.case.should_trigger: + return "TP" if self.triggered else "FN" + return "FP" if self.triggered else "TN" + + +def _ratio(a: int | float, b: int | float) -> float: + return round(a / b, 4) if b else 0.0 + + +@dataclass(frozen=True) +class TriggerReport: + """Precision / recall / false-trigger rate for one skill description.""" + + skill: str + k: int + threshold: float + outcomes: tuple[CaseOutcome, ...] + + def _count(self, kind: str) -> int: + return sum(1 for o in self.outcomes if o.kind == kind) + + @property + def tp(self) -> int: + return self._count("TP") + + @property + def fp(self) -> int: + return self._count("FP") + + @property + def fn(self) -> int: + return self._count("FN") + + @property + def tn(self) -> int: + return self._count("TN") + + @property + def precision(self) -> float: + return _ratio(self.tp, self.tp + self.fp) + + @property + def recall(self) -> float: + return _ratio(self.tp, self.tp + self.fn) + + @property + def false_trigger_rate(self) -> float: + return _ratio(self.fp, self.fp + self.tn) + + @property + def f1(self) -> float: + p, r = self.precision, self.recall + return round(2 * p * r / (p + r), 4) if (p + r) else 0.0 + + @property + def accuracy(self) -> float: + return _ratio(self.tp + self.tn, len(self.outcomes)) + + @property + def run_level(self) -> dict[str, float]: + """Raw per-run firing rates, before the per-case threshold.""" + pos = [o for o in self.outcomes if o.case.should_trigger] + neg = [o for o in self.outcomes if not o.case.should_trigger] + return { + "negative_fire_rate": _ratio(sum(o.fired for o in neg), sum(o.runs for o in neg)), + "positive_fire_rate": _ratio(sum(o.fired for o in pos), sum(o.runs for o in pos)), + } + + @property + def flaky(self) -> tuple[CaseOutcome, ...]: + return tuple(o for o in self.outcomes if o.flaky) + + def failures(self) -> tuple[CaseOutcome, ...]: + """False negatives and false triggers.""" + return tuple(o for o in self.outcomes if not o.correct) + + def as_dict(self) -> dict[str, object]: + return { + "accuracy": self.accuracy, + "counts": {"fn": self.fn, "fp": self.fp, "tn": self.tn, "tp": self.tp}, + "f1": self.f1, + "false_trigger_rate": self.false_trigger_rate, + "flaky": [o.case.tid for o in self.flaky], + "k": self.k, + "precision": self.precision, + "recall": self.recall, + "run_level": self.run_level, + "skill": self.skill, + "threshold": self.threshold, + } + + def table(self) -> str: + """Markdown summary plus a per-case table (worst first).""" + lines = [ + f"# Trigger eval: `{self.skill}` (k={self.k}, threshold={self.threshold:g})", + "", + f"precision {self.precision:.2f} · recall {self.recall:.2f} · " + f"false-trigger rate {self.false_trigger_rate:.2f} · F1 {self.f1:.2f} · " + f"flaky {len(self.flaky)}", + "", + "| case | should | fired | rate | result | note |", + "|---|---|---|---|---|---|", + ] + order = {"FP": 0, "FN": 1, "TP": 2, "TN": 3} + for o in sorted(self.outcomes, key=lambda o: (order[o.kind], o.case.tid)): + should = "yes" if o.case.should_trigger else "no" + lines.append( + f"| `{o.case.tid}` | {should} | {o.fired}/{o.runs} | {o.rate:.2f} | " + f"{o.kind}{' (flaky)' if o.flaky else ''} | {o.case.note} |" + ) + return "\n".join(lines) + "\n" + + +def _fired(result: AgentResult, skill: SkillSpec) -> bool: + if isinstance(result, bool): + return result + if result is None: + return False + if isinstance(result, str): + return result == skill.name + if isinstance(result, Collection): + return skill.name in result + raise TypeError(f"agent must return bool, skill name, names, or None; got {type(result).__name__}") + + +def run_triggers( + cases: Sequence[TriggerCase], + skill: SkillSpec, + agent: TriggerAgent, + *, + k: int = 3, + threshold: float = 0.5, + competing: Sequence[SkillSpec] = (), +) -> TriggerReport: + """Run each case ``k`` times and score trigger precision/recall. + + ``agent(case, offered, run_index)`` gets the skills offered for this run + (``skill`` first, then ``competing``) and returns which skill(s) loaded: + a skill name, a collection of names, ``None``, or a bool meaning + "``skill`` loaded". A case counts as triggered when it fired on at least + ``threshold`` of its runs. + """ + if k < 1: + raise ValueError("k must be >= 1") + if not 0.0 < threshold <= 1.0: + raise ValueError("threshold must be in (0, 1]") + if not cases: + raise ValueError("need at least one case") + seen: set[str] = set() + for c in cases: + if c.tid in seen: + raise ValueError(f"duplicate case id: {c.tid!r}") + seen.add(c.tid) + if any(s.name == skill.name for s in competing): + raise ValueError("competing skills must have different names from the skill under test") + offered = (skill, *competing) + outcomes = [] + for case in cases: + fired = sum(1 for i in range(k) if _fired(agent(case, offered, i), skill)) + outcomes.append(CaseOutcome(case=case, runs=k, fired=fired, threshold=threshold)) + return TriggerReport(skill=skill.name, k=k, threshold=threshold, outcomes=tuple(outcomes)) + + +def from_observations( + cases: Sequence[TriggerCase], + skill: str, + fired: Sequence[Sequence[bool]], + *, + threshold: float = 0.5, +) -> TriggerReport: + """Score pre-recorded runs: ``fired[i]`` is the per-run flags for ``cases[i]``.""" + if len(cases) != len(fired): + raise ValueError("cases and fired must align") + if not 0.0 < threshold <= 1.0: + raise ValueError("threshold must be in (0, 1]") + outcomes = tuple( + CaseOutcome(case=c, runs=len(runs), fired=sum(1 for f in runs if f), threshold=threshold) + for c, runs in zip(cases, fired, strict=True) + ) + ks = {o.runs for o in outcomes} + return TriggerReport(skill=skill, k=max(ks) if ks else 0, threshold=threshold, outcomes=outcomes) + + +def split_cases( + cases: Sequence[TriggerCase], + *, + val_fraction: float = 0.4, + seed: int = 0, +) -> tuple[list[TriggerCase], list[TriggerCase]]: + """Deterministic train/validation split, stratified by ``should_trigger``. + + Iterate on the description against ``train``; report on ``val`` so the + description isn't overfit to the cases you looked at. + """ + if not 0.0 < val_fraction < 1.0: + raise ValueError("val_fraction must be in (0, 1)") + rng = random.Random(seed) + train: list[TriggerCase] = [] + val: list[TriggerCase] = [] + for label in (True, False): + group = [c for c in cases if c.should_trigger is label] + rng.shuffle(group) + n_val = round(len(group) * val_fraction) + if len(group) >= 2: + n_val = min(max(n_val, 1), len(group) - 1) + val.extend(group[:n_val]) + train.extend(group[n_val:]) + key = {c.tid: i for i, c in enumerate(cases)} + return sorted(train, key=lambda c: key[c.tid]), sorted(val, key=lambda c: key[c.tid]) diff --git a/tests/test_trigger.py b/tests/test_trigger.py new file mode 100644 index 0000000..6baf2fb --- /dev/null +++ b/tests/test_trigger.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import pytest + +from skill_eval import ( + SkillSpec, + TriggerCase, + from_observations, + run_triggers, + split_cases, +) + +PDF = SkillSpec("pdf-forms", text="Fill and extract PDF form fields.") +DOCX = SkillSpec("docx", text="Edit Word documents.") + +CASES = [ + TriggerCase("p1", "fill the form fields in invoice.pdf", True), + TriggerCase("p2", "extract the text boxes from this PDF form", True), + TriggerCase("p3", "what fields does tax-form.pdf have?", True), + TriggerCase("n1", "convert report.docx to PDF", False, note="near-miss: mentions PDF"), + TriggerCase("n2", "summarize this PDF article", False, note="near-miss: PDF but no form"), + TriggerCase("n3", "fill in the blanks in this sentence", False, note="near-miss: 'fill'"), +] + + +def greedy_agent(case: TriggerCase, offered, run: int): + """A description that fires on any mention of 'pdf' or 'fill'.""" + text = case.prompt.lower() + return "pdf-forms" if ("pdf" in text or "fill" in text) else None + + +def test_recall_only_hides_false_triggers_precision_exposes_them() -> None: + r = run_triggers(CASES, PDF, greedy_agent, k=3) + assert r.recall == 1.0 # what the A/B trigger rate would report + assert r.fp == 3 and r.tn == 0 + assert r.false_trigger_rate == 1.0 + assert r.precision == 0.5 + assert r.f1 == pytest.approx(0.6667, abs=1e-4) + assert {o.case.tid for o in r.failures()} == {"n1", "n2", "n3"} + + +def test_k_repeats_threshold_and_flaky_cases() -> None: + # p1 fires 2/3 runs, n1 fires 1/3 runs, others deterministic. + pattern = {"p1": [True, True, False], "n1": [False, True, False]} + + def agent(case, offered, run): + if case.tid in pattern: + return pattern[case.tid][run] + return case.should_trigger + + r = run_triggers(CASES, PDF, agent, k=3, threshold=0.5) + assert r.tp == 3 and r.fp == 0 # 2/3 ≥ 0.5 triggers; 1/3 does not + assert sorted(o.case.tid for o in r.flaky) == ["n1", "p1"] + assert r.run_level["positive_fire_rate"] == pytest.approx(8 / 9, abs=1e-4) + assert r.run_level["negative_fire_rate"] == pytest.approx(1 / 9, abs=1e-4) + strict = run_triggers(CASES, PDF, agent, k=3, threshold=1.0) + assert strict.fn == 1 # p1 no longer counts at a strict threshold + + +def test_competing_skills_are_offered_and_names_are_resolved() -> None: + seen = [] + + def agent(case, offered, run): + seen.append(tuple(s.name for s in offered)) + if "docx" in case.prompt: + return "docx" # picked the competitor — not a trigger for pdf-forms + if case.tid == "p2": + return {"pdf-forms", "docx"} # both loaded counts as fired + return case.should_trigger + + r = run_triggers(CASES, PDF, agent, k=2, competing=[DOCX]) + assert set(seen) == {("pdf-forms", "docx")} + assert len(seen) == len(CASES) * 2 + assert r.precision == 1.0 and r.recall == 1.0 and r.false_trigger_rate == 0.0 + + +def test_table_and_dict_list_worst_first() -> None: + r = run_triggers(CASES, PDF, greedy_agent, k=1) + md = r.table() + assert md.index("`n1`") < md.index("`p1`") # false triggers listed first + assert "near-miss: mentions PDF" in md + d = r.as_dict() + assert d["counts"] == {"fn": 0, "fp": 3, "tn": 0, "tp": 3} + assert d["skill"] == "pdf-forms" and d["k"] == 1 + + +def test_from_observations_scores_recorded_runs() -> None: + fired = [[True] * 3, [True, False, True], [False] * 3, [False] * 3, [True] * 3, [False] * 3] + r = from_observations(CASES, "pdf-forms", fired) + assert (r.tp, r.fn, r.fp, r.tn) == (2, 1, 1, 2) + assert r.k == 3 + with pytest.raises(ValueError): + from_observations(CASES, "pdf-forms", fired[:2]) + + +def test_split_cases_is_stratified_and_deterministic() -> None: + train, val = split_cases(CASES, val_fraction=0.34, seed=7) + assert len(train) + len(val) == len(CASES) + assert {c.tid for c in train}.isdisjoint({c.tid for c in val}) + assert any(c.should_trigger for c in val) and any(not c.should_trigger for c in val) + assert any(c.should_trigger for c in train) and any(not c.should_trigger for c in train) + assert split_cases(CASES, val_fraction=0.34, seed=7) == (train, val) + + +def test_validation() -> None: + with pytest.raises(ValueError): + run_triggers(CASES, PDF, greedy_agent, k=0) + with pytest.raises(ValueError): + run_triggers(CASES, PDF, greedy_agent, threshold=0) + with pytest.raises(ValueError): + run_triggers([CASES[0], CASES[0]], PDF, greedy_agent) + with pytest.raises(ValueError): + run_triggers(CASES, PDF, greedy_agent, competing=[PDF]) + with pytest.raises(TypeError): + run_triggers(CASES, PDF, lambda c, o, i: 3.5) # type: ignore[arg-type,return-value] + with pytest.raises(ValueError): + split_cases(CASES, val_fraction=1.0)