diff --git a/README.md b/README.md index 481cf3a..84084c4 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,42 @@ print(t.summary()) `html_report(suite, "report.html")` writes a dark, printable table — pass/fail per case, no login. +## Redaction before disk + +Traces and CI artifacts are where pasted API keys and customer emails end up +living forever. spanlite redacts records **before** any sink or artifact +writer sees them: + +```python +from spanlite import JsonlSink, Tracer +from spanlite.redact import Redactor + +# Default: high-confidence secrets only (provider API keys, GitHub/Slack/AWS +# tokens, JWTs, bearer headers, PEM private keys, values under keys like +# "password" / "authorization"). +t = Tracer("run_01", sinks=[JsonlSink("traces/run.jsonl")]) + +# Opt in to PII (email, Luhn-checked cards, phone, IPv4) and add your own rule: +t = Tracer("run_01", sinks=[JsonlSink("traces/run.jsonl")], + redactor=Redactor.with_pii().extend(("ticket", r"TCK-\d+"))) + +# Opt out entirely: +t = Tracer("run_01", redactor=None) +``` + +- Matches become `[REDACTED:]`. `redactor.counts` shows how many hits each rule had. +- The `suite_artifact` pytest fixture and `html_report` redact too, and accept + the same `redactor=` argument. +- `RedactingSink(inner, redactor)` wraps any sink you call directly. +- **Fail closed:** if redaction raises (for example a `custom=` detector is + down), the record is dropped and counted in `tracer.dropped_records`. It is + never written raw. +- PII is opt-in because those patterns can match ordinary data such as ids, + numbers and versions. Secret patterns are specific enough to stay on by + default. Regex redaction is defense in depth, not a guarantee. +- In-memory `tracer.spans` keep raw values for assertions in tests. Only what + is emitted gets redacted. + ## Usefulness You can fail a pull request when: diff --git a/src/spanlite/pytest_plugin.py b/src/spanlite/pytest_plugin.py index dd18f7b..9c3255c 100644 --- a/src/spanlite/pytest_plugin.py +++ b/src/spanlite/pytest_plugin.py @@ -7,8 +7,11 @@ import pytest from spanlite.evals.base import Suite +from spanlite.redact import Redactor, default_redactor from spanlite.trace import MemorySink, Tracer +_DEFAULT: object = object() + @pytest.fixture def tracer() -> Tracer: @@ -17,16 +20,30 @@ def tracer() -> Tracer: @pytest.fixture -def suite_artifact() -> Callable[[Suite, str | Path], Path | None]: - """Write a readable Suite artifact only when one or more rows have failed.""" - - def write(suite: Suite, path: str | Path) -> Path | None: +def suite_artifact() -> Callable[..., Path | None]: + """Write a readable Suite artifact only when one or more rows have failed. + + The artifact is redacted first (secrets by default; pass + ``redactor=Redactor.with_pii()`` for PII too, or ``redactor=None`` to + write it raw). + """ + + def write( + suite: Suite, + path: str | Path, + *, + redactor: Redactor | None | object = _DEFAULT, + ) -> Path | None: if all(row.passed for row in suite.rows): return None + data = suite.to_json() + active = default_redactor() if redactor is _DEFAULT else redactor + if isinstance(active, Redactor): + data = active.redact(data) artifact = Path(path) artifact.parent.mkdir(parents=True, exist_ok=True) artifact.write_text( - json.dumps(suite.to_json(), indent=2, sort_keys=True) + "\n", + json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) return artifact diff --git a/src/spanlite/redact.py b/src/spanlite/redact.py new file mode 100644 index 0000000..3c7e1b9 --- /dev/null +++ b/src/spanlite/redact.py @@ -0,0 +1,221 @@ +"""Redact secrets and PII before traces and CI artifacts reach disk. + +spanlite writes JSONL traces and CI artifacts, and those files are exactly +where pasted API keys and customer emails end up living forever. A +``Redactor`` walks a record (dicts, lists, strings) and replaces matches with +``[REDACTED:]`` before any sink or artifact writer sees it. + +Defaults are deliberately conservative: + +- ``SECRET_RULES`` are **on by default**. They match high-confidence token + shapes (provider key prefixes, JWTs, bearer headers, PEM private keys), so + false positives on normal agent text are unlikely. +- ``PII_RULES`` (email, phone, IPv4, card numbers) are **opt-in** with + ``Redactor.with_pii()`` because they can match ordinary data (version + strings, ids, numbers) that you may want to keep. +- Values under sensitive keys (``password``, ``api_key``, ``authorization``, + ...) are replaced whole, whatever they look like. + +Regex redaction is best-effort defense in depth, not a guarantee. Keep +secrets out of prompts and attrs where you can. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Callable, Iterable, Mapping + + +@dataclass(frozen=True) +class Rule: + """A named regex. Matches become ``replacement.format(name=name)``.""" + + name: str + pattern: re.Pattern[str] + + @classmethod + def compile(cls, name: str, pattern: str, flags: int = 0) -> "Rule": + return cls(name, re.compile(pattern, flags)) + + +# High-confidence secret shapes. Order matters: specific before generic. +SECRET_RULES: tuple[Rule, ...] = ( + Rule.compile( + "private_key", + r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----", + ), + Rule.compile("jwt", r"\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"), + Rule.compile("bearer", r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{16,}"), + Rule.compile("anthropic_key", r"\bsk-ant-[A-Za-z0-9_-]{20,}"), + Rule.compile("openai_key", r"\bsk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}"), + Rule.compile("github_token", r"\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{40,})"), + Rule.compile("slack_token", r"\bxox[abposr]-[A-Za-z0-9-]{10,}"), + Rule.compile("aws_access_key", r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), + Rule.compile("google_api_key", r"\bAIza[0-9A-Za-z_-]{35}\b"), + Rule.compile("stripe_key", r"\b(?:sk|rk)_(?:live|test)_[0-9A-Za-z]{16,}"), + Rule.compile("hf_token", r"\bhf_[A-Za-z0-9]{30,}"), + Rule.compile("xai_key", r"\bxai-[A-Za-z0-9]{30,}"), +) + +# Broader PII shapes; opt-in because they can match ordinary data. +PII_RULES: tuple[Rule, ...] = ( + Rule.compile("email", r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), + Rule.compile("card", r"\b(?:\d[ -]?){13,19}\b"), + Rule.compile("phone", r"(? bool: + nums = [int(c) for c in digits if c.isdigit()] + if not 13 <= len(nums) <= 19: + return False + total = 0 + for i, n in enumerate(reversed(nums)): + if i % 2 == 1: + n *= 2 + if n > 9: + n -= 9 + total += n + return total % 10 == 0 + + +@dataclass(frozen=True) +class Redactor: + """Deep-redacts strings inside JSON-like values. + + ``custom`` runs after the regex rules on every string (e.g. a project + specific id format, or a call into a heavier PII detector you own). + """ + + rules: tuple[Rule, ...] = SECRET_RULES + secret_keys: frozenset[str] = DEFAULT_SECRET_KEYS + replacement: str = "[REDACTED:{name}]" + custom: Callable[[str], str] | None = None + _counts: dict[str, int] = field(default_factory=dict, compare=False, repr=False) + + @classmethod + def with_pii(cls, **kw: Any) -> "Redactor": + """Secrets plus email / card (Luhn-checked) / phone / IPv4.""" + return cls(rules=SECRET_RULES + PII_RULES, **kw) + + def extend(self, *rules: Rule | tuple[str, str]) -> "Redactor": + """Return a copy with extra rules, e.g. ``extend(("ticket", r"TCK-\\d+"))``.""" + extra = tuple(r if isinstance(r, Rule) else Rule.compile(*r) for r in rules) + return Redactor( + rules=self.rules + extra, + secret_keys=self.secret_keys, + replacement=self.replacement, + custom=self.custom, + ) + + @property + def counts(self) -> dict[str, int]: + """How many replacements each rule made so far (for a CI summary).""" + return dict(self._counts) + + def _bump(self, name: str, n: int = 1) -> None: + self._counts[name] = self._counts.get(name, 0) + n + + def text(self, value: str) -> str: + """Redact one string.""" + out = value + for rule in self.rules: + label = self.replacement.format(name=rule.name) + if rule.name == "card": + + def card_sub(m: re.Match[str], _label: str = label) -> str: + if _luhn_ok(m.group(0)): + self._bump("card") + return _label + return m.group(0) + + out = rule.pattern.sub(card_sub, out) + continue + out, n = rule.pattern.subn(label, out) + if n: + self._bump(rule.name, n) + if self.custom is not None: + out = self.custom(out) + return out + + def _is_secret_key(self, key: Any) -> bool: + return isinstance(key, str) and key.strip().lower().replace(" ", "_") in self.secret_keys + + def redact(self, value: Any) -> Any: + """Return a redacted deep copy of ``value`` (dict/list/tuple/str/scalars).""" + if isinstance(value, str): + return self.text(value) + if isinstance(value, Mapping): + out: dict[Any, Any] = {} + for k, v in value.items(): + if self._is_secret_key(k) and isinstance(v, str) and v: + self._bump("secret_key") + out[k] = self.replacement.format(name="secret_key") + else: + out[k] = self.redact(v) + return out + if isinstance(value, (list, tuple)): + items = [self.redact(v) for v in value] + return items if isinstance(value, list) else tuple(items) + return value + + +def default_redactor() -> Redactor: + """A fresh secrets-only redactor (the Tracer / artifact default).""" + return Redactor() + + +def redact(value: Any, redactor: Redactor | None = None) -> Any: + """Convenience: redact ``value`` with ``redactor`` (default: secrets only).""" + return (redactor or default_redactor()).redact(value) + + +class RedactingSink: + """Decorator sink: redact every record, then forward to ``inner``. + + Fails closed: if redaction raises, the record is dropped and counted in + ``dropped`` instead of being written unredacted. + """ + + def __init__(self, inner: Any, redactor: Redactor | None = None) -> None: + self.inner = inner + self.redactor = redactor or default_redactor() + self.dropped = 0 + + def emit(self, record: Mapping[str, Any]) -> None: + try: + clean = self.redactor.redact(record) + except Exception: + self.dropped += 1 + return + self.inner.emit(clean) + + +def rules_by_name(names: Iterable[str]) -> tuple[Rule, ...]: + """Pick built-in rules by name, e.g. ``rules_by_name(["email", "openai_key"])``.""" + table = {r.name: r for r in SECRET_RULES + PII_RULES} + missing = [n for n in names if n not in table] + if missing: + raise KeyError(f"unknown redaction rule(s): {', '.join(missing)}") + return tuple(table[n] for n in names) diff --git a/src/spanlite/report.py b/src/spanlite/report.py index d98b280..03c79da 100644 --- a/src/spanlite/report.py +++ b/src/spanlite/report.py @@ -4,6 +4,9 @@ from pathlib import Path from spanlite.evals.base import Suite +from spanlite.redact import Redactor, default_redactor + +_DEFAULT: object = object() def summary_table(suite: Suite) -> str: @@ -19,19 +22,28 @@ def summary_table(suite: Suite) -> str: return "\n".join(lines) -def html_report(suite: Suite, path: str | Path | None = None) -> str: +def html_report( + suite: Suite, + path: str | Path | None = None, + *, + redactor: Redactor | None | object = _DEFAULT, +) -> str: + """Render a static HTML report. Case ids and score names are redacted + (secrets by default; ``redactor=None`` to disable).""" + active = default_redactor() if redactor is _DEFAULT else redactor + clean = active.text if isinstance(active, Redactor) else (lambda s: s) rows = [] for row in suite.rows: cells = "".join( - f"{escape(s.name)} {s.value}" + f"{escape(clean(s.name))} {s.value}" for s in row.scores ) rows.append( - f"{escape(row.case_id)}" + f"{escape(clean(row.case_id))}" f"{'pass' if row.passed else 'fail'}{cells}" ) html = f""" -{escape(suite.name)} +{escape(clean(suite.name))} -

{escape(suite.name)}

+

{escape(clean(suite.name))}

{len(suite.rows)} cases · pass {suite.pass_rate():.0%}

{''.join(rows)}
caseresultscores
diff --git a/src/spanlite/trace.py b/src/spanlite/trace.py index 8970893..fa40f1a 100644 --- a/src/spanlite/trace.py +++ b/src/spanlite/trace.py @@ -9,6 +9,8 @@ from pathlib import Path from typing import Any, Iterator, Mapping, Protocol +from spanlite.redact import Redactor, default_redactor + _PARENT: ContextVar[str | None] = ContextVar("spanlite_parent", default=None) @@ -76,10 +78,21 @@ def ms(self) -> float: return round((end - self.t0) * 1000, 3) +_DEFAULT_REDACTOR: Any = object() + + class Tracer: """Facade over nested spans. Inject Clock and IdFactory in tests. Never sleeps. Never talks to a vendor. + + Every record is passed through ``redactor`` before any sink sees it. The + default redacts high-confidence secrets only (API keys, tokens, JWTs, + private keys, values under keys like ``password``); use + ``Redactor.with_pii()`` to also mask emails/phones/cards/IPs, or + ``redactor=None`` to turn redaction off. If redaction raises, the record + is dropped (``dropped_records``), never written raw. In-memory ``spans`` + keep the original values. """ def __init__( @@ -92,6 +105,7 @@ def __init__( clock: Clock | None = None, ids: IdFactory | None = None, sinks: list[Sink] | None = None, + redactor: Redactor | None = _DEFAULT_REDACTOR, ) -> None: self.run_id = run_id self.model = model @@ -101,8 +115,19 @@ def __init__( self._ids = ids or UuidFactory() self._sinks = sinks or [MemorySink()] self.spans: list[Span] = [] + self.redactor: Redactor | None = ( + default_redactor() if redactor is _DEFAULT_REDACTOR else redactor + ) + self.dropped_records = 0 def _emit(self, record: Mapping[str, Any]) -> None: + if self.redactor is not None: + try: + record = self.redactor.redact(record) + except Exception: + # Fail closed: never write a record we could not redact. + self.dropped_records += 1 + return for sink in self._sinks: sink.emit(record) diff --git a/tests/test_redact.py b/tests/test_redact.py new file mode 100644 index 0000000..08a285c --- /dev/null +++ b/tests/test_redact.py @@ -0,0 +1,189 @@ +"""Redaction tests. All secrets are synthetic and built at runtime so no +token-shaped literal lives in the repo.""" + +from __future__ import annotations + +import json + +import pytest + +from spanlite import AgentCase, JsonlSink, MemorySink, Output, Suite, TaskJudge, Tracer, html_report +from spanlite.redact import ( + PII_RULES, + SECRET_RULES, + RedactingSink, + Redactor, + Rule, + redact, + rules_by_name, +) + +OPENAI = "sk-" + "proj-" + "A1b2C3d4" * 4 +ANTHROPIC = "sk-" + "ant-" + "api03-" + "Z9y8X7w6" * 4 +GITHUB = "gh" + "p_" + "a" * 36 +AWS = "AK" + "IA" + "ABCDEFGHIJKLMNOP" +SLACK = "xo" + "xb-" + "1234567890-abcdefghij" +GOOGLE = "AI" + "za" + "S" * 35 +JWT = "ey" + "JhbGciOiJIUzI1NiJ9" + ".ey" + "JzdWIiOiIxMjM0NSJ9" + "." + "s1gn4tur3_abcdefgh" +PEM = "-----BEGIN " + "RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKj\n-----END RSA PRIVATE KEY-----" +EMAIL = "ada" + "@" + "example.com" +CARD = "4111 1111 1111 1111" # Luhn-valid test number + + +class Clock: + def __init__(self) -> None: + self.t = 0.0 + + def now(self) -> float: + self.t += 0.01 + return self.t + + +def ids(): + n = iter(range(1, 10_000)) + return lambda: f"s{next(n)}" + + +@pytest.mark.parametrize( + "secret,rule", + [ + (OPENAI, "openai_key"), + (ANTHROPIC, "anthropic_key"), + (GITHUB, "github_token"), + (AWS, "aws_access_key"), + (SLACK, "slack_token"), + (GOOGLE, "google_api_key"), + (JWT, "jwt"), + (PEM, "private_key"), + ("Bearer " + "abcDEF123456ghiJKL789", "bearer"), + ], +) +def test_default_rules_catch_secret_shapes(secret: str, rule: str) -> None: + r = Redactor() + out = r.text(f"calling api with {secret} now") + assert secret not in out + assert f"[REDACTED:{rule}]" in out + assert r.counts.get(rule) == 1 + + +def test_defaults_leave_ordinary_agent_text_and_pii_alone() -> None: + text = ( + f"Paris is the capital. v1.2.3 took 1500 ms, task-skeleton-key, " + f"see https://example.com/docs, contact {EMAIL}, 10.0.0.1" + ) + assert Redactor().text(text) == text + + +def test_with_pii_masks_email_card_phone_ip_and_checks_luhn() -> None: + r = Redactor.with_pii() + out = r.text(f"mail {EMAIL}, card {CARD}, call +1 415-555-0100, host 10.0.0.1") + assert EMAIL not in out and "[REDACTED:email]" in out + assert "[REDACTED:card]" in out + assert "[REDACTED:phone]" in out + assert "[REDACTED:ipv4]" in out + # A 16-digit number that fails Luhn is not treated as a card. + assert "[REDACTED:card]" not in Redactor(rules=rules_by_name(["card"])).text("id 1234567812345678") + + +def test_deep_redaction_and_secret_keys() -> None: + record = { + "attrs": { + "prompt": f"my key is {OPENAI}", + "headers": {"Authorization": "Basic dXNlcjpwYXNz", "Accept": "json"}, + "password": "hunter2", + "token": 42, # non-string under a sensitive key is left alone + "history": [f"jwt={JWT}", ("tuple", OPENAI)], + }, + "tokens_in": 10, + } + out = redact(record) + blob = json.dumps(out) + for raw in (OPENAI, JWT, "hunter2", "dXNlcjpwYXNz"): + assert raw not in blob + assert out["attrs"]["headers"]["Accept"] == "json" + assert out["attrs"]["token"] == 42 + assert isinstance(out["attrs"]["history"][1], tuple) + assert record["attrs"]["password"] == "hunter2" # input not mutated + + +def test_tracer_redacts_before_jsonl_sink(tmp_path) -> None: + path = tmp_path / "trace.jsonl" + mem = MemorySink() + t = Tracer("r1", clock=Clock(), ids=ids(), sinks=[JsonlSink(path), mem]) + with t.span("answer", "llm", prompt=f"use {OPENAI} and {GITHUB}", user=EMAIL): + pass + disk = path.read_text(encoding="utf-8") + assert OPENAI not in disk and GITHUB not in disk + assert "[REDACTED:openai_key]" in disk + assert EMAIL in disk # PII is opt-in + assert mem.records[0]["attrs"]["prompt"] == "use [REDACTED:openai_key] and [REDACTED:github_token]" + assert t.spans[0].attrs["prompt"].startswith("use sk-") # in-memory span untouched + + +def test_tracer_pii_opt_in_custom_rule_and_opt_out(tmp_path) -> None: + mem = MemorySink() + r = Redactor.with_pii().extend(("ticket", r"TCK-\d+")) + t = Tracer("r2", clock=Clock(), ids=ids(), sinks=[mem], redactor=r) + with t.span("tool", "tool", tool="crm", q=f"{EMAIL} TCK-123"): + pass + assert mem.records[0]["attrs"]["q"] == "[REDACTED:email] [REDACTED:ticket]" + + raw = MemorySink() + t2 = Tracer("r3", clock=Clock(), ids=ids(), sinks=[raw], redactor=None) + with t2.span("x", prompt=OPENAI): + pass + assert raw.records[0]["attrs"]["prompt"] == OPENAI + + +def test_tracer_fails_closed_when_redaction_raises() -> None: + def boom(_s: str) -> str: + raise RuntimeError("detector down") + + mem = MemorySink() + t = Tracer("r4", clock=Clock(), ids=ids(), sinks=[mem], redactor=Redactor(custom=boom)) + with t.span("x", prompt="hello"): + pass + assert mem.records == [] + assert t.dropped_records == 1 + + +def test_redacting_sink_wrapper_fails_closed() -> None: + inner = MemorySink() + sink = RedactingSink(inner) + sink.emit({"prompt": OPENAI}) + assert inner.records == [{"prompt": "[REDACTED:openai_key]"}] + bad = RedactingSink(inner, Redactor(custom=lambda s: 1 / 0)) # type: ignore[arg-type,return-value] + bad.emit({"prompt": "x"}) + assert bad.dropped == 1 and len(inner.records) == 1 + + +def _failing_suite(case_id: str) -> Suite: + suite = Suite("agent", [TaskJudge()]) + suite.run([AgentCase(case_id, "capital", expect="Delhi")], lambda _c: Output(text="Mumbai")) + return suite + + +def test_suite_artifact_is_redacted_by_default(tmp_path, suite_artifact) -> None: + suite = _failing_suite(f"leak-{OPENAI}") + path = suite_artifact(suite, tmp_path / "a.json") + text = path.read_text(encoding="utf-8") + assert OPENAI not in text and "[REDACTED:openai_key]" in text + raw = suite_artifact(suite, tmp_path / "raw.json", redactor=None) + assert OPENAI in raw.read_text(encoding="utf-8") + + +def test_html_report_redacts_case_ids(tmp_path) -> None: + suite = _failing_suite(f"user-{EMAIL}-{OPENAI}") + html = html_report(suite, tmp_path / "r.html", redactor=Redactor.with_pii()) + on_disk = (tmp_path / "r.html").read_text(encoding="utf-8") + for doc in (html, on_disk): + assert OPENAI not in doc and EMAIL not in doc + + +def test_rule_helpers() -> None: + assert {r.name for r in rules_by_name(["email", "jwt"])} == {"email", "jwt"} + with pytest.raises(KeyError): + rules_by_name(["nope"]) + assert not {r.name for r in SECRET_RULES} & {r.name for r in PII_RULES} + custom = Redactor(rules=(Rule.compile("order", r"ORD\d{4}"),), replacement="***") + assert custom.text("ORD1234 shipped") == "*** shipped"