Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,42 @@ print(t.summary())

`html_report(suite, "report.html")` writes a dark, printable table — pass/fail per case, no login.

## Redaction before disk

Traces and CI artifacts are where pasted API keys and customer emails end up
living forever. spanlite redacts records **before** any sink or artifact
writer sees them:

```python
from spanlite import JsonlSink, Tracer
from spanlite.redact import Redactor

# Default: high-confidence secrets only (provider API keys, GitHub/Slack/AWS
# tokens, JWTs, bearer headers, PEM private keys, values under keys like
# "password" / "authorization").
t = Tracer("run_01", sinks=[JsonlSink("traces/run.jsonl")])

# Opt in to PII (email, Luhn-checked cards, phone, IPv4) and add your own rule:
t = Tracer("run_01", sinks=[JsonlSink("traces/run.jsonl")],
redactor=Redactor.with_pii().extend(("ticket", r"TCK-\d+")))

# Opt out entirely:
t = Tracer("run_01", redactor=None)
```

- Matches become `[REDACTED:<rule>]`. `redactor.counts` shows how many hits each rule had.
- The `suite_artifact` pytest fixture and `html_report` redact too, and accept
the same `redactor=` argument.
- `RedactingSink(inner, redactor)` wraps any sink you call directly.
- **Fail closed:** if redaction raises (for example a `custom=` detector is
down), the record is dropped and counted in `tracer.dropped_records`. It is
never written raw.
- PII is opt-in because those patterns can match ordinary data such as ids,
numbers and versions. Secret patterns are specific enough to stay on by
default. Regex redaction is defense in depth, not a guarantee.
- In-memory `tracer.spans` keep raw values for assertions in tests. Only what
is emitted gets redacted.

## Usefulness

You can fail a pull request when:
Expand Down
27 changes: 22 additions & 5 deletions src/spanlite/pytest_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@
import pytest

from spanlite.evals.base import Suite
from spanlite.redact import Redactor, default_redactor
from spanlite.trace import MemorySink, Tracer

_DEFAULT: object = object()


@pytest.fixture
def tracer() -> Tracer:
Expand All @@ -17,16 +20,30 @@ def tracer() -> Tracer:


@pytest.fixture
def suite_artifact() -> Callable[[Suite, str | Path], Path | None]:
"""Write a readable Suite artifact only when one or more rows have failed."""

def write(suite: Suite, path: str | Path) -> Path | None:
def suite_artifact() -> Callable[..., Path | None]:
"""Write a readable Suite artifact only when one or more rows have failed.

The artifact is redacted first (secrets by default; pass
``redactor=Redactor.with_pii()`` for PII too, or ``redactor=None`` to
write it raw).
"""

def write(
suite: Suite,
path: str | Path,
*,
redactor: Redactor | None | object = _DEFAULT,
) -> Path | None:
if all(row.passed for row in suite.rows):
return None
data = suite.to_json()
active = default_redactor() if redactor is _DEFAULT else redactor
if isinstance(active, Redactor):
data = active.redact(data)
artifact = Path(path)
artifact.parent.mkdir(parents=True, exist_ok=True)
artifact.write_text(
json.dumps(suite.to_json(), indent=2, sort_keys=True) + "\n",
json.dumps(data, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return artifact
Expand Down
221 changes: 221 additions & 0 deletions src/spanlite/redact.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
"""Redact secrets and PII before traces and CI artifacts reach disk.

spanlite writes JSONL traces and CI artifacts, and those files are exactly
where pasted API keys and customer emails end up living forever. A
``Redactor`` walks a record (dicts, lists, strings) and replaces matches with
``[REDACTED:<rule>]`` before any sink or artifact writer sees it.

Defaults are deliberately conservative:

- ``SECRET_RULES`` are **on by default**. They match high-confidence token
shapes (provider key prefixes, JWTs, bearer headers, PEM private keys), so
false positives on normal agent text are unlikely.
- ``PII_RULES`` (email, phone, IPv4, card numbers) are **opt-in** with
``Redactor.with_pii()`` because they can match ordinary data (version
strings, ids, numbers) that you may want to keep.
- Values under sensitive keys (``password``, ``api_key``, ``authorization``,
...) are replaced whole, whatever they look like.

Regex redaction is best-effort defense in depth, not a guarantee. Keep
secrets out of prompts and attrs where you can.
"""

from __future__ import annotations

import re
from dataclasses import dataclass, field
from typing import Any, Callable, Iterable, Mapping


@dataclass(frozen=True)
class Rule:
"""A named regex. Matches become ``replacement.format(name=name)``."""

name: str
pattern: re.Pattern[str]

@classmethod
def compile(cls, name: str, pattern: str, flags: int = 0) -> "Rule":
return cls(name, re.compile(pattern, flags))


# High-confidence secret shapes. Order matters: specific before generic.
SECRET_RULES: tuple[Rule, ...] = (
Rule.compile(
"private_key",
r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----",
),
Rule.compile("jwt", r"\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"),
Rule.compile("bearer", r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{16,}"),
Rule.compile("anthropic_key", r"\bsk-ant-[A-Za-z0-9_-]{20,}"),
Rule.compile("openai_key", r"\bsk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}"),
Rule.compile("github_token", r"\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{40,})"),
Rule.compile("slack_token", r"\bxox[abposr]-[A-Za-z0-9-]{10,}"),
Rule.compile("aws_access_key", r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"),
Rule.compile("google_api_key", r"\bAIza[0-9A-Za-z_-]{35}\b"),
Rule.compile("stripe_key", r"\b(?:sk|rk)_(?:live|test)_[0-9A-Za-z]{16,}"),
Rule.compile("hf_token", r"\bhf_[A-Za-z0-9]{30,}"),
Rule.compile("xai_key", r"\bxai-[A-Za-z0-9]{30,}"),
)

# Broader PII shapes; opt-in because they can match ordinary data.
PII_RULES: tuple[Rule, ...] = (
Rule.compile("email", r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"),
Rule.compile("card", r"\b(?:\d[ -]?){13,19}\b"),
Rule.compile("phone", r"(?<![\w.])\+?\d{1,3}[ .-]?\(?\d{2,4}\)?[ .-]?\d{3,4}[ .-]?\d{3,4}(?![\w.])"),
Rule.compile("ipv4", r"\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b"),
)

DEFAULT_SECRET_KEYS: frozenset[str] = frozenset(
{
"api_key",
"apikey",
"access_token",
"authorization",
"client_secret",
"cookie",
"password",
"passwd",
"private_key",
"refresh_token",
"secret",
"set-cookie",
"token",
"x-api-key",
}
)


def _luhn_ok(digits: str) -> bool:
nums = [int(c) for c in digits if c.isdigit()]
if not 13 <= len(nums) <= 19:
return False
total = 0
for i, n in enumerate(reversed(nums)):
if i % 2 == 1:
n *= 2
if n > 9:
n -= 9
total += n
return total % 10 == 0


@dataclass(frozen=True)
class Redactor:
"""Deep-redacts strings inside JSON-like values.

``custom`` runs after the regex rules on every string (e.g. a project
specific id format, or a call into a heavier PII detector you own).
"""

rules: tuple[Rule, ...] = SECRET_RULES
secret_keys: frozenset[str] = DEFAULT_SECRET_KEYS
replacement: str = "[REDACTED:{name}]"
custom: Callable[[str], str] | None = None
_counts: dict[str, int] = field(default_factory=dict, compare=False, repr=False)

@classmethod
def with_pii(cls, **kw: Any) -> "Redactor":
"""Secrets plus email / card (Luhn-checked) / phone / IPv4."""
return cls(rules=SECRET_RULES + PII_RULES, **kw)

def extend(self, *rules: Rule | tuple[str, str]) -> "Redactor":
"""Return a copy with extra rules, e.g. ``extend(("ticket", r"TCK-\\d+"))``."""
extra = tuple(r if isinstance(r, Rule) else Rule.compile(*r) for r in rules)
return Redactor(
rules=self.rules + extra,
secret_keys=self.secret_keys,
replacement=self.replacement,
custom=self.custom,
)

@property
def counts(self) -> dict[str, int]:
"""How many replacements each rule made so far (for a CI summary)."""
return dict(self._counts)

def _bump(self, name: str, n: int = 1) -> None:
self._counts[name] = self._counts.get(name, 0) + n

def text(self, value: str) -> str:
"""Redact one string."""
out = value
for rule in self.rules:
label = self.replacement.format(name=rule.name)
if rule.name == "card":

def card_sub(m: re.Match[str], _label: str = label) -> str:
if _luhn_ok(m.group(0)):
self._bump("card")
return _label
return m.group(0)

out = rule.pattern.sub(card_sub, out)
continue
out, n = rule.pattern.subn(label, out)
if n:
self._bump(rule.name, n)
if self.custom is not None:
out = self.custom(out)
return out

def _is_secret_key(self, key: Any) -> bool:
return isinstance(key, str) and key.strip().lower().replace(" ", "_") in self.secret_keys

def redact(self, value: Any) -> Any:
"""Return a redacted deep copy of ``value`` (dict/list/tuple/str/scalars)."""
if isinstance(value, str):
return self.text(value)
if isinstance(value, Mapping):
out: dict[Any, Any] = {}
for k, v in value.items():
if self._is_secret_key(k) and isinstance(v, str) and v:
self._bump("secret_key")
out[k] = self.replacement.format(name="secret_key")
else:
out[k] = self.redact(v)
return out
if isinstance(value, (list, tuple)):
items = [self.redact(v) for v in value]
return items if isinstance(value, list) else tuple(items)
return value


def default_redactor() -> Redactor:
"""A fresh secrets-only redactor (the Tracer / artifact default)."""
return Redactor()


def redact(value: Any, redactor: Redactor | None = None) -> Any:
"""Convenience: redact ``value`` with ``redactor`` (default: secrets only)."""
return (redactor or default_redactor()).redact(value)


class RedactingSink:
"""Decorator sink: redact every record, then forward to ``inner``.

Fails closed: if redaction raises, the record is dropped and counted in
``dropped`` instead of being written unredacted.
"""

def __init__(self, inner: Any, redactor: Redactor | None = None) -> None:
self.inner = inner
self.redactor = redactor or default_redactor()
self.dropped = 0

def emit(self, record: Mapping[str, Any]) -> None:
try:
clean = self.redactor.redact(record)
except Exception:
self.dropped += 1
return
self.inner.emit(clean)


def rules_by_name(names: Iterable[str]) -> tuple[Rule, ...]:
"""Pick built-in rules by name, e.g. ``rules_by_name(["email", "openai_key"])``."""
table = {r.name: r for r in SECRET_RULES + PII_RULES}
missing = [n for n in names if n not in table]
if missing:
raise KeyError(f"unknown redaction rule(s): {', '.join(missing)}")
return tuple(table[n] for n in names)
22 changes: 17 additions & 5 deletions src/spanlite/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
from pathlib import Path

from spanlite.evals.base import Suite
from spanlite.redact import Redactor, default_redactor

_DEFAULT: object = object()


def summary_table(suite: Suite) -> str:
Expand All @@ -19,19 +22,28 @@ def summary_table(suite: Suite) -> str:
return "\n".join(lines)


def html_report(suite: Suite, path: str | Path | None = None) -> str:
def html_report(
suite: Suite,
path: str | Path | None = None,
*,
redactor: Redactor | None | object = _DEFAULT,
) -> str:
"""Render a static HTML report. Case ids and score names are redacted
(secrets by default; ``redactor=None`` to disable)."""
active = default_redactor() if redactor is _DEFAULT else redactor
clean = active.text if isinstance(active, Redactor) else (lambda s: s)
rows = []
for row in suite.rows:
cells = "".join(
f"<td class='{'ok' if s.passed else 'bad'}'>{escape(s.name)} {s.value}</td>"
f"<td class='{'ok' if s.passed else 'bad'}'>{escape(clean(s.name))} {s.value}</td>"
for s in row.scores
)
rows.append(
f"<tr class='{'ok' if row.passed else 'bad'}'><td>{escape(row.case_id)}</td>"
f"<tr class='{'ok' if row.passed else 'bad'}'><td>{escape(clean(row.case_id))}</td>"
f"<td>{'pass' if row.passed else 'fail'}</td>{cells}</tr>"
)
html = f"""<!doctype html>
<html><head><meta charset=\"utf-8\"><title>{escape(suite.name)}</title>
<html><head><meta charset=\"utf-8\"><title>{escape(clean(suite.name))}</title>
<style>
body{{font:14px/1.45 ui-sans-serif,system-ui;background:#0e1210;color:#e8ebe4;margin:32px}}
h1{{font-weight:500}} table{{border-collapse:collapse;width:100%}}
Expand All @@ -40,7 +52,7 @@ def html_report(suite: Suite, path: str | Path | None = None) -> str:
.meta{{color:#8b9388}}
</style></head>
<body>
<h1>{escape(suite.name)}</h1>
<h1>{escape(clean(suite.name))}</h1>
<p class=\"meta\">{len(suite.rows)} cases · pass {suite.pass_rate():.0%}</p>
<table><thead><tr><th>case</th><th>result</th><th colspan=\"8\">scores</th></tr></thead>
<tbody>{''.join(rows)}</tbody></table>
Expand Down
Loading
Loading