diff --git a/.agents/skills/sdk-integrations/SKILL.md b/.agents/skills/sdk-integrations/SKILL.md index 8dc34fcb..54f1d406 100644 --- a/.agents/skills/sdk-integrations/SKILL.md +++ b/.agents/skills/sdk-integrations/SKILL.md @@ -5,331 +5,262 @@ description: Create or update Braintrust Python SDK integrations built on the in # SDK Integrations -Use this skill for integration work under `py/src/braintrust/integrations/`. +Work under `py/src/braintrust/integrations/`. Use `sdk-wrapper-migrations` instead if the task is moving a real implementation from `py/src/braintrust/wrappers//` into the integrations API. -Use `sdk-wrapper-migrations` instead when the provider already has a real implementation under `py/src/braintrust/wrappers//` and the task is to move that implementation into the integrations API. +## Spec -## Quick Start +Authoritative source for span shape, allowed fields, and instrumentation rules: https://github.com/braintrustdata/braintrust-spec/blob/main/docs/instrumentation-guide.md -Before editing: - -1. Read the shared integration primitives. -2. Read the target provider package. -3. Pick the nearest existing integration as a reference. -4. Confirm provider versions and nox sessions from source files, not memory. -5. Decide the span shape before writing patchers. -6. Run the narrowest provider nox session first. - -Do not design a new integration shape from scratch if an existing provider already matches the problem. +Read it first for anything involving span type/name, `input`/`output`/`metadata`/`metrics`/`error`/`context` shape, tool calls, prompt provenance, streaming, reasoning models, prompt caching, multimodal, completion-vs-agentic, or new attribute namespacing. Do not invent new top-level span fields, metric keys, or span types; update the spec first. ## Read First -Always read: - -- `py/src/braintrust/integrations/base.py` -- `py/src/braintrust/integrations/versioning.py` -- `py/src/braintrust/integrations/__init__.py` -- `py/src/braintrust/integrations/utils.py` -- `py/pyproject.toml` for provider matrix pins and cassette directory mappings -- `py/noxfile.py` - -Read these when working on an existing integration: +Always: `integrations/base.py`, `integrations/versioning.py`, `integrations/__init__.py`, `integrations/utils.py`, `pyproject.toml`, `noxfile.py`. -- `py/src/braintrust/integrations//__init__.py` -- `py/src/braintrust/integrations//integration.py` -- `py/src/braintrust/integrations//patchers.py` -- `py/src/braintrust/integrations//tracing.py` -- `py/src/braintrust/integrations//test_*.py` +Existing integration: `integrations//{__init__,integration,patchers,tracing,test_*}.py`. -Read these when relevant: +Situational: +- `auto.py` + `integrations/auto_test_scripts/` — anything touching `auto_instrument()` or import timing +- `conftest.py`, `integrations/conftest.py` — VCR / per-version cassette resolution +- `integrations/test_utils.py` — shared attachment / multimodal helpers +- `integrations/{adk,anthropic,google_genai}/tracing.py` — attachment + multimodal reference implementations -- `py/src/braintrust/auto.py` for `auto_instrument()` changes -- `py/src/braintrust/conftest.py` for VCR behavior -- `py/src/braintrust/integrations/conftest.py` for per-version cassette directory resolution -- `py/src/braintrust/integrations/auto_test_scripts/` for subprocess auto-instrument coverage -- `py/src/braintrust/integrations/test_utils.py` when touching shared attachment materialization or multimodal payload shaping -- `py/src/braintrust/integrations/adk/test_adk.py`, `py/src/braintrust/integrations/anthropic/test_anthropic.py`, and `py/src/braintrust/integrations/google_genai/test_google_genai.py` for attachment-focused test layout patterns -- `py/src/braintrust/integrations/adk/tracing.py`, `py/src/braintrust/integrations/anthropic/tracing.py`, and `py/src/braintrust/integrations/google_genai/tracing.py` when handling multimodal content, binary inputs, generated media, or attachment materialization behavior - -Do not forget `auto.py` and `auto_test_scripts/`. Import-order and subprocess regressions often only show up there. +Import-order and subprocess regressions usually only surface via `auto_test_scripts/`. Do not skip it. ## Version And CI Routing -Do not guess which provider versions or sessions apply. - -Use these files as the routing chain: +Never guess versions or session names. The routing chain: -- `py/pyproject.toml` `[tool.braintrust.matrix]`: supported provider versions and what `latest` resolves to -- `py/pyproject.toml` `[tool.braintrust.cassette-dirs]`: versioned cassette directory ownership -- `py/src/braintrust/integrations/versioning.py`: supported version helpers and gates -- `py/noxfile.py`: actual session names, package installation, and `BRAINTRUST_TEST_PACKAGE_VERSION` -- `.github/workflows/checks.yaml`: CI matrix and which sessions run in shards or static checks +- `pyproject.toml` `[tool.braintrust.matrix]` — supported provider versions, what `latest` resolves to +- `pyproject.toml` `[tool.braintrust.cassette-dirs]` — versioned cassette directory ownership +- `integrations/versioning.py` — supported version helpers, gates +- `noxfile.py` — session names, package install, `BRAINTRUST_TEST_PACKAGE_VERSION` +- `.github/workflows/checks.yaml` — CI matrix + shard membership -When changing version-gated behavior: - -1. Identify every matrix version for the provider. -2. Check whether the integration has `min_version`, `max_version`, `superseded_by`, or feature-detection branches. -3. Test the narrowest affected version first. -4. Add or update cassettes only for versions whose observable provider behavior intentionally changed. +When changing version-gated behavior: identify every matrix version, check `min_version`/`max_version`/`superseded_by`/feature-detect branches, test the narrowest affected version first, re-record cassettes only for versions whose observable provider behavior intentionally changed. ## Pick A Reference -Start from the nearest current integration: - -- ADK: direct method patching, `target_module`, `CompositeFunctionWrapperPatcher`, manual `wrap_*()` helpers, context propagation, inline data to `Attachment` -- Agno: multi-target patching, several related patchers, version-conditional fallbacks with `superseded_by` -- Anthropic: compact constructor patching, a small public surface, and multimodal request blocks that distinguish image vs document attachment payloads -- Google GenAI: multimodal tracing, generated media, output-side `Attachment` handling, and nested attachment materialization while preserving non-attachment values +Start from the nearest existing integration: -Choose the reference based on the hardest part of the task: - -- patcher topology -- tracing shape -- streaming behavior -- multimodal or binary payload handling +- **ADK** — direct method patching, `target_module`, `CompositeFunctionWrapperPatcher`, manual `wrap_*()`, context propagation, inline data → `Attachment` +- **Agno** — multi-target patching, related patchers, version-conditional fallbacks with `superseded_by` +- **Anthropic** — compact constructor patching, small public surface, multimodal image-vs-document attachment payloads +- **Google GenAI** — multimodal tracing, generated media, output-side attachments, nested materialization that preserves non-attachment values ## Default Workflow -Use this order unless the task is clearly narrower: - -1. Read shared primitives and the provider package. -2. Decide which public surface is being patched. -3. Define the span shape: - - `input` - - `output` - - `metadata` - - `metrics` - - `error` when failures matter -4. Implement or update patchers. -5. Implement or update tracing helpers. -6. Add or update focused tests. -7. For provider-behavior bugs, make the primary regression test cassette-backed when practical, even if the implementation change is in local tracing/span post-processing of the provider response. -8. Be suspicious of mock/fake coverage for integrations. Do not choose mocks because they are convenient, faster, or easier to control. -9. Run the narrowest nox session first, then expand only if shared code changed. - -Do not start by wiring wrappers and only later decide what the span should contain. +1. Read the spec section(s) covering the affected span shape. +2. Read the provider package + shared primitives. +3. Decide completion-style vs agentic-style (see [Span Design](#span-design)). +4. Pick the public API surface to patch — prefer stable public entry points over internal helpers. +5. Define span shape (`type`, `name`, `input`, `output`, `metadata`, `metrics`, `error`) before writing patchers. +6. Implement patchers, then tracing helpers. +7. For behavior changes: add/update a failing test first, then fix (red → green). Cassette-backed if the provider payload triggers the bug. +8. Run the narrowest nox session first. Expand only if shared code changed. + +Do not start by wiring patchers and defer span-shape decisions. ## Route The Task -### New provider integration - -1. Create `py/src/braintrust/integrations//`. -2. Use this layout unless the provider is exceptionally small: - - `__init__.py` - - `integration.py` - - `patchers.py` - - `tracing.py` - - `test_.py` - - `cassettes//` when the provider uses HTTP (one subdirectory per version in the nox matrix, plus `latest/`) -3. Export the integration from `py/src/braintrust/integrations/__init__.py`. -4. Add or update the provider session in `py/noxfile.py`. -5. Update `py/src/braintrust/auto.py` only if the integration should participate in `auto_instrument()`. -6. Add subprocess coverage in `py/src/braintrust/integrations/auto_test_scripts/` when `auto_instrument()` changes. +### New integration + +1. Create `integrations//` with `__init__.py`, `integration.py`, `patchers.py`, `tracing.py`, `test_.py`, and `cassettes//` (one dir per matrix version + `latest/`). +2. Export from `integrations/__init__.py`. +3. Add the provider session in `noxfile.py`. +4. Update `auto.py` only if it should participate in `auto_instrument()`; add a subprocess test in `auto_test_scripts/` when it does. +5. Add the `_INSTRUMENTATION = "-auto"` shadow (see [Span origin](#span-origin)) and assert it in the test suite. ### Existing integration update -1. Read the current provider package before editing. -2. Change only the affected patchers, tracing helpers, exports, tests, and cassettes. -3. Preserve the provider's public setup and `wrap_*()` surface unless the task explicitly changes it. -4. Do not touch `auto.py`, `integrations/__init__.py`, or `py/noxfile.py` unless the task requires it. -5. Even if `auto.py` does not change, check whether the behavior change also needs an auto-instrument subprocess test update. -6. Preserve existing span shape conventions unless the task is intentionally correcting them. +Change only affected patchers, tracing, exports, tests, cassettes. Preserve the public `setup_*()` / `wrap_*()` surface. Don't touch `auto.py` / `integrations/__init__.py` / `noxfile.py` unless required. Check whether the change also needs a subprocess auto-instrument test update. ### `auto_instrument()` only -1. Update `py/src/braintrust/auto.py`. -2. Prefer `_instrument_integration(...)` over a custom `_instrument_*` helper when the standard pattern fits. -3. Add the integration import near the other integration imports. -4. Add or update the relevant subprocess auto-instrument test. - -### Setup, manual wrapping, and auto-instrument +Update `auto.py`. Prefer `_instrument_integration(...)` over a custom `_instrument_*` helper. Add/update the subprocess auto-instrument test. -Treat these as distinct entry points: +### Entry-point parity -- `setup_()`: explicit package-level patching -- public `wrap_*()` helpers: manual wrapping of a provided class, function, or client -- `auto_instrument()`: import-order-sensitive discovery and setup +The three entry points must emit equivalent spans: +- `setup_()` — package-level patching +- `wrap_*()` helpers — manual wrapping +- `auto_instrument()` — import-order-sensitive discovery -When changing one entry point, check whether the other two should keep equivalent span behavior. If `auto_instrument()` changes or could be affected by import timing, validate it with a subprocess test instead of only calling the integration in-process. +If a behavior change lands in one, check the other two. Validate `auto_instrument()` with a subprocess test, not just in-process. -## Package Layout Rules +## Package Layout -Keep provider-specific behavior in `py/src/braintrust/integrations//`. +Ownership: +- `__init__.py` — public exports, `setup_()`, `wrap_*()` helpers +- `integration.py` — `BaseIntegration` subclass + patcher registration (keep thin) +- `patchers.py` — patchers and manual wrapping helpers +- `tracing.py` — request/response normalization, metadata extraction, stream handling, error logging +- `test_*.py` — provider behavior tests +- `cassettes/` — VCR recordings -Typical ownership: - -- `__init__.py`: public exports, `setup_()`, public `wrap_*()` helpers -- `integration.py`: `BaseIntegration` subclass and patcher registration -- `patchers.py`: patchers and manual `wrap_*()` helpers -- `tracing.py`: request/response normalization, metadata extraction, stream handling, error logging -- `test_*.py`: provider behavior tests -- `cassettes/`: VCR recordings for provider HTTP traffic - -Keep `integration.py` thin. - -If logic is genuinely shared across integrations, move it to `py/src/braintrust/integrations/utils.py` instead of copying it into multiple providers. +If logic is shared across integrations, put it in `integrations/utils.py`, not copies. Check `utils.py` and neighboring integrations before writing a local helper. ## Integration Rules -Set up the integration declaratively: - -- set `name` -- set `import_names` -- set `patchers` -- set `min_version` or `max_version` only when feature detection is not enough +Declarative setup on the `BaseIntegration` subclass: `name`, `import_names`, `patchers`. Use `min_version`/`max_version` only when feature detection isn't enough. Prefer `detect_module_version(...)` / `version_satisfies(...)` / `make_specifier(...)`. -Prefer feature detection first and version checks second. Use: +Let `BaseIntegration.resolve_patchers()` reject duplicate patcher ids; don't hide duplicates. -- `detect_module_version(...)` -- `version_satisfies(...)` -- `make_specifier(...)` +**Non-invasiveness:** +- Provider errors MUST propagate. Never swallow or rewrap. +- Preserve return types, iterator/async-iterator semantics, generator/subclass behavior. +- Sync and async traced schemas MUST stay aligned when both exist. +- Setup / teardown / patching MUST be idempotent (rely on the base patcher marker). +- Contain instrumentation failures. Extraction/normalization/logging errors must be logged or ignored, never raised into the user's call path. +- Treat provider inputs/results/events/headers as untrusted. Avoid arbitrary attribute access; don't mutate third-party objects. +- Only instrument AI-generation-relevant ops (LLM calls, embeddings, tool exec, media gen, agent runs). No unrelated CRUD. -Let `BaseIntegration.resolve_patchers()` reject duplicate patcher ids. Do not hide duplicates. +## Span Design -Preserve provider behavior. Tracing code must not change return values, control flow, or error behavior unless the task explicitly requires it. +### Type + name -Keep sync and async traced schemas aligned when the provider exposes both. +- `llm` — one provider API call +- `task` — parent for an agent run, pipeline step, or named operation +- `tool` — model-initiated tool/function execution -## Span Design Rules +Use spec-recommended names (`Chat Completion`, `anthropic.messages.create`, `generate_content`). For new providers, pick a stable provider-specific name. -Build readable spans. Do not dump raw `args` and `kwargs` unless the provider API already exposes a clean schema. +### Completion-style vs agentic-style -Use this rubric: +- **Completion** (single request/response, user runs any tool calls): one `llm` span, no children. Tool calls appear in `output`. +- **Agentic** (SDK/framework runs the tool loop): one parent `task` wrapping child `llm` (per model call) + `tool` (per tool exec) spans in execution order. -- `input`: the meaningful user request -- `output`: the meaningful provider result -- `metadata`: supporting context such as ids, finish reasons, safety data, model revisions -- `metrics`: timing and numeric accounting such as token counts or elapsed time -- `error`: exceptions or failure information +Completion-style *frameworks* (LiteLLM, OpenRouter, etc.) still emit `llm` spans with `metadata.provider` set to the underlying provider, NOT the framework name. Agentic frameworks (Vercel AI SDK w/ tools, LangChain agents, OpenAI Agents SDK, Claude Agent SDK, ADK, Agno) emit the full parent+children tree. -Avoid double-counting token metrics: +### Payload format -- the integration that directly owns the model/provider API response should own token accounting -- orchestration/framework integrations should usually not log token metrics when underlying provider integrations can create leaf spans with usage metrics -- do not add fragile provider-specific ownership checks such as "if OpenAI is patched, skip metrics"; prefer a clear span ownership rule instead +Canonical shape is OpenAI Chat Completions format. Providers with dedicated UI normalizers (OpenAI, Anthropic, Google) MAY preserve provider-native payloads if `metadata.provider` is set correctly; all other providers MUST convert to OpenAI shape. If a provider passes system prompt separately, insert it as a `role: "system"` entry. -Good span shaping usually means: +### Fields -- flatten positional arguments into named fields -- normalize provider SDK objects into dicts, lists, or scalars when that improves readability -- drop duplicate or noisy transport fields -- aggregate streaming chunks into one final `output` plus stream-specific `metrics` +- `input` — meaningful user request (messages / prompt / provider-native input) +- `output` — meaningful provider result (normalized, not opaque SDK instances) +- `metadata` — see below +- `metrics` — spec-listed keys only (see [Metrics](#metrics)) +- `error` — pass the `Exception` instance directly to span logging; do not pre-format -Do not over-serialize in integration code. Braintrust handles serialization when sending/logging spans, so integration tracing helpers usually only need to shape readable Python dicts/lists/scalars and materialize attachments where appropriate. Avoid unnecessary JSON dumps/loads, recursive conversion, or stringification just to make values serializable. +**Every `llm` span MUST include `metadata.model` and `metadata.provider`** (`provider` = whose pricing applies, even when going through a gateway or framework). -Keep wrapper bodies thin: prepare traced input, open the span, call the provider, normalize the result, and log `output`/`metadata`/`metrics`. +**Tool definitions go in `metadata.tools`** (OpenAI-shaped regardless of underlying provider), NOT in `input` messages. Preserve provider-native built-in tool types (Anthropic `computer_use`, OpenAI `web_search`) as opaque JSON-serializable entries. Never log executable tool handlers. -Braintrust span logging methods are boundary-safe and should not throw during normal integration use. Do not wrap `span.log(...)`, `span.set_attributes(...)`, or similar Braintrust span methods in broad `try`/`except` blocks. Only catch exceptions around provider calls or around integration-owned conversion code when there is a specific expected failure mode and a clear fallback. +**Prompt provenance goes in `metadata.prompt`** (`id`, `project_id`, `version`, `variables`, plus `prompt_session_id` for playground calls). Not in `input`, request payloads, or `metadata.tools`. Strip carrier fields like `span_info` before logging. -Prefer provider-local helpers in `tracing.py`, for example: - -```python -def _prepare_traced_call(args: list[Any], kwargs: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: - ... +**`metadata` allowlist-per-provider:** capture the spec-defined keys plus provider-specific detail fields (request/response ids, safety data, model-family flags) chosen deliberately. Do not dump entire raw request/response objects. Redact secrets. If a provider-specific field ends up broadly useful, promote it to the spec. +### Token metrics -def _process_result(result: Any, start: float) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: - ... -``` +The integration that directly owns the model/provider API response owns token accounting. Orchestration/framework integrations should not log token metrics when underlying provider integrations create leaf spans with usage. Agentic parent `task` spans MAY aggregate across children when the framework doesn't delegate to a separately instrumented provider client. Do not add "if OpenAI is patched, skip metrics" checks — define clear ownership instead. -Treat binary payloads as attachments, not logged bytes: +### Shaping guidance -- prefer the shared `_materialize_attachment(...)` helper in `py/src/braintrust/integrations/utils.py` over provider-local base64 or file-decoding code -- convert provider-owned raw `bytes`, base64 payloads, data URLs, file inputs, and generated media into `braintrust.logger.Attachment` objects when Braintrust should upload the content -- preserve normal remote URLs as strings -- use the repo's existing multimodal payload shapes after materialization: - - images -> `{"image_url": {"url": attachment}}` - - non-image media/documents/files -> `{"file": {"file_data": attachment, "filename": resolved.filename}}` -- do not force non-image payloads through `image_url` shims -- if attachment materialization fails, keep the original value instead of dropping it or replacing it with `None` -- preserve non-attachment values while walking nested payloads unless you are intentionally normalizing them for readability -- keep useful metadata such as MIME type, size, safety data, filenames, or provider ids next to the attachment +- Flatten positional args into named fields; normalize SDK objects into dicts/lists/scalars; drop noisy transport fields. +- Aggregate streaming chunks into one final `output` + stream-specific `metrics`. One `llm` span per API call, not per chunk. +- Do not over-serialize. Braintrust serializes at send/log time. Integration tracing only needs readable Python dicts/lists/scalars and materialized attachments. +- Keep wrapper bodies thin: prepare traced input, open span, call provider, normalize result, log. +- Do not wrap `span.log(...)` / `span.set_attributes(...)` in broad try/except. Braintrust span methods are boundary-safe. -## Patcher Rules +### Span origin -Create one patcher per coherent patch target. +Every span an integration creates MUST carry `context.span_origin.instrumentation.name = "-auto"` (e.g. `openai-auto`, `anthropic-auto`, `adk-auto`, `google-genai-auto`). Match the JS SDK exactly for cross-SDK filtering. -Prefer: +`start_span(...)` and all provider-level `start_span` methods accept `internal: SpanInternalOptions | None`, a TypedDict reserved for SDK internals — external callers should not use it. Integrations pass `internal={"instrumentation": "-auto"}`. When unset, spans fall back to the channel default (`braintrust-python-logger` / `braintrust-python-otel`). -- `FunctionWrapperPatcher` for one import path or one constructor/method surface -- `CompositeFunctionWrapperPatcher` for one logical surface spread across multiple related targets -- `CallbackPatcher` for setup side effects after applicability succeeds +The name does NOT propagate through the parent/child edge — each `start_span` call decides independently. User-owned spans nested inside a wrapped call (e.g. a `@traced` scorer) must NOT inherit the integration's name. -Use `target_module` when the patch target lives outside the module named by `import_names`, especially for optional or deep submodules. +Standard pattern at the top of `tracing.py` (or `callbacks.py` / `plugin.py`): -Use `superseded_by` for version-conditional fallbacks instead of custom target-selection logic. +```python +from braintrust.logger import start_span as _bt_start_span -Use lower `priority` only when patch ordering really matters, such as context propagation before tracing. +_INSTRUMENTATION = "-auto" -Manual wrapping helpers should be thin: -```python -def wrap_agent(Agent: Any) -> Any: - return AgentPatcher.wrap_target(Agent) +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) ``` -Require every patcher to have: +Module-level `start_span(...)` calls flow through automatically. Calls that go through a `Logger` / `Experiment` / parent-`Span` instance (`logger.start_span(...)`, `parent.start_span(...)`) need `internal={"instrumentation": _INSTRUMENTATION}` explicitly. Tests assert `span["context"]["span_origin"]["instrumentation"]["name"] == "-auto"`; `SpanImpl._instrumentation` exposes the resolved value. -- a stable `name` -- clean existence checks -- version gating only when necessary -- idempotence through the base patcher marker +## Metrics -## Testing Rules +Only emit spec-listed keys: +- `tokens`, `prompt_tokens`, `completion_tokens` — required for LLM spans; values MUST be non-negative +- `time_to_first_token` — required for streaming; SDK measures from request start to first chunk +- `completion_reasoning_tokens` — required when provider reports reasoning tokens (o-series etc.) +- `prompt_cached_tokens`, `prompt_cache_creation_tokens`, `prompt_cache_creation_5m_tokens`, `prompt_cache_creation_1h_tokens` — provider prompt caching +- `prompt_audio_tokens`, `completion_audio_tokens`, `completion_image_tokens` — audio/image models when provider reports +- `start`, `end` — standard span timing + +For streaming, still produce one span per API call with accumulated `input`/`output`. Capture usage from stream metadata (e.g. OpenAI `stream_options.include_usage`) when surfaced. -Keep tests in the provider package. +For reasoning models, capture the full output structure (reasoning summaries + message blocks) and include prior reasoning in `input` for multi-turn calls. -Default bug-fix workflow: red -> green. +## Multimodal And Attachments -- First add or update a focused test that reproduces the integration bug. -- Then implement the fix. -- Only skip this when the task explicitly asks for a different approach. +Inline binary media (images, PDFs, audio, video) MUST be replaced by a Braintrust attachment at the leaf position. Do not add a separate top-level `attachments` list. -Prefer VCR-backed real provider coverage with `@pytest.mark.vcr`. This includes span-shaping/tracing bugs where the bad behavior is triggered by a real provider response payload; do not treat those as mock-first just because the code path is local. +- Prefer `_materialize_attachment(...)` in `integrations/utils.py`. Don't reimplement base64/file decoding. +- Convert raw `bytes` / base64 / data URLs / file inputs / generated media → `braintrust.logger.Attachment`. Preserve remote URLs as strings — do not fetch to create an attachment. +- Payload shape after materialization: + - images → `{"image_url": {"url": attachment}}` + - non-image media/documents/files → `{"file": {"file_data": attachment, "filename": resolved.filename}}` +- Do not force non-image payloads through `image_url`. +- If materialization fails, keep the original value; never drop, null it out, or raise. +- Preserve non-attachment values while walking nested payloads unless intentionally normalizing. +- Keep MIME type, size, safety data, filenames, provider ids next to the attachment. +- Providers with UI normalizers (OpenAI, Anthropic, Google) — MAY preserve the provider-native structure and only replace the raw media leaf. +- Generated media → log the attachment in `output`, not `metadata`. For streaming, aggregate into one final `output` attachment. -Default stance: if the behavior is provider-facing, assume mocks/fakes are the wrong tool until proven otherwise. A mock should need justification, not the other way around. +## Patcher Rules -Use mocks or fakes only for cases that are hard to drive through recordings, such as: +One patcher per coherent target. Prefer: -- narrow error injection -- purely local version-routing logic -- patcher existence checks -- provider-independent helper logic where the provider response shape is not part of the contract being validated +- `FunctionWrapperPatcher` — one import path / one constructor / one method surface +- `CompositeFunctionWrapperPatcher` — one logical surface across multiple related targets +- `CallbackPatcher` — setup side effects after applicability succeeds -Do not replace or skip a cassette-backed regression with a mock/fake test merely because the implementation change lives in `tracing.py`, a serializer, or another local post-processing layer. If a real provider payload is what triggers the bug, the main regression test should reflect that real payload. +Use `target_module` for patch targets outside the module named by `import_names` (deep or optional submodules). Use `superseded_by` for version-conditional fallbacks — not custom target-selection logic. Use lower `priority` only when ordering matters (e.g. context propagation before tracing). -Test emitted spans, not just provider return values. +Manual wrapping helpers stay thin: -Cover the surfaces that changed: +```python +def wrap_agent(Agent: Any) -> Any: + return AgentPatcher.wrap_target(Agent) +``` -- direct `wrap_*()` behavior -- setup-time patching -- sync behavior -- async behavior -- streaming behavior -- idempotence -- failure and error logging -- patcher resolution and duplicate detection when relevant -- attachment conversion for binary inputs or generated media, including assertions that images land under `image_url.url`, non-image payloads land under `file.file_data`, and traced payloads contain `Attachment` objects rather than raw bytes or base64 blobs -- span structure, especially `input`, `output`, `metadata`, and `metrics` +Every patcher needs: stable `name`, clean existence checks, version gating only when necessary, idempotence via the base patcher marker. Prefer patching stable public API surfaces; internal helpers need version-specific maintenance. -For streaming changes, verify both: +## Testing Rules -- the provider still returns the expected iterator or async iterator -- the final logged span contains the aggregated `output` and stream-specific `metrics` +Tests live in the provider package. Default bug-fix workflow: red → green — add/update a failing test first, then fix. -Also verify, when relevant: +**Prefer VCR-backed real provider coverage with `@pytest.mark.vcr`.** The burden of proof is on mocks: use them only for narrow error injection, purely local version-routing logic, patcher existence checks, or provider-independent helpers where the response shape is not part of the contract. Do not swap a cassette-backed regression for a mock just because the fix lives in `tracing.py` or a serializer — if a real payload triggers the bug, the primary regression stays cassette-backed. -- the `input` contains the expected model/messages/prompt/config fields -- the `output` contains normalized provider results rather than opaque SDK instances -- the `metadata` contains finish reasons, ids, or annotations in the expected place -- binary payloads are represented as `Attachment` objects where applicable, while remote URLs and non-attachment values remain unchanged and unmaterialized file inputs are preserved rather than dropped +Assert on emitted spans (not just provider return values): +- span `type` and `name` +- `input` shape (spec-required messages/prompt/config fields) +- `output` shape (normalized, not opaque SDK instances) +- `metadata.model`, `metadata.provider`, `metadata.tools`/`tool_choice`/`prompt` when present +- required `metrics` keys (tokens for LLM; `time_to_first_token` for streaming) +- parent/child structure for agentic APIs (parent `task`, child `llm`+`tool` in execution order) +- `context.span_origin.instrumentation.name == "-auto"` +- attachments: images under `image_url.url`, non-images under `file.file_data`, `Attachment` objects (not raw bytes/base64) +- error propagation, error logging +- setup/teardown/wrapping idempotence, patcher resolution when relevant -Keep VCR cassettes in `py/src/braintrust/integrations//cassettes//` (e.g. `cassettes/latest/`, `cassettes/0.48.0/`). Nox sessions set `BRAINTRUST_TEST_PACKAGE_VERSION` automatically so cassettes land in the correct version subdirectory. Do not add per-test `vcr_cassette_dir` or `cassette_library_dir` fixtures; the shared `py/src/braintrust/integrations/conftest.py` handles version routing. Re-record only when behavior intentionally changes. +For streaming, assert both the provider iterator/async-iterator still works AND the final span has aggregated `output` + stream-specific `metrics`. -When the provider returns binary HTTP responses or generated media, sanitize cassettes as needed so fixtures do not store raw file bytes. +Cassettes live in `integrations//cassettes//` (e.g. `cassettes/latest/`, `cassettes/0.48.0/`). Nox sets `BRAINTRUST_TEST_PACKAGE_VERSION` so cassettes land correctly. Do not add per-test `vcr_cassette_dir` / `cassette_library_dir` fixtures — `integrations/conftest.py` handles it. Re-record only when behavior intentionally changed. Sanitize cassettes when the provider returns binary bodies. -When choosing test commands, confirm the actual session name in `py/noxfile.py` instead of assuming it matches the provider folder. +Confirm the exact session name from `noxfile.py` — don't assume it matches the folder. ## Commands @@ -343,27 +274,29 @@ cd py && make lint ## Validation Checklist -- Run the narrowest provider session first. -- If the change touches patchers, setup behavior, import timing, or anything that could affect `auto_instrument()`, run the relevant subprocess auto-instrument test from `py/src/braintrust/integrations/auto_test_scripts/`. -- Run the relevant auto-instrument subprocess test if `auto.py` changed. -- Run `cd py && make test-core` if shared integration code changed. -- Run `cd py && make lint` before handoff when shared files or repo-level wiring changed. +- Narrowest provider session first. +- Subprocess auto-instrument test if patchers / setup / import timing / `auto.py` changed. +- `make test-core` if shared integration code changed. +- `make lint` before handoff if shared files or repo wiring changed. ## Common Mistakes -Avoid these failures: - -- treating a wrapper migration as fresh integration work -- changing shared integration primitives when provider-local code should own the behavior -- combining unrelated patch targets into one patcher -- forgetting repo-level wiring for new providers: `integrations/__init__.py`, `py/noxfile.py`, and sometimes `auto.py` -- forgetting the subprocess auto-instrument tests -- forgetting async or streaming coverage -- re-recording cassettes when behavior did not intentionally change -- adding a custom `_instrument_*` helper where `_instrument_integration()` already fits -- forgetting `target_module` for deep or optional patch targets -- double-counting token metrics in both orchestration/framework spans and provider leaf spans -- adding provider-specific token ownership detection instead of defining clear metric ownership for the integration -- doing excessive serialization/stringification in tracing code even though Braintrust serializes span payloads at send/log time -- wrapping Braintrust span logging methods in broad `try`/`except` blocks even though those methods are designed not to throw -- forcing non-image attachments through `image_url` shims, dropping unrecognized file inputs, or re-serializing non-attachment values while materializing payloads +- Treating a wrapper migration as fresh integration work (use `sdk-wrapper-migrations`). +- Changing shared primitives when provider-local code should own the behavior. +- Combining unrelated patch targets into one patcher. +- Forgetting repo wiring for new providers (`integrations/__init__.py`, `noxfile.py`, sometimes `auto.py`). +- Forgetting subprocess auto-instrument tests, or async/streaming coverage. +- Re-recording cassettes when behavior didn't intentionally change. +- Custom `_instrument_*` helper where `_instrument_integration()` fits. +- Missing `target_module` for deep/optional patch targets. +- Inventing new top-level span fields / metric keys / span types (metadata is looser — see allowlist-per-provider). +- Tool definitions in `input` instead of `metadata.tools`; prompt provenance outside `metadata.prompt`. +- Missing `internal={"instrumentation": "-auto"}` on integration spans, or leaking it onto user spans nested inside. +- Framework name in `metadata.provider` for a completion-style framework wrapper. +- Per-chunk spans for streamed calls (should be one accumulated span per API call). +- Instrumentation errors escaping into the user's call path, or swallowing real provider errors. +- Double-counting tokens across orchestration + provider spans, or provider-specific ownership checks instead of clear rules. +- Over-serializing (JSON dumps/loads, recursive conversion) in tracing code. +- `try`/`except` around Braintrust span-logging methods — they're boundary-safe. +- Forcing non-image attachments through `image_url` shims; dropping unrecognized file inputs; re-serializing non-attachment values. +- Patching internal helpers when a stable public API surface would work. diff --git a/py/src/braintrust/integrations/adk/test_adk.py b/py/src/braintrust/integrations/adk/test_adk.py index afaad199..462f2eba 100644 --- a/py/src/braintrust/integrations/adk/test_adk.py +++ b/py/src/braintrust/integrations/adk/test_adk.py @@ -198,6 +198,8 @@ async def run_message(text: str) -> str: invocation_spans = [row for row in spans if row["span_attributes"]["name"] == f"invocation [{app_name}]"] assert len(invocation_spans) == 2 + for span in invocation_spans: + assert span["context"]["span_origin"]["instrumentation"]["name"] == "adk-auto" assert {span["metadata"]["session_id"] for span in invocation_spans} == {session_id} assert {span["input"]["new_message"]["parts"][0]["text"] for span in invocation_spans} == { "Hi, my name is Alice.", diff --git a/py/src/braintrust/integrations/adk/tracing.py b/py/src/braintrust/integrations/adk/tracing.py index c84b51b2..c6367e8e 100644 --- a/py/src/braintrust/integrations/adk/tracing.py +++ b/py/src/braintrust/integrations/adk/tracing.py @@ -12,7 +12,19 @@ from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "adk-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/agentscope/test_agentscope.py b/py/src/braintrust/integrations/agentscope/test_agentscope.py index 455a9ab4..372d675d 100644 --- a/py/src/braintrust/integrations/agentscope/test_agentscope.py +++ b/py/src/braintrust/integrations/agentscope/test_agentscope.py @@ -119,6 +119,7 @@ async def test_agentscope_simple_agent_run(memory_logger): agent_span = next(span for span in spans if span["span_attributes"]["name"] == "Friday.reply") llm_spans = [span for span in spans if _span_type(span) == SpanTypeAttribute.LLM] + assert agent_span["context"]["span_origin"]["instrumentation"]["name"] == "agentscope-auto" assert _span_type(agent_span) == "task" assert llm_spans assert llm_spans[0]["metadata"]["model"] == "gpt-4o-mini" diff --git a/py/src/braintrust/integrations/agentscope/tracing.py b/py/src/braintrust/integrations/agentscope/tracing.py index 06bea174..f96e3cfd 100644 --- a/py/src/braintrust/integrations/agentscope/tracing.py +++ b/py/src/braintrust/integrations/agentscope/tracing.py @@ -5,7 +5,19 @@ from contextlib import aclosing from typing import Any -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "agentscope-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/agno/test_agno.py b/py/src/braintrust/integrations/agno/test_agno.py index 59549747..0d17262d 100644 --- a/py/src/braintrust/integrations/agno/test_agno.py +++ b/py/src/braintrust/integrations/agno/test_agno.py @@ -62,6 +62,7 @@ def test_agno_simple_agent_execution(memory_logger): assert len(spans) == 2, f"Expected 2 spans, got {len(spans)}" root_span = spans[0] + assert root_span["context"]["span_origin"]["instrumentation"]["name"] == "agno-auto" assert root_span["span_attributes"]["name"] == "Author Agent.run" assert root_span["span_attributes"]["type"].value == "task" root_input = root_span["input"] diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py index 65a6ff7e..5c8983d4 100644 --- a/py/src/braintrust/integrations/agno/tracing.py +++ b/py/src/braintrust/integrations/agno/tracing.py @@ -3,7 +3,19 @@ from typing import Any from braintrust.integrations.utils import _try_to_dict -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "agno-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index a2b7fca9..08d5c20b 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -482,6 +482,7 @@ def test_anthropic_messages_create_with_image_attachment_input(memory_logger): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "anthropic-auto" content = span["input"][0]["content"] image_block = content[1] diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index eacacb6e..c316ea9a 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -6,7 +6,20 @@ from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.anthropic._utils import Wrapper, _try_to_dict, extract_anthropic_usage from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import log_exc_info_to_span, start_span +from braintrust.logger import log_exc_info_to_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "anthropic-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/autogen/test_autogen.py b/py/src/braintrust/integrations/autogen/test_autogen.py index 355227ac..70630150 100644 --- a/py/src/braintrust/integrations/autogen/test_autogen.py +++ b/py/src/braintrust/integrations/autogen/test_autogen.py @@ -47,6 +47,7 @@ async def test_autogen_agent_run_creates_braintrust_spans(memory_logger): assert result.messages[-1].content spans = memory_logger.pop() agent_span = next(span for span in spans if span["span_attributes"]["name"] == "assistant.run") + assert agent_span["context"]["span_origin"]["instrumentation"]["name"] == "autogen-auto" assert _span_type(agent_span) == "task" assert agent_span["input"]["task"] == "Say hello in exactly two words." assert agent_span["metadata"]["component"] == "agent" diff --git a/py/src/braintrust/integrations/autogen/tracing.py b/py/src/braintrust/integrations/autogen/tracing.py index 6072d644..3fa798be 100644 --- a/py/src/braintrust/integrations/autogen/tracing.py +++ b/py/src/braintrust/integrations/autogen/tracing.py @@ -4,7 +4,19 @@ from contextvars import ContextVar from typing import Any -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "autogen-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py index 73e19249..583b9f86 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py +++ b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py @@ -222,6 +222,7 @@ def test_wrap_bedrock_converse_stream(memory_logger): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "bedrock-runtime-auto" _assert_converse_span(span, name="bedrock.converse-stream", start=start, end=end, model_id=CONVERSE_STREAM_MODEL) assert span["metadata"]["endpoint"] == "converse-stream" assert span["metadata"]["stream"] is True diff --git a/py/src/braintrust/integrations/bedrock_runtime/tracing.py b/py/src/braintrust/integrations/bedrock_runtime/tracing.py index 0d1f6605..fee3f1f1 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/tracing.py +++ b/py/src/braintrust/integrations/bedrock_runtime/tracing.py @@ -11,7 +11,19 @@ _materialize_attachment, _timing_metrics, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "bedrock-runtime-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py index 1ff40395..258a6a46 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py @@ -162,6 +162,7 @@ async def calculator_handler(args): assert len(task_spans) == 1, f"Should have exactly one task span, got {len(task_spans)}" task_span = task_spans[0] + assert task_span["context"]["span_origin"]["instrumentation"]["name"] == "claude-agent-sdk-auto" assert task_span["span_attributes"]["name"] == "Claude Agent" assert "15 multiplied by 7" in task_span["input"] assert task_span["output"] is not None diff --git a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py index 631b6f99..ed0f8d3c 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py @@ -23,7 +23,19 @@ MessageClassName, SerializedContentType, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "claude-agent-sdk-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/cohere/test_cohere.py b/py/src/braintrust/integrations/cohere/test_cohere.py index d131039b..0a99b639 100644 --- a/py/src/braintrust/integrations/cohere/test_cohere.py +++ b/py/src/braintrust/integrations/cohere/test_cohere.py @@ -247,6 +247,7 @@ def test_wrap_cohere_chat_v2_sync(memory_logger): assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "cohere-auto" assert span["span_attributes"]["name"] == "cohere.chat" assert span["span_attributes"]["type"] == "llm" assert span["metadata"]["provider"] == "cohere" diff --git a/py/src/braintrust/integrations/cohere/tracing.py b/py/src/braintrust/integrations/cohere/tracing.py index 0f563616..5bf72cd0 100644 --- a/py/src/braintrust/integrations/cohere/tracing.py +++ b/py/src/braintrust/integrations/cohere/tracing.py @@ -21,7 +21,19 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "cohere-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/crewai/test_crewai.py b/py/src/braintrust/integrations/crewai/test_crewai.py index 4e15fb9a..86f4d4a7 100644 --- a/py/src/braintrust/integrations/crewai/test_crewai.py +++ b/py/src/braintrust/integrations/crewai/test_crewai.py @@ -262,6 +262,7 @@ def test_kickoff_llm_event_tree_parents_and_shape(memory_logger): kickoff_span = by_name["crewai.kickoff"][0] llm_span = by_name["crewai.llm"][0] + assert kickoff_span["context"]["span_origin"]["instrumentation"]["name"] == "crewai-auto" assert kickoff_span["span_attributes"]["type"] == "task" assert llm_span["span_attributes"]["type"] == "llm" diff --git a/py/src/braintrust/integrations/crewai/tracing.py b/py/src/braintrust/integrations/crewai/tracing.py index 77a1ce4e..baf4b046 100644 --- a/py/src/braintrust/integrations/crewai/tracing.py +++ b/py/src/braintrust/integrations/crewai/tracing.py @@ -47,7 +47,20 @@ _parse_openai_usage_metrics, _try_to_dict, ) -from braintrust.logger import NOOP_SPAN, Span, current_span, start_span +from braintrust.logger import NOOP_SPAN, Span, current_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "crewai-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index e9ccef9d..9fa1b52c 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -43,6 +43,7 @@ def test_dspy_callback(memory_logger): spans_by_name = {span["span_attributes"]["name"]: span for span in spans} lm_span = spans_by_name["dspy.lm"] + assert lm_span["context"]["span_origin"]["instrumentation"]["name"] == "dspy-auto" assert "metadata" in lm_span assert "model" in lm_span["metadata"] assert MODEL in lm_span["metadata"]["model"] diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index dbd376f7..e44b70f1 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -2,7 +2,20 @@ from typing import Any -from braintrust.logger import current_span, start_span +from braintrust.logger import current_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "dspy-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/google_genai/test_google_genai.py b/py/src/braintrust/integrations/google_genai/test_google_genai.py index a0cf1b41..dc9f71ea 100644 --- a/py/src/braintrust/integrations/google_genai/test_google_genai.py +++ b/py/src/braintrust/integrations/google_genai/test_google_genai.py @@ -215,6 +215,7 @@ def test_basic_completion(memory_logger, mode): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "google-genai-auto" assert span["metadata"]["model"] == MODEL assert "What is the capital of France?" in str(span["input"]) assert span["output"] diff --git a/py/src/braintrust/integrations/google_genai/tracing.py b/py/src/braintrust/integrations/google_genai/tracing.py index 3f231f73..af75d83d 100644 --- a/py/src/braintrust/integrations/google_genai/tracing.py +++ b/py/src/braintrust/integrations/google_genai/tracing.py @@ -10,7 +10,28 @@ from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import NOOP_SPAN, _state, current_logger, current_span, parent_context, start_span +from braintrust.logger import ( + NOOP_SPAN, + _state, + current_logger, + current_span, + parent_context, +) +from braintrust.logger import ( + start_span as _bt_start_span, +) + + +_INSTRUMENTATION = "google-genai-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones @@ -1158,6 +1179,7 @@ def _stream_span_context( input=input, metadata=metadata, lookup_span_parent=False, + internal={"instrumentation": _INSTRUMENTATION}, ) as span: yield span return diff --git a/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py b/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py index 49f42252..b1ae4985 100644 --- a/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py +++ b/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py @@ -244,6 +244,7 @@ def test_wrap_huggingface_hub_chat_completion_sync(memory_logger): assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "huggingface-hub-auto" assert span["span_attributes"]["name"] == "huggingface.chat_completion" assert span["span_attributes"]["type"] == "llm" # With no parent span on the stack, the LLM span is its own root and has diff --git a/py/src/braintrust/integrations/huggingface_hub/tracing.py b/py/src/braintrust/integrations/huggingface_hub/tracing.py index 2781bd1c..839d3345 100644 --- a/py/src/braintrust/integrations/huggingface_hub/tracing.py +++ b/py/src/braintrust/integrations/huggingface_hub/tracing.py @@ -34,7 +34,19 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "huggingface-hub-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/instructor/test_instructor.py b/py/src/braintrust/integrations/instructor/test_instructor.py index 0450e3e6..4f024c44 100644 --- a/py/src/braintrust/integrations/instructor/test_instructor.py +++ b/py/src/braintrust/integrations/instructor/test_instructor.py @@ -121,6 +121,7 @@ def test_instructor_openai_single_success(self, setup_logger, memory_logger): assert len(llm_spans) == 1, f"Expected 1 llm span, got {len(llm_spans)}: names={_names(spans)}" parent = task_spans[0] + assert parent["context"]["span_origin"]["instrumentation"]["name"] == "instructor-auto" assert parent["span_attributes"]["name"] == "instructor.create" meta = parent.get("metadata", {}) assert meta.get("response_model") == "Person" diff --git a/py/src/braintrust/integrations/instructor/tracing.py b/py/src/braintrust/integrations/instructor/tracing.py index e419af0f..48b52b3b 100644 --- a/py/src/braintrust/integrations/instructor/tracing.py +++ b/py/src/braintrust/integrations/instructor/tracing.py @@ -28,7 +28,19 @@ from collections.abc import AsyncIterator, Iterator from typing import Any -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "instructor-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/langchain/callbacks.py b/py/src/braintrust/integrations/langchain/callbacks.py index 2cb9b37d..9d9517c9 100644 --- a/py/src/braintrust/integrations/langchain/callbacks.py +++ b/py/src/braintrust/integrations/langchain/callbacks.py @@ -11,7 +11,20 @@ from uuid import UUID from braintrust.generated_types import SpanAttributes -from braintrust.logger import NOOP_SPAN, Logger, Span, current_span, init_logger, start_span +from braintrust.logger import NOOP_SPAN, Logger, Span, current_span, init_logger +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "langchain-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.version import VERSION as sdk_version from langchain_core.agents import AgentAction, AgentFinish @@ -132,6 +145,7 @@ def _start_span( start_time=start_time, set_current=set_current, parent=parent, + internal={"instrumentation": _INSTRUMENTATION}, **event, ) @@ -146,6 +160,7 @@ def _start_span( start_time=start_time, set_current=set_current, parent=parent, + internal={"instrumentation": _INSTRUMENTATION}, **event, ) diff --git a/py/src/braintrust/integrations/langchain/test_callbacks.py b/py/src/braintrust/integrations/langchain/test_callbacks.py index b5fa5e81..1543c1fd 100644 --- a/py/src/braintrust/integrations/langchain/test_callbacks.py +++ b/py/src/braintrust/integrations/langchain/test_callbacks.py @@ -61,6 +61,8 @@ def test_llm_calls(logger_memory_logger): spans = memory_logger.pop() assert len(spans) == 3 + for span in spans: + assert span["context"]["span_origin"]["instrumentation"]["name"] == "langchain-auto" # ``root_span_id`` is the root span's own span_id (the parent reference for # its children); ``trace_root_id`` is the trace shared by every span. diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 549d3e33..33e97ca5 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -68,6 +68,7 @@ def test_litellm_completion_metrics(memory_logger) -> None: assert len(spans) == 1 span = spans[0] assert span + assert span["context"]["span_origin"]["instrumentation"]["name"] == "litellm-auto" metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert span["metadata"]["model"] == TEST_MODEL diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index 3dd1d8f9..750ad2d3 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -14,7 +14,20 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import Span, start_span +from braintrust.logger import Span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "litellm-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones, is_numeric, merge_dicts diff --git a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py index afa48881..a27740fb 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py +++ b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py @@ -159,6 +159,7 @@ async def cancelled_start(): logs = memory_logger.pop() session_logs = _spans_named(logs, "livekit_agent_session") assert len(session_logs) == 1 + assert session_logs[0]["context"]["span_origin"]["instrumentation"]["name"] == "livekit-agents-auto" @pytest.mark.asyncio diff --git a/py/src/braintrust/integrations/livekit_agents/tracing.py b/py/src/braintrust/integrations/livekit_agents/tracing.py index 7adfab31..98e5fd29 100644 --- a/py/src/braintrust/integrations/livekit_agents/tracing.py +++ b/py/src/braintrust/integrations/livekit_agents/tracing.py @@ -16,8 +16,20 @@ _state, current_span, parent_context, - start_span, ) +from braintrust.logger import ( + start_span as _bt_start_span, +) + + +_INSTRUMENTATION = "livekit-agents-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) _SESSION_PARENT_ATTR = "__braintrust_livekit_session_parent__" diff --git a/py/src/braintrust/integrations/llamaindex/test_llamaindex.py b/py/src/braintrust/integrations/llamaindex/test_llamaindex.py index 315a8ce0..2c27cfc6 100644 --- a/py/src/braintrust/integrations/llamaindex/test_llamaindex.py +++ b/py/src/braintrust/integrations/llamaindex/test_llamaindex.py @@ -114,6 +114,7 @@ def test_llm_complete(logger_memory_logger): assert len(llm_spans) >= 1 llm_span = llm_spans[0] + assert llm_span["context"]["span_origin"]["instrumentation"]["name"] == "llamaindex-auto" assert llm_span["span_attributes"]["name"] == "OpenAI" assert llm_span["input"] is not None assert llm_span["output"] is not None diff --git a/py/src/braintrust/integrations/llamaindex/tracing.py b/py/src/braintrust/integrations/llamaindex/tracing.py index ee2a6542..f63a60bb 100644 --- a/py/src/braintrust/integrations/llamaindex/tracing.py +++ b/py/src/braintrust/integrations/llamaindex/tracing.py @@ -4,7 +4,20 @@ import time from typing import Any -from braintrust.logger import NOOP_SPAN, Span, current_span, start_span +from braintrust.logger import NOOP_SPAN, Span, current_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "llamaindex-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute @@ -201,7 +214,13 @@ def new_span( event["input"] = input_data if parent_bt_span is not None: - bt_span = parent_bt_span.start_span(name=span_name, type=span_type, start_time=start_time, **event) + bt_span = parent_bt_span.start_span( + name=span_name, + type=span_type, + start_time=start_time, + internal={"instrumentation": _INSTRUMENTATION}, + **event, + ) else: bt_span = start_span(name=span_name, type=span_type, start_time=start_time, **event) diff --git a/py/src/braintrust/integrations/mistral/test_mistral.py b/py/src/braintrust/integrations/mistral/test_mistral.py index 6833bbd5..e8a9f41f 100644 --- a/py/src/braintrust/integrations/mistral/test_mistral.py +++ b/py/src/braintrust/integrations/mistral/test_mistral.py @@ -246,6 +246,7 @@ def test_wrap_mistral_chat_complete_sync(memory_logger): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "mistral-auto" assert span["input"] == [{"role": "user", "content": "What is 2+2? Reply with just the number."}] assert span["metadata"]["provider"] == "mistral" assert span["metadata"]["model"] == CHAT_MODEL diff --git a/py/src/braintrust/integrations/mistral/tracing.py b/py/src/braintrust/integrations/mistral/tracing.py index 30cfb20c..7a1ed296 100644 --- a/py/src/braintrust/integrations/mistral/tracing.py +++ b/py/src/braintrust/integrations/mistral/tracing.py @@ -16,7 +16,19 @@ _materialize_attachment, _merge_timing_and_usage_metrics, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "mistral-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 84418d53..b95ce084 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -119,6 +119,7 @@ def test_openai_chat_metrics(memory_logger): assert len(spans) == 1 span = spans[0] assert span + assert span["context"]["span_origin"]["instrumentation"]["name"] == "openai-auto" metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert TEST_MODEL in span["metadata"]["model"] diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 8739cf00..6348f481 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -19,7 +19,20 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import Span, start_span +from braintrust.logger import Span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "openai-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones, merge_dicts from wrapt import FunctionWrapper diff --git a/py/src/braintrust/integrations/openai_agents/test_openai_agents.py b/py/src/braintrust/integrations/openai_agents/test_openai_agents.py index 7a6db120..51f442cd 100644 --- a/py/src/braintrust/integrations/openai_agents/test_openai_agents.py +++ b/py/src/braintrust/integrations/openai_agents/test_openai_agents.py @@ -90,6 +90,7 @@ def export(self): spans = memory_logger.pop() root_span = spans[0] + assert root_span["context"]["span_origin"]["instrumentation"]["name"] == "openai-agents-auto" assert root_span["metadata"]["conversation_id"] == "test-12345" diff --git a/py/src/braintrust/integrations/openai_agents/tracing.py b/py/src/braintrust/integrations/openai_agents/tracing.py index bf1a7274..bc939504 100644 --- a/py/src/braintrust/integrations/openai_agents/tracing.py +++ b/py/src/braintrust/integrations/openai_agents/tracing.py @@ -4,7 +4,20 @@ from typing import Any from agents import tracing -from braintrust.logger import NOOP_SPAN, Experiment, Logger, Span, current_span, flush, start_span +from braintrust.logger import NOOP_SPAN, Experiment, Logger, Span, current_span, flush +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "openai-agents-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute @@ -132,6 +145,7 @@ def on_trace_start(self, trace: tracing.Trace) -> None: name=trace.name, span_attributes={"type": "task", "name": trace.name}, metadata=metadata, + internal={"instrumentation": _INSTRUMENTATION}, ) elif self._logger is not None: span = self._logger.start_span( @@ -139,6 +153,7 @@ def on_trace_start(self, trace: tracing.Trace) -> None: span_id=trace.trace_id, root_span_id=trace.trace_id, metadata=metadata, + internal={"instrumentation": _INSTRUMENTATION}, ) else: span = start_span( @@ -336,6 +351,7 @@ def on_span_start(self, span: tracing.Span[tracing.SpanData]) -> None: name=_span_name(span), type=_span_type(span), start_time=_timestamp_from_maybe_iso(span.started_at), + internal={"instrumentation": _INSTRUMENTATION}, ) self._spans[span.span_id] = created_span created_span.set_current() diff --git a/py/src/braintrust/integrations/openrouter/test_openrouter.py b/py/src/braintrust/integrations/openrouter/test_openrouter.py index 0839c230..fa1ec892 100644 --- a/py/src/braintrust/integrations/openrouter/test_openrouter.py +++ b/py/src/braintrust/integrations/openrouter/test_openrouter.py @@ -50,6 +50,7 @@ def test_wrap_openrouter_chat_send_sync(memory_logger): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "openrouter-auto" assert span["input"] == [{"role": "user", "content": "What is 2+2? Reply with just the number."}] assert span["metadata"]["provider"] == "openai" assert span["metadata"]["model"] == "gpt-4o-mini" diff --git a/py/src/braintrust/integrations/openrouter/tracing.py b/py/src/braintrust/integrations/openrouter/tracing.py index f9711bb2..a425e1fc 100644 --- a/py/src/braintrust/integrations/openrouter/tracing.py +++ b/py/src/braintrust/integrations/openrouter/tracing.py @@ -13,7 +13,19 @@ _log_error_and_end_span, _merge_timing_and_usage_metrics, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "openrouter-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index 60462e5c..814aeaf3 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -131,6 +131,7 @@ async def test_direct_model_request_creates_nested_chat_span_without_class_scan( assert direct_span is not None, "model_request span not found" assert chat_span is not None, "chat span not found" + assert direct_span["context"]["span_origin"]["instrumentation"]["name"] == "pydantic-ai-auto" assert chat_span["span_parents"] == [direct_span["span_id"]] assert chat_span["metadata"]["model"] == "gpt-4o-mini" assert chat_span["metadata"]["provider"] == "openai" diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index 64de3a6b..8374d508 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -8,7 +8,20 @@ from typing import Any from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import _internal_get_global_state, start_span +from braintrust.logger import _internal_get_global_state +from braintrust.logger import start_span as _bt_start_span + + +_INSTRUMENTATION = "pydantic-ai-auto" + + +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) + + from braintrust.span_types import SpanTypeAttribute from wrapt import wrap_function_wrapper diff --git a/py/src/braintrust/integrations/temporal/plugin.py b/py/src/braintrust/integrations/temporal/plugin.py index 669a9fe0..53961c7a 100644 --- a/py/src/braintrust/integrations/temporal/plugin.py +++ b/py/src/braintrust/integrations/temporal/plugin.py @@ -247,6 +247,7 @@ async def execute_activity(self, input: temporalio.worker.ExecuteActivityInput) # Create Braintrust span for activity execution, linked to workflow span span = logger.start_span( + internal={"instrumentation": "temporal-auto"}, name=f"temporal.activity.{info.activity_type}", type="task", parent=parent_span_context or None, @@ -317,6 +318,7 @@ async def execute_workflow(self, input: temporalio.worker.ExecuteWorkflowInput) self._parent_span_context = _workflow_span_context(parent, ids) else: span = logger.start_span( + internal={"instrumentation": "temporal-auto"}, name=f"temporal.workflow.{info.workflow_type}", type="task", parent=parent, @@ -333,6 +335,7 @@ async def execute_workflow(self, input: temporalio.worker.ExecuteWorkflowInput) self._parent_span_context = _workflow_span_context(parent, ids) elif not temporalio.workflow.unsafe.is_replaying(): span = logger.start_span( + internal={"instrumentation": "temporal-auto"}, name=f"temporal.workflow.{info.workflow_type}", type="task", parent=parent_span_context or None, diff --git a/py/src/braintrust/integrations/temporal/test_temporal.py b/py/src/braintrust/integrations/temporal/test_temporal.py index da2c4bba..c726e865 100644 --- a/py/src/braintrust/integrations/temporal/test_temporal.py +++ b/py/src/braintrust/integrations/temporal/test_temporal.py @@ -526,6 +526,10 @@ async def test_plugin_activity_after_replay_stays_under_workflow_span( assert len(workflow_spans) == 1 assert len(activity_spans) == 2 workflow_span = workflow_spans[0] + assert workflow_span["context"]["span_origin"]["instrumentation"]["name"] == "temporal-auto" + assert all( + span["context"]["span_origin"]["instrumentation"]["name"] == "temporal-auto" for span in activity_spans + ) assert all(workflow_span["span_id"] in span.get("span_parents", []) for span in activity_spans) assert all(workflow_span["root_span_id"] == span["root_span_id"] for span in activity_spans) if with_client_parent: diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index f560874b..7c89d51a 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -150,6 +150,20 @@ class ParametersRef(TypedDict, total=False): version: str +class SpanInternalOptions(TypedDict, total=False): + """Options reserved for Braintrust SDK internals. + + Passed as the `internal` kwarg on `start_span` and its variants. External + callers should not use these fields; keys and semantics may change without + notice. + """ + + instrumentation: str + """Identifier for the instrumentation code creating the span. Stamped into + `context.span_origin.instrumentation.name`. Set by SDK integrations + (`openai-auto`, `anthropic-auto`, etc.).""" + + T = TypeVar("T") TMapping = TypeVar("TMapping", bound=Mapping[str, Any]) TMutableMapping = TypeVar("TMutableMapping", bound=MutableMapping[str, Any]) @@ -217,6 +231,7 @@ def start_span( start_time: float | None = None, set_current: bool | None = None, parent: str | dict | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> "Span": """Create a new span. This is useful if you want to log more detailed trace information beyond the scope of a single log event. Data logged over several calls to `Span.log` will be merged into one logical row. @@ -230,6 +245,7 @@ def start_span( :param start_time: Optional start time of the span, as a timestamp in seconds. :param set_current: If true (the default), the span will be marked as the currently-active span for the duration of the context manager. :param parent: Optional parent info string for the span. The string can be generated from `[Span,Experiment,Logger].export`. If not provided, the current span will be used (depending on context). This is useful for adding spans to an existing trace. + :param internal: Reserved for Braintrust SDK internals (see `SpanInternalOptions`). Not for external callers; the keys inside this dict are unstable and may change without notice. :param **event: Data to be logged. See `Experiment.log` for full details. :returns: The newly-created `Span` """ @@ -360,6 +376,7 @@ def start_span( start_time: float | None = None, set_current: bool | None = None, parent: str | dict | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ): return self @@ -2918,6 +2935,7 @@ def start_span( parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, state: BraintrustState | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: """Lower-level alternative to `@traced` for starting a span at the toplevel. It creates a span under the first active object (using the same precedence order as `@traced`), or if `parent` is specified, under the specified parent row, or returns a no-op span object. @@ -2957,6 +2975,7 @@ def start_span( event=event, state=state, lookup_span_parent=False, + internal=internal, ) else: return parent_obj.start_span( @@ -2967,6 +2986,7 @@ def start_span( set_current=set_current, parent=parent, propagated_event=propagated_event, + internal=internal, **event, ) @@ -4262,6 +4282,7 @@ def start_span( set_current: bool | None = None, parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: """Create a new toplevel span underneath the experiment. The name defaults to "root" and the span type to "eval". @@ -4277,6 +4298,7 @@ def start_span( set_current=set_current, parent=parent, propagated_event=propagated_event, + internal=internal, **event, ) @@ -4416,6 +4438,7 @@ def _start_span_impl( parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, lookup_span_parent: bool = True, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: parent_args = _start_span_parent_args( @@ -4437,6 +4460,7 @@ def _start_span_impl( set_current=set_current, event=event, state=self.state, + internal=internal, ) def __enter__(self) -> "Experiment": @@ -4525,6 +4549,7 @@ def __init__( root_span_id: str | None = None, state: BraintrustState | None = None, lookup_span_parent: bool = True, + internal: SpanInternalOptions | None = None, ): if span_attributes is None: span_attributes = SpanAttributes() @@ -4590,9 +4615,10 @@ def __init__( span_attributes=dict(**{"type": type, "name": name, **span_attributes}, exec_counter=exec_counter), created=datetime.datetime.now(datetime.timezone.utc).isoformat(), ) + self._instrumentation = (internal or {}).get("instrumentation") or "braintrust-python-logger" internal_data["context"] = merge_span_origin_context( caller_location or {}, - "braintrust-python-logger", + self._instrumentation, self.state.span_origin_environment, ) @@ -4719,6 +4745,7 @@ def start_span( set_current: bool | None = None, parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: if parent: @@ -4749,6 +4776,7 @@ def start_span( event=event, lookup_span_parent=lookup_span_parent, state=self.state, + internal=internal, ) def end(self, end_time: float | None = None) -> float: @@ -5738,6 +5766,7 @@ def start_span( propagated_event: dict[str, Any] | None = None, span_id: str | None = None, root_span_id: str | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: """Create a new toplevel span underneath the logger. The name defaults to "root" and the span type to "task". @@ -5755,6 +5784,7 @@ def start_span( propagated_event=propagated_event, span_id=span_id, root_span_id=root_span_id, + internal=internal, **event, ) @@ -5789,6 +5819,7 @@ def _start_span_impl( span_id: str | None = None, root_span_id: str | None = None, lookup_span_parent: bool = True, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: parent_args = _start_span_parent_args( @@ -5812,6 +5843,7 @@ def _start_span_impl( root_span_id=root_span_id, lookup_span_parent=lookup_span_parent, state=self.state, + internal=internal, ) def export(self) -> str: diff --git a/py/src/braintrust/test_otel.py b/py/src/braintrust/test_otel.py index 01faf0a6..bb4adcbb 100644 --- a/py/src/braintrust/test_otel.py +++ b/py/src/braintrust/test_otel.py @@ -146,6 +146,32 @@ def test_braintrust_span_processor_merges_span_origin_with_context_json_set_afte provider.shutdown() +def test_merge_span_origin_context_uses_passed_instrumentation_name(): + from braintrust.span_origin import merge_span_origin_context + + merged = merge_span_origin_context({}, "openai-auto", None) + assert merged["span_origin"]["instrumentation"] == {"name": "openai-auto"} + + +def test_span_impl_records_internal_instrumentation(monkeypatch): + monkeypatch.setenv("BRAINTRUST_API_KEY", "test-key") + monkeypatch.delenv("BRAINTRUST_ENVIRONMENT_TYPE", raising=False) + monkeypatch.delenv("BRAINTRUST_ENVIRONMENT_NAME", raising=False) + + from braintrust.logger import init_logger, start_span + + init_logger(project="test_instrumentation_kwarg") + + with start_span(name="parent", internal={"instrumentation": "openai-auto"}) as parent: + with start_span(name="child") as child: + pass + + assert parent._instrumentation == "openai-auto" + # Child spans do NOT inherit the parent's instrumentation name; + # they fall back to the channel default. + assert child._instrumentation == "braintrust-python-logger" + + def test_detect_environment_classifies_aws_ecs_before_lambda(monkeypatch): from braintrust.span_origin import detect_environment