From 826d8b2018b08b62d29e46302c4464966623a534 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Wed, 15 Jul 2026 23:20:50 +0000 Subject: [PATCH 1/9] feat: Per-integration instrumentation.name on span origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every span an integration creates now carries a stable identifier in context.span_origin.instrumentation.name (e.g. openai-auto, anthropic-auto, temporal-auto), matching the naming convention in the Braintrust instrumentation spec example. Previously all integration spans were tagged with the shared braintrust-python-logger default, which made per-integration filtering on span_origin impossible. New public API on start_span: from braintrust import start_span with start_span(name="my-op", instrumentation="my-integration") as s: ... The instrumentation kwarg is threaded through every span-creation surface: top-level start_span, Span.start_span, SpanImpl.start_span, Experiment.start_span, Dataset.start_span, Logger.start_span, and _NoopSpan.start_span. When set, SpanImpl stamps the value into context.span_origin.instrumentation.name via merge_span_origin_context (which gained an override_instrumentation_name parameter). When unset, spans fall back to the channel default (braintrust-python-logger for direct logging, braintrust-python-otel for the OTel processor). The kwarg does NOT propagate through the parent/child edge. Each start_span call independently decides its own value, so a user's @traced scorer nested inside a wrapped provider call keeps the default identifier — the integration's name only lands on spans the integration itself opens. Migration for 23 integrations: - 21 integrations (adk, agentscope, agno, anthropic, autogen, bedrock_runtime, claude_agent_sdk, cohere, crewai, dspy, google_genai, huggingface_hub, instructor, langchain, litellm, livekit_agents, llamaindex, mistral, openai, openai_agents, openrouter, pydantic_ai) use a two-line shadow at the top of their tracing/callbacks module: from braintrust.logger import start_span as _bt_start_span _INSTRUMENTATION = "-auto" def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) Every existing start_span(...) call site in the module then flows through the shadow with zero further edits. - temporal passes instrumentation="temporal-auto" explicitly at each logger.start_span(...) site (not covered by the module shadow because those calls go through a Logger instance, not the imported top-level function). - openai_agents passes instrumentation=_INSTRUMENTATION explicitly on its three parent.start_span(...) / self._logger.start_span(...) sites for the same reason. Tests: - py/src/braintrust/test_otel.py — 4 new unit tests covering merge_span_origin_context resolution order (default, override, empty override falls back), and a SpanImpl-level assertion that child spans do NOT inherit the parent's instrumentation. - py/src/braintrust/integrations/test_span_origin.py — new file with 46 parametrized tests asserting (a) each integration declares _INSTRUMENTATION = "-auto", (b) a span opened through the integration's local start_span emits context.span_origin.instrumentation.name matching, plus static assertions for temporal and openai_agents that every logger.start_span / parent.start_span site carries the explicit kwarg. Docs: - .agents/skills/sdk-integrations/SKILL.md gains a Span origin section documenting the shadow pattern, the no-inheritance rule, and the -auto naming convention. Same file relaxes the earlier strict metadata guidance to an allowlist-per-provider model. Validation: - test_core: 647 passed, 66 skipped, 12 xfailed - test_types: 16 passed (pyright + mypy clean) - pylint: successful - 30 provider (latest) sessions run — every session in the noxfile whose Python 3.14 wheels are available. Zero failures. test_livekit_agents and test_crewai skipped due to pre-existing Python 3.14 wheel gaps in onnxruntime / pydantic-core, unrelated to this change. Note: the JS SDK's mirror PR (braintrust-sdk-javascript#2226) is still open with only braintrust-js-logger / braintrust-otel-js hardcoded and no per-integration wiring. Aligning JS on the same -auto names is worth doing before this ships so cross-SDK dashboards work. --- .agents/skills/sdk-integrations/SKILL.md | 220 ++++++++++++++---- py/src/braintrust/integrations/adk/tracing.py | 10 +- .../integrations/agentscope/tracing.py | 10 +- .../braintrust/integrations/agno/tracing.py | 10 +- .../integrations/anthropic/tracing.py | 10 +- .../integrations/autogen/tracing.py | 10 +- .../integrations/bedrock_runtime/tracing.py | 10 +- .../integrations/claude_agent_sdk/tracing.py | 10 +- .../braintrust/integrations/cohere/tracing.py | 10 +- .../braintrust/integrations/crewai/tracing.py | 10 +- .../braintrust/integrations/dspy/tracing.py | 10 +- .../integrations/google_genai/tracing.py | 10 +- .../integrations/huggingface_hub/tracing.py | 10 +- .../integrations/instructor/tracing.py | 10 +- .../integrations/langchain/callbacks.py | 10 +- .../integrations/litellm/tracing.py | 10 +- .../integrations/livekit_agents/tracing.py | 10 +- .../integrations/llamaindex/tracing.py | 10 +- .../integrations/mistral/tracing.py | 10 +- .../braintrust/integrations/openai/tracing.py | 10 +- .../integrations/openai_agents/tracing.py | 13 +- .../integrations/openrouter/tracing.py | 10 +- .../integrations/pydantic_ai/tracing.py | 10 +- .../integrations/temporal/plugin.py | 3 + .../integrations/test_span_origin.py | 131 +++++++++++ py/src/braintrust/logger.py | 19 ++ py/src/braintrust/span_origin.py | 4 +- py/src/braintrust/test_otel.py | 50 ++++ 28 files changed, 587 insertions(+), 63 deletions(-) create mode 100644 py/src/braintrust/integrations/test_span_origin.py diff --git a/.agents/skills/sdk-integrations/SKILL.md b/.agents/skills/sdk-integrations/SKILL.md index 8dc34fcb..18d35496 100644 --- a/.agents/skills/sdk-integrations/SKILL.md +++ b/.agents/skills/sdk-integrations/SKILL.md @@ -13,14 +13,34 @@ Use `sdk-wrapper-migrations` instead when the provider already has a real implem Before editing: -1. Read the shared integration primitives. -2. Read the target provider package. -3. Pick the nearest existing integration as a reference. -4. Confirm provider versions and nox sessions from source files, not memory. -5. Decide the span shape before writing patchers. -6. Run the narrowest provider nox session first. +1. Read the Braintrust instrumentation spec (see [Spec](#spec)). +2. Read the shared integration primitives. +3. Read the target provider package. +4. Pick the nearest existing integration as a reference. +5. Confirm provider versions and nox sessions from source files, not memory. +6. Decide the span shape before writing patchers. +7. Run the narrowest provider nox session first. -Do not design a new integration shape from scratch if an existing provider already matches the problem. +Do not design a new integration shape from scratch if an existing provider already matches the problem. Do not invent span fields, metrics, or metadata keys that are not already in the spec. + +## Spec + +The authoritative source for what integrations must capture, how spans must be structured, and which fields are allowed is the Braintrust instrumentation guide: + +- `braintrustdata/braintrust-spec` -> `docs/instrumentation-guide.md` + +Read the spec first when the task involves: + +- span type, name, or hierarchy decisions +- what to put in `input`, `output`, `metadata`, `metrics`, `error`, or `context` +- tool calls, tool results, or available tool definitions +- prompt provenance metadata +- streaming, reasoning models, or prompt caching +- multimodal or attachment payloads +- distinguishing completion-style vs agentic-style APIs +- naming or namespacing new attributes + +Instrumentation MUST NOT capture data or emit metric/metadata keys that are not explicitly allowed by the spec. If the task needs a new field, update the spec before shipping the SDK change. ## Read First @@ -92,23 +112,28 @@ Choose the reference based on the hardest part of the task: Use this order unless the task is clearly narrower: -1. Read shared primitives and the provider package. -2. Decide which public surface is being patched. -3. Define the span shape: +1. Read the spec sections that cover the affected span shape. +2. Read shared primitives and the provider package. +3. Decide whether the target API is completion-style or agentic-style (see [Span Design Rules](#span-design-rules)). +4. Decide which public surface is being patched. Prefer stable public API entry points over internal helpers; internal targets break more often across provider versions. +5. Define the span shape: + - span `type` and `name` - `input` - `output` - `metadata` - `metrics` - `error` when failures matter -4. Implement or update patchers. -5. Implement or update tracing helpers. -6. Add or update focused tests. -7. For provider-behavior bugs, make the primary regression test cassette-backed when practical, even if the implementation change is in local tracing/span post-processing of the provider response. -8. Be suspicious of mock/fake coverage for integrations. Do not choose mocks because they are convenient, faster, or easier to control. -9. Run the narrowest nox session first, then expand only if shared code changed. +6. Implement or update patchers. +7. Implement or update tracing helpers. +8. Add or update focused tests. +9. For provider-behavior bugs, make the primary regression test cassette-backed when practical, even if the implementation change is in local tracing/span post-processing of the provider response. +10. Be suspicious of mock/fake coverage for integrations. Do not choose mocks because they are convenient, faster, or easier to control. +11. Run the narrowest nox session first, then expand only if shared code changed. Do not start by wiring wrappers and only later decide what the span should contain. +For behavior changes, prefer adding or adjusting a failing test first, then implementing until it passes. + ## Route The Task ### New provider integration @@ -125,6 +150,7 @@ Do not start by wiring wrappers and only later decide what the span should conta 4. Add or update the provider session in `py/noxfile.py`. 5. Update `py/src/braintrust/auto.py` only if the integration should participate in `auto_instrument()`. 6. Add subprocess coverage in `py/src/braintrust/integrations/auto_test_scripts/` when `auto_instrument()` changes. +7. Add the `_INSTRUMENTATION = "-auto"` shadow described in [Span origin](#span-origin-instrumentationname) at the top of `tracing.py`, and include an assertion in the integration's test suite that `context.span_origin.instrumentation.name` matches. ### Existing integration update @@ -133,7 +159,7 @@ Do not start by wiring wrappers and only later decide what the span should conta 3. Preserve the provider's public setup and `wrap_*()` surface unless the task explicitly changes it. 4. Do not touch `auto.py`, `integrations/__init__.py`, or `py/noxfile.py` unless the task requires it. 5. Even if `auto.py` does not change, check whether the behavior change also needs an auto-instrument subprocess test update. -6. Preserve existing span shape conventions unless the task is intentionally correcting them. +6. Preserve existing span shape conventions unless the task is intentionally correcting them to match the spec. ### `auto_instrument()` only @@ -150,7 +176,7 @@ Treat these as distinct entry points: - public `wrap_*()` helpers: manual wrapping of a provided class, function, or client - `auto_instrument()`: import-order-sensitive discovery and setup -When changing one entry point, check whether the other two should keep equivalent span behavior. If `auto_instrument()` changes or could be affected by import timing, validate it with a subprocess test instead of only calling the integration in-process. +When changing one entry point, check whether the other two should keep equivalent span behavior. Auto-instrument and manual paths should emit equivalent spans; if a provider-facing behavior change goes into one path, it usually needs to go into the other. If `auto_instrument()` changes or could be affected by import timing, validate it with a subprocess test instead of only calling the integration in-process. ## Package Layout Rules @@ -167,7 +193,7 @@ Typical ownership: Keep `integration.py` thin. -If logic is genuinely shared across integrations, move it to `py/src/braintrust/integrations/utils.py` instead of copying it into multiple providers. +If logic is genuinely shared across integrations, move it to `py/src/braintrust/integrations/utils.py` instead of copying it into multiple providers. Before adding a local helper, check `utils.py` and neighboring integrations for an existing one. ## Integration Rules @@ -186,28 +212,107 @@ Prefer feature detection first and version checks second. Use: Let `BaseIntegration.resolve_patchers()` reject duplicate patcher ids. Do not hide duplicates. +### Non-invasiveness and safety + +Instrumentation must not change user-visible behavior: + +- Errors from the provider MUST propagate to the caller. Do not swallow or rewrap them. +- Return values, iterator/async-iterator semantics, and generator/subclass behavior MUST be preserved. +- Sync and async traced schemas MUST stay aligned when the provider exposes both. +- Setup, teardown, and patching MUST be idempotent. Applying setup twice, tearing down twice, or wrapping twice must remain safe. Rely on the base patcher's idempotence marker. +- Contain instrumentation failures. Errors in extraction, normalization, or logging code must be logged or ignored, not raised into the user's call path. Only intentional provider errors should surface. +- Treat provider inputs, results, events, headers, and metadata as untrusted. Avoid attribute access that could trigger arbitrary code, and avoid mutating third-party objects. +- Limit instrumentation to AI-generation-relevant operations (LLM calls, embeddings, tool executions, media generation, agent runs). Do not instrument unrelated CRUD or platform-management APIs. + +### Scope of instrumentation + +Follow the spec closely for spec-defined fields — `input`, `output`, `metrics`, `error`, tool-call structure, prompt provenance, attachments. Do not invent new top-level fields, metric keys, or span types; update the spec first if you genuinely need one. + +`metadata` is treated slightly more loosely: use an **allowlist per provider**, not a denylist. Capture the spec-defined keys (`model`, `provider`, `tools`, `tool_choice`, `parallel_tool_calls`, `max_tool_calls`, `prompt`, and the permitted request-config subset) plus provider-specific detail fields you deliberately choose to include — provider request/response ids, safety data, model-family flags, and similar. Do not dump entire raw request/response objects into metadata; each captured key should be an intentional decision. Redact obvious secrets. If a provider-specific field ends up broadly useful across SDKs, promote it to the spec. + Preserve provider behavior. Tracing code must not change return values, control flow, or error behavior unless the task explicitly requires it. -Keep sync and async traced schemas aligned when the provider exposes both. +### Span origin (`instrumentation.name`) + +Every span an integration creates MUST carry `context.span_origin.instrumentation.name` identifying which integration produced it. Naming convention: `-auto` (e.g. `openai-auto`, `anthropic-auto`, `adk-auto`, `google-genai-auto`). Match the JS SDK exactly for cross-SDK dashboards to work. + +Braintrust's `start_span(...)` (and every provider-level `start_span` method) accepts an `instrumentation: str | None` kwarg. When set, the SDK stamps it into `context.span_origin.instrumentation.name`. When unset, spans fall back to the channel default (`braintrust-python-logger` for direct logging, `braintrust-python-otel` for the OTel processor). + +**Only integration-created spans should carry the integration's name.** User-owned spans nested inside a wrapped provider call — for example a `@traced` scorer running inside a wrapped agent turn — must NOT inherit the integration's `instrumentation.name`. The value does not propagate through the span parent/child edge; each `start_span` call independently decides its own value. + +The standard per-integration pattern is a two-line shadow at the top of the integration's `tracing.py` (or `callbacks.py`/`plugin.py`): + +```python +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "-auto" + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) +``` + +Every direct `start_span(...)` call site inside the module then flows through the shadow automatically. If the integration opens spans through a `Logger`, `Experiment`, or existing `Span` instance (i.e. `logger.start_span(...)`, `parent.start_span(...)`), pass `instrumentation=_INSTRUMENTATION` explicitly at those sites — the shadow only catches module-level `start_span` calls. + +Tests should assert on the emitted span's `context.span_origin.instrumentation.name`. `SpanImpl` also exposes the resolved value as `span._instrumentation` for direct inspection in tests. ## Span Design Rules +### Span type and name + +Every span must set the correct `type` and a stable `name`: + +- `llm`: a single provider API call +- `task`: a parent span for an agent run, pipeline step, or named operation +- `tool`: a model-initiated tool/function execution + +Use spec-recommended names where they exist (for example `Chat Completion` for OpenAI, `anthropic.messages.create` for Anthropic, `generate_content` for Google). For new providers, pick a stable, provider-specific name. + +### Completion-style vs agentic-style + +Decide the span shape before writing patchers: + +- Completion-style APIs (single request/response, user executes any tool calls): one `llm` span per API call, no children. Tool calls the model requests appear in the span's `output`; the SDK does not create `tool` spans. +- Agentic-style APIs (the SDK/framework runs the tool loop internally): one parent `task` span wrapping child `llm` spans (one per underlying model call) and child `tool` spans (one per tool execution). Preserve execution order. + +For frameworks: completion-style frameworks (for example LiteLLM) must produce provider-shaped `llm` spans and set `metadata.provider` to the underlying provider (`openai`, `anthropic`, etc.), not the framework name. Agentic-style frameworks (Vercel AI SDK with tools, LangChain agents, OpenAI Agents SDK, Claude Agent SDK, ADK, Agno) must produce the full parent-plus-children tree. + +### Canonical payload format + +Braintrust uses the OpenAI Chat Completions message format as the canonical representation for LLM `input` and `output`. The UI parses, displays, and diffs spans assuming this format. + +- For providers that have a dedicated Braintrust UI normalizer (currently OpenAI, Anthropic, Google), instrumentation MAY preserve the provider-native payload. Set `metadata.provider` correctly so the UI can normalize. +- For any other provider, instrumentation MUST convert payloads into the OpenAI Chat Completions format so the UI can render them. If the provider exposes a system prompt as a separate parameter, insert it into the messages array as a `role: "system"` entry. + +### Input, output, metadata, metrics, error + Build readable spans. Do not dump raw `args` and `kwargs` unless the provider API already exposes a clean schema. Use this rubric: -- `input`: the meaningful user request -- `output`: the meaningful provider result -- `metadata`: supporting context such as ids, finish reasons, safety data, model revisions -- `metrics`: timing and numeric accounting such as token counts or elapsed time -- `error`: exceptions or failure information +- `input`: the meaningful user request (messages, prompt, or provider-native input structure) +- `output`: the meaningful provider result (choices, message, or provider-native response structure) +- `metadata`: only spec-allowed fields such as `model`, `provider`, `tools`, `tool_choice`, `parallel_tool_calls`, `max_tool_calls`, `prompt`, and the permitted request-config subset (`temperature`, `top_p`, `max_tokens`, `frequency_penalty`, `presence_penalty`, `stop`, `response_format`) +- `metrics`: only spec-allowed keys (see [Metrics](#metrics)) +- `error`: exceptions or failure information; pass the `Exception` instance directly to span logging rather than pre-formatting it into strings + +Every `llm` span MUST include `metadata.model` (the resolved model id from the response, including any version suffix when the API returns one) and `metadata.provider` (the provider whose pricing applies, even when the caller went through a gateway or framework). + +Tool definitions are request configuration, not conversation content: the schemas passed to the model MUST go in `metadata.tools` (OpenAI-shaped, regardless of the underlying provider), not in the `input` messages. Preserve provider-native built-in tool types (for example Anthropic `computer_use`, OpenAI `web_search`) as opaque JSON-serializable entries in `metadata.tools`. Do not log executable tool handlers or closures. + +Prompt provenance for Braintrust-managed prompts MUST go in `metadata.prompt` (`id`, `project_id`, `version`, `variables`, and `prompt_session_id` for playground/prompt-session calls). Do not put prompt provenance in `input`, request payloads, or `metadata.tools`. Wrapper-only carrier fields such as `span_info` are internal plumbing; strip them from the provider request and do not log them. + +### Token metrics Avoid double-counting token metrics: - the integration that directly owns the model/provider API response should own token accounting -- orchestration/framework integrations should usually not log token metrics when underlying provider integrations can create leaf spans with usage metrics +- orchestration/framework integrations should usually not log token metrics when underlying provider integrations can create leaf spans with usage metrics; agentic parent `task` spans MAY aggregate token counts across children when the framework does not delegate to a separately instrumented provider client - do not add fragile provider-specific ownership checks such as "if OpenAI is patched, skip metrics"; prefer a clear span ownership rule instead +### Shaping guidance + Good span shaping usually means: - flatten positional arguments into named fields @@ -232,18 +337,42 @@ def _process_result(result: Any, start: float) -> tuple[dict[str, Any], dict[str ... ``` +## Metrics + +Only emit spec-listed metric keys. Do not invent metric keys. + +Standard LLM metrics: + +- `tokens`, `prompt_tokens`, `completion_tokens`: required for LLM spans (values MUST be non-negative) +- `time_to_first_token`: required for streaming spans; measured by the SDK from request start to the first chunk +- `completion_reasoning_tokens`: required when the provider reports reasoning tokens (for example OpenAI o-series) +- `prompt_cached_tokens`, `prompt_cache_creation_tokens`, `prompt_cache_creation_5m_tokens`, `prompt_cache_creation_1h_tokens`: capture when the provider reports prompt caching +- `prompt_audio_tokens`, `completion_audio_tokens`, `completion_image_tokens`: capture when the provider reports them for audio/image models +- `start`, `end`: standard span timing + +Streaming spans still produce a single `llm` span per API call, with `input`/`output` accumulated from chunks. Capture token counts from stream usage metadata when the provider surfaces it (for example OpenAI `stream_options.include_usage`). + +For reasoning models, capture the full output structure (reasoning summaries plus assistant message blocks) when the provider exposes them, and include prior reasoning blocks in the `input` for multi-turn calls. + +## Multimodal And Attachments + +When a request or response contains inline binary media (images, PDFs, audio, video, and similar), replace the raw media leaf in the span payload with a Braintrust attachment. Do not add a separate top-level `attachments` list. + Treat binary payloads as attachments, not logged bytes: - prefer the shared `_materialize_attachment(...)` helper in `py/src/braintrust/integrations/utils.py` over provider-local base64 or file-decoding code -- convert provider-owned raw `bytes`, base64 payloads, data URLs, file inputs, and generated media into `braintrust.logger.Attachment` objects when Braintrust should upload the content -- preserve normal remote URLs as strings +- convert provider-owned raw `bytes`, base64 payloads, data URLs, file inputs, and generated media into `braintrust.logger.Attachment` objects when Braintrust should upload the content (the SDK converts these into `braintrust_attachment` references on the wire) +- preserve normal remote URLs as strings; do not fetch remote URLs solely to create an attachment - use the repo's existing multimodal payload shapes after materialization: - images -> `{"image_url": {"url": attachment}}` - non-image media/documents/files -> `{"file": {"file_data": attachment, "filename": resolved.filename}}` - do not force non-image payloads through `image_url` shims -- if attachment materialization fails, keep the original value instead of dropping it or replacing it with `None` +- if attachment materialization or upload fails, keep the original value instead of dropping it or replacing it with `None`, and do not raise into the user's call path - preserve non-attachment values while walking nested payloads unless you are intentionally normalizing them for readability - keep useful metadata such as MIME type, size, safety data, filenames, or provider ids next to the attachment +- when a provider-native payload has a dedicated UI normalizer (OpenAI, Anthropic, Google), you may preserve the provider-native structure and replace only the raw media leaf + +For generated media on the output side, log the attachment inside `output` (not `metadata`). For streaming media outputs, aggregate chunks into the final `output` attachment; do not log raw chunks as separate blobs. ## Patcher Rules @@ -275,6 +404,8 @@ Require every patcher to have: - version gating only when necessary - idempotence through the base patcher marker +Prefer patching stable public API surfaces (documented client methods, top-level constructors). Reach into internal helpers only when the public surface is genuinely insufficient, and expect those patches to need version-specific maintenance. + ## Testing Rules Keep tests in the provider package. @@ -298,7 +429,19 @@ Use mocks or fakes only for cases that are hard to drive through recordings, suc Do not replace or skip a cassette-backed regression with a mock/fake test merely because the implementation change lives in `tracing.py`, a serializer, or another local post-processing layer. If a real provider payload is what triggers the bug, the main regression test should reflect that real payload. -Test emitted spans, not just provider return values. +Test emitted spans, not just provider return values. In particular, assert on: + +- span `type` and `name` +- `input` shape (messages/prompt/config fields the spec requires) +- `output` shape (normalized provider result, not opaque SDK instances) +- `metadata.model`, `metadata.provider`, and any `metadata.tools` / `metadata.tool_choice` / `metadata.prompt` present +- required `metrics` keys (token counts for LLM spans; `time_to_first_token` for streaming) +- parent/child structure for agentic APIs (parent `task`, child `llm` and `tool` spans in execution order) +- `context.span_origin.instrumentation.name` equals the integration's `-auto` identifier +- attachment conversion for binary inputs or generated media, including that images land under `image_url.url`, non-image payloads land under `file.file_data`, and traced payloads contain `Attachment` objects rather than raw bytes or base64 blobs +- error propagation and error logging behavior +- idempotence of setup/teardown/wrapping +- patcher resolution and duplicate detection when relevant Cover the surfaces that changed: @@ -309,22 +452,12 @@ Cover the surfaces that changed: - streaming behavior - idempotence - failure and error logging -- patcher resolution and duplicate detection when relevant -- attachment conversion for binary inputs or generated media, including assertions that images land under `image_url.url`, non-image payloads land under `file.file_data`, and traced payloads contain `Attachment` objects rather than raw bytes or base64 blobs -- span structure, especially `input`, `output`, `metadata`, and `metrics` For streaming changes, verify both: - the provider still returns the expected iterator or async iterator - the final logged span contains the aggregated `output` and stream-specific `metrics` -Also verify, when relevant: - -- the `input` contains the expected model/messages/prompt/config fields -- the `output` contains normalized provider results rather than opaque SDK instances -- the `metadata` contains finish reasons, ids, or annotations in the expected place -- binary payloads are represented as `Attachment` objects where applicable, while remote URLs and non-attachment values remain unchanged and unmaterialized file inputs are preserved rather than dropped - Keep VCR cassettes in `py/src/braintrust/integrations//cassettes//` (e.g. `cassettes/latest/`, `cassettes/0.48.0/`). Nox sessions set `BRAINTRUST_TEST_PACKAGE_VERSION` automatically so cassettes land in the correct version subdirectory. Do not add per-test `vcr_cassette_dir` or `cassette_library_dir` fixtures; the shared `py/src/braintrust/integrations/conftest.py` handles version routing. Re-record only when behavior intentionally changes. When the provider returns binary HTTP responses or generated media, sanitize cassettes as needed so fixtures do not store raw file bytes. @@ -362,8 +495,15 @@ Avoid these failures: - re-recording cassettes when behavior did not intentionally change - adding a custom `_instrument_*` helper where `_instrument_integration()` already fits - forgetting `target_module` for deep or optional patch targets +- inventing new top-level span fields, metric keys, or span types beyond what the spec allows (metadata is treated more loosely — see the metadata allowlist guidance above) +- putting tool definitions in `input` instead of `metadata.tools`, or including prompt provenance anywhere other than `metadata.prompt` +- forgetting to set `instrumentation=-auto` on spans an integration opens, or leaking that identifier onto user-owned spans nested inside a wrapped call +- using the framework name as `metadata.provider` for a completion-style framework wrapper instead of the underlying provider +- producing per-chunk spans for a streamed call instead of one accumulated span per API call +- letting instrumentation errors surface into the user's call path, or swallowing real provider errors - double-counting token metrics in both orchestration/framework spans and provider leaf spans - adding provider-specific token ownership detection instead of defining clear metric ownership for the integration - doing excessive serialization/stringification in tracing code even though Braintrust serializes span payloads at send/log time - wrapping Braintrust span logging methods in broad `try`/`except` blocks even though those methods are designed not to throw - forcing non-image attachments through `image_url` shims, dropping unrecognized file inputs, or re-serializing non-attachment values while materializing payloads +- patching internal helpers when a stable public API surface would work diff --git a/py/src/braintrust/integrations/adk/tracing.py b/py/src/braintrust/integrations/adk/tracing.py index c84b51b2..7d74e426 100644 --- a/py/src/braintrust/integrations/adk/tracing.py +++ b/py/src/braintrust/integrations/adk/tracing.py @@ -12,7 +12,15 @@ from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "adk-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/agentscope/tracing.py b/py/src/braintrust/integrations/agentscope/tracing.py index 06bea174..f8f9e742 100644 --- a/py/src/braintrust/integrations/agentscope/tracing.py +++ b/py/src/braintrust/integrations/agentscope/tracing.py @@ -5,7 +5,15 @@ from contextlib import aclosing from typing import Any -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "agentscope-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py index 65a6ff7e..8761f581 100644 --- a/py/src/braintrust/integrations/agno/tracing.py +++ b/py/src/braintrust/integrations/agno/tracing.py @@ -3,7 +3,15 @@ from typing import Any from braintrust.integrations.utils import _try_to_dict -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "agno-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index eacacb6e..388968c0 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -6,7 +6,15 @@ from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.anthropic._utils import Wrapper, _try_to_dict, extract_anthropic_usage from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import log_exc_info_to_span, start_span +from braintrust.logger import log_exc_info_to_span, start_span as _bt_start_span + +_INSTRUMENTATION = "anthropic-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/autogen/tracing.py b/py/src/braintrust/integrations/autogen/tracing.py index 6072d644..586003c3 100644 --- a/py/src/braintrust/integrations/autogen/tracing.py +++ b/py/src/braintrust/integrations/autogen/tracing.py @@ -4,7 +4,15 @@ from contextvars import ContextVar from typing import Any -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "autogen-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/bedrock_runtime/tracing.py b/py/src/braintrust/integrations/bedrock_runtime/tracing.py index 0d1f6605..151f6caf 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/tracing.py +++ b/py/src/braintrust/integrations/bedrock_runtime/tracing.py @@ -11,7 +11,15 @@ _materialize_attachment, _timing_metrics, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "bedrock-runtime-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py index 631b6f99..b9a79d50 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py @@ -23,7 +23,15 @@ MessageClassName, SerializedContentType, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "claude-agent-sdk-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/cohere/tracing.py b/py/src/braintrust/integrations/cohere/tracing.py index 0f563616..499b530c 100644 --- a/py/src/braintrust/integrations/cohere/tracing.py +++ b/py/src/braintrust/integrations/cohere/tracing.py @@ -21,7 +21,15 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "cohere-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/crewai/tracing.py b/py/src/braintrust/integrations/crewai/tracing.py index 77a1ce4e..b261160c 100644 --- a/py/src/braintrust/integrations/crewai/tracing.py +++ b/py/src/braintrust/integrations/crewai/tracing.py @@ -47,7 +47,15 @@ _parse_openai_usage_metrics, _try_to_dict, ) -from braintrust.logger import NOOP_SPAN, Span, current_span, start_span +from braintrust.logger import NOOP_SPAN, Span, current_span, start_span as _bt_start_span + +_INSTRUMENTATION = "crewai-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index dbd376f7..7a5de914 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -2,7 +2,15 @@ from typing import Any -from braintrust.logger import current_span, start_span +from braintrust.logger import current_span, start_span as _bt_start_span + +_INSTRUMENTATION = "dspy-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/google_genai/tracing.py b/py/src/braintrust/integrations/google_genai/tracing.py index 3f231f73..4e02adb0 100644 --- a/py/src/braintrust/integrations/google_genai/tracing.py +++ b/py/src/braintrust/integrations/google_genai/tracing.py @@ -10,7 +10,15 @@ from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import NOOP_SPAN, _state, current_logger, current_span, parent_context, start_span +from braintrust.logger import NOOP_SPAN, _state, current_logger, current_span, parent_context, start_span as _bt_start_span + +_INSTRUMENTATION = "google-genai-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/huggingface_hub/tracing.py b/py/src/braintrust/integrations/huggingface_hub/tracing.py index 2781bd1c..435a5e93 100644 --- a/py/src/braintrust/integrations/huggingface_hub/tracing.py +++ b/py/src/braintrust/integrations/huggingface_hub/tracing.py @@ -34,7 +34,15 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "huggingface-hub-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/instructor/tracing.py b/py/src/braintrust/integrations/instructor/tracing.py index e419af0f..e2f2bace 100644 --- a/py/src/braintrust/integrations/instructor/tracing.py +++ b/py/src/braintrust/integrations/instructor/tracing.py @@ -28,7 +28,15 @@ from collections.abc import AsyncIterator, Iterator from typing import Any -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "instructor-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/langchain/callbacks.py b/py/src/braintrust/integrations/langchain/callbacks.py index 2cb9b37d..376da3c9 100644 --- a/py/src/braintrust/integrations/langchain/callbacks.py +++ b/py/src/braintrust/integrations/langchain/callbacks.py @@ -11,7 +11,15 @@ from uuid import UUID from braintrust.generated_types import SpanAttributes -from braintrust.logger import NOOP_SPAN, Logger, Span, current_span, init_logger, start_span +from braintrust.logger import NOOP_SPAN, Logger, Span, current_span, init_logger, start_span as _bt_start_span + +_INSTRUMENTATION = "langchain-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.version import VERSION as sdk_version from langchain_core.agents import AgentAction, AgentFinish diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index 3dd1d8f9..8a9352c3 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -14,7 +14,15 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import Span, start_span +from braintrust.logger import Span, start_span as _bt_start_span + +_INSTRUMENTATION = "litellm-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones, is_numeric, merge_dicts diff --git a/py/src/braintrust/integrations/livekit_agents/tracing.py b/py/src/braintrust/integrations/livekit_agents/tracing.py index 7adfab31..def4fabb 100644 --- a/py/src/braintrust/integrations/livekit_agents/tracing.py +++ b/py/src/braintrust/integrations/livekit_agents/tracing.py @@ -16,10 +16,18 @@ _state, current_span, parent_context, - start_span, + start_span as _bt_start_span, ) +_INSTRUMENTATION = "livekit-agents-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + + _SESSION_PARENT_ATTR = "__braintrust_livekit_session_parent__" _SESSION_SPAN_ATTR = "__braintrust_livekit_session_span__" _USER_SPEAKING_SPAN_ATTR = "__braintrust_livekit_user_speaking_span__" diff --git a/py/src/braintrust/integrations/llamaindex/tracing.py b/py/src/braintrust/integrations/llamaindex/tracing.py index ee2a6542..151594d2 100644 --- a/py/src/braintrust/integrations/llamaindex/tracing.py +++ b/py/src/braintrust/integrations/llamaindex/tracing.py @@ -4,7 +4,15 @@ import time from typing import Any -from braintrust.logger import NOOP_SPAN, Span, current_span, start_span +from braintrust.logger import NOOP_SPAN, Span, current_span, start_span as _bt_start_span + +_INSTRUMENTATION = "llamaindex-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/mistral/tracing.py b/py/src/braintrust/integrations/mistral/tracing.py index 30cfb20c..f57d9987 100644 --- a/py/src/braintrust/integrations/mistral/tracing.py +++ b/py/src/braintrust/integrations/mistral/tracing.py @@ -16,7 +16,15 @@ _materialize_attachment, _merge_timing_and_usage_metrics, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "mistral-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 8739cf00..ead44b30 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -19,7 +19,15 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import Span, start_span +from braintrust.logger import Span, start_span as _bt_start_span + +_INSTRUMENTATION = "openai-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones, merge_dicts from wrapt import FunctionWrapper diff --git a/py/src/braintrust/integrations/openai_agents/tracing.py b/py/src/braintrust/integrations/openai_agents/tracing.py index bf1a7274..1641daa7 100644 --- a/py/src/braintrust/integrations/openai_agents/tracing.py +++ b/py/src/braintrust/integrations/openai_agents/tracing.py @@ -4,7 +4,15 @@ from typing import Any from agents import tracing -from braintrust.logger import NOOP_SPAN, Experiment, Logger, Span, current_span, flush, start_span +from braintrust.logger import NOOP_SPAN, Experiment, Logger, Span, current_span, flush, start_span as _bt_start_span + +_INSTRUMENTATION = "openai-agents-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute @@ -132,6 +140,7 @@ def on_trace_start(self, trace: tracing.Trace) -> None: name=trace.name, span_attributes={"type": "task", "name": trace.name}, metadata=metadata, + instrumentation=_INSTRUMENTATION, ) elif self._logger is not None: span = self._logger.start_span( @@ -139,6 +148,7 @@ def on_trace_start(self, trace: tracing.Trace) -> None: span_id=trace.trace_id, root_span_id=trace.trace_id, metadata=metadata, + instrumentation=_INSTRUMENTATION, ) else: span = start_span( @@ -336,6 +346,7 @@ def on_span_start(self, span: tracing.Span[tracing.SpanData]) -> None: name=_span_name(span), type=_span_type(span), start_time=_timestamp_from_maybe_iso(span.started_at), + instrumentation=_INSTRUMENTATION, ) self._spans[span.span_id] = created_span created_span.set_current() diff --git a/py/src/braintrust/integrations/openrouter/tracing.py b/py/src/braintrust/integrations/openrouter/tracing.py index f9711bb2..8474b905 100644 --- a/py/src/braintrust/integrations/openrouter/tracing.py +++ b/py/src/braintrust/integrations/openrouter/tracing.py @@ -13,7 +13,15 @@ _log_error_and_end_span, _merge_timing_and_usage_metrics, ) -from braintrust.logger import start_span +from braintrust.logger import start_span as _bt_start_span + +_INSTRUMENTATION = "openrouter-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index 64de3a6b..0c979b2b 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -8,7 +8,15 @@ from typing import Any from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import _internal_get_global_state, start_span +from braintrust.logger import _internal_get_global_state, start_span as _bt_start_span + +_INSTRUMENTATION = "pydantic-ai-auto" # instrumentation shadow: do not edit + + +def start_span(*args, **kwargs): + kwargs.setdefault("instrumentation", _INSTRUMENTATION) + return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from wrapt import wrap_function_wrapper diff --git a/py/src/braintrust/integrations/temporal/plugin.py b/py/src/braintrust/integrations/temporal/plugin.py index 669a9fe0..52f82ce0 100644 --- a/py/src/braintrust/integrations/temporal/plugin.py +++ b/py/src/braintrust/integrations/temporal/plugin.py @@ -247,6 +247,7 @@ async def execute_activity(self, input: temporalio.worker.ExecuteActivityInput) # Create Braintrust span for activity execution, linked to workflow span span = logger.start_span( + instrumentation="temporal-auto", name=f"temporal.activity.{info.activity_type}", type="task", parent=parent_span_context or None, @@ -317,6 +318,7 @@ async def execute_workflow(self, input: temporalio.worker.ExecuteWorkflowInput) self._parent_span_context = _workflow_span_context(parent, ids) else: span = logger.start_span( + instrumentation="temporal-auto", name=f"temporal.workflow.{info.workflow_type}", type="task", parent=parent, @@ -333,6 +335,7 @@ async def execute_workflow(self, input: temporalio.worker.ExecuteWorkflowInput) self._parent_span_context = _workflow_span_context(parent, ids) elif not temporalio.workflow.unsafe.is_replaying(): span = logger.start_span( + instrumentation="temporal-auto", name=f"temporal.workflow.{info.workflow_type}", type="task", parent=parent_span_context or None, diff --git a/py/src/braintrust/integrations/test_span_origin.py b/py/src/braintrust/integrations/test_span_origin.py new file mode 100644 index 00000000..d477c016 --- /dev/null +++ b/py/src/braintrust/integrations/test_span_origin.py @@ -0,0 +1,131 @@ +"""Cross-integration assertions for `context.span_origin.instrumentation.name`. + +Each integration's `tracing.py` (or `callbacks.py`/`plugin.py`) shadows the +module-level `start_span` to stamp the integration's identifier. This test +suite opens a span through each shadow and confirms the emitted context +carries the expected `-auto` name. Provider packages are NOT +imported here — only the local shadow — so the file runs cleanly under +`test_core`. +""" + +from __future__ import annotations + +import importlib +from typing import Any + +import pytest + +from braintrust import logger +from braintrust.test_helpers import init_test_logger + +# Each entry: (module path, expected `-auto` name). +_SHADOW_TARGETS: list[tuple[str, str]] = [ + ("braintrust.integrations.adk.tracing", "adk-auto"), + ("braintrust.integrations.agentscope.tracing", "agentscope-auto"), + ("braintrust.integrations.agno.tracing", "agno-auto"), + ("braintrust.integrations.anthropic.tracing", "anthropic-auto"), + ("braintrust.integrations.autogen.tracing", "autogen-auto"), + ("braintrust.integrations.bedrock_runtime.tracing", "bedrock-runtime-auto"), + ("braintrust.integrations.claude_agent_sdk.tracing", "claude-agent-sdk-auto"), + ("braintrust.integrations.cohere.tracing", "cohere-auto"), + ("braintrust.integrations.crewai.tracing", "crewai-auto"), + ("braintrust.integrations.dspy.tracing", "dspy-auto"), + ("braintrust.integrations.google_genai.tracing", "google-genai-auto"), + ("braintrust.integrations.huggingface_hub.tracing", "huggingface-hub-auto"), + ("braintrust.integrations.instructor.tracing", "instructor-auto"), + ("braintrust.integrations.langchain.callbacks", "langchain-auto"), + ("braintrust.integrations.litellm.tracing", "litellm-auto"), + ("braintrust.integrations.livekit_agents.tracing", "livekit-agents-auto"), + ("braintrust.integrations.llamaindex.tracing", "llamaindex-auto"), + ("braintrust.integrations.mistral.tracing", "mistral-auto"), + ("braintrust.integrations.openai.tracing", "openai-auto"), + ("braintrust.integrations.openai_agents.tracing", "openai-agents-auto"), + ("braintrust.integrations.openrouter.tracing", "openrouter-auto"), + ("braintrust.integrations.pydantic_ai.tracing", "pydantic-ai-auto"), +] + +_PROJECT = "test-span-origin" + + +@pytest.fixture +def memory_logger() -> Any: + init_test_logger(_PROJECT) + with logger._internal_with_memory_background_logger() as bgl: + yield bgl + logger._state.current_experiment = None + logger._state.reset_parent_state() + + +def _try_import(module_path: str): + try: + return importlib.import_module(module_path) + except Exception: + return None + + +@pytest.mark.parametrize(("module_path", "expected"), _SHADOW_TARGETS) +def test_integration_shadow_declares_instrumentation(module_path: str, expected: str) -> None: + """Every integration exposes `_INSTRUMENTATION = "-auto"` at module scope.""" + module = _try_import(module_path) + if module is None: + pytest.skip(f"{module_path}: optional provider dep not installed") + assert getattr(module, "_INSTRUMENTATION", None) == expected, ( + f"{module_path} should declare _INSTRUMENTATION = {expected!r}" + ) + + +@pytest.mark.parametrize(("module_path", "expected"), _SHADOW_TARGETS) +def test_integration_shadow_stamps_span_origin(module_path: str, expected: str, memory_logger) -> None: + """A span opened via the integration's local `start_span` carries the expected instrumentation name.""" + module = _try_import(module_path) + if module is None: + pytest.skip(f"{module_path}: optional provider dep not installed") + start_span = getattr(module, "start_span", None) + if start_span is None: + pytest.skip(f"{module_path}: no module-level start_span shadow") + + with start_span(name="span-origin-check") as span: + # SpanImpl exposes the resolved instrumentation name as `_instrumentation`. + assert getattr(span, "_instrumentation", None) == expected + + spans = memory_logger.pop() + assert spans, f"{module_path}: expected at least one logged span" + origin = spans[0]["context"]["span_origin"] + assert origin["instrumentation"]["name"] == expected, ( + f"{module_path} emitted {origin['instrumentation']['name']!r}, expected {expected!r}" + ) + + +def test_temporal_plugin_declares_instrumentation() -> None: + """Temporal passes `instrumentation="temporal-auto"` at every `logger.start_span(...)` site.""" + from pathlib import Path + + src = Path(__file__).parent.joinpath("temporal", "plugin.py").read_text() + # Every logger.start_span( occurrence should be followed nearby by + # instrumentation="temporal-auto". + logger_calls = src.count("logger.start_span(") + stamped = src.count('instrumentation="temporal-auto"') + assert stamped == logger_calls, ( + f"temporal/plugin.py has {logger_calls} logger.start_span() sites but only " + f"{stamped} carry instrumentation=\"temporal-auto\"" + ) + + +def test_openai_agents_stamps_instrumentation_on_method_start_span() -> None: + """openai_agents opens spans via `parent.start_span(...)` and `logger.start_span(...)`; + every such site must pass `instrumentation=_INSTRUMENTATION` explicitly.""" + from pathlib import Path + + src = Path(__file__).parent.joinpath("openai_agents", "tracing.py").read_text() + # Both `current_context.start_span(...)` and `self._logger.start_span(...)` + # and `parent.start_span(...)` should include instrumentation=_INSTRUMENTATION. + method_calls = ( + src.count("current_context.start_span(") + + src.count("self._logger.start_span(") + + src.count("parent.start_span(") + ) + stamped = src.count("instrumentation=_INSTRUMENTATION") + assert stamped == method_calls, ( + f"openai_agents/tracing.py has {method_calls} span-method call sites but only " + f"{stamped} carry instrumentation=_INSTRUMENTATION" + ) diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index f560874b..c95b1bc1 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -217,6 +217,7 @@ def start_span( start_time: float | None = None, set_current: bool | None = None, parent: str | dict | None = None, + instrumentation: str | None = None, **event: Any, ) -> "Span": """Create a new span. This is useful if you want to log more detailed trace information beyond the scope of a single log event. Data logged over several calls to `Span.log` will be merged into one logical row. @@ -230,6 +231,7 @@ def start_span( :param start_time: Optional start time of the span, as a timestamp in seconds. :param set_current: If true (the default), the span will be marked as the currently-active span for the duration of the context manager. :param parent: Optional parent info string for the span. The string can be generated from `[Span,Experiment,Logger].export`. If not provided, the current span will be used (depending on context). This is useful for adding spans to an existing trace. + :param instrumentation: Optional identifier for the instrumentation code creating this span. Used by SDK integrations to stamp `context.span_origin.instrumentation.name` (e.g. `"openai-auto"`). Leave unset for manual tracing. :param **event: Data to be logged. See `Experiment.log` for full details. :returns: The newly-created `Span` """ @@ -360,6 +362,7 @@ def start_span( start_time: float | None = None, set_current: bool | None = None, parent: str | dict | None = None, + instrumentation: str | None = None, **event: Any, ): return self @@ -2918,6 +2921,7 @@ def start_span( parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, state: BraintrustState | None = None, + instrumentation: str | None = None, **event: Any, ) -> Span: """Lower-level alternative to `@traced` for starting a span at the toplevel. It creates a span under the first active object (using the same precedence order as `@traced`), or if `parent` is specified, under the specified parent row, or returns a no-op span object. @@ -2957,6 +2961,7 @@ def start_span( event=event, state=state, lookup_span_parent=False, + instrumentation=instrumentation, ) else: return parent_obj.start_span( @@ -2967,6 +2972,7 @@ def start_span( set_current=set_current, parent=parent, propagated_event=propagated_event, + instrumentation=instrumentation, **event, ) @@ -4262,6 +4268,7 @@ def start_span( set_current: bool | None = None, parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, + instrumentation: str | None = None, **event: Any, ) -> Span: """Create a new toplevel span underneath the experiment. The name defaults to "root" and the span type to "eval". @@ -4277,6 +4284,7 @@ def start_span( set_current=set_current, parent=parent, propagated_event=propagated_event, + instrumentation=instrumentation, **event, ) @@ -4416,6 +4424,7 @@ def _start_span_impl( parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, lookup_span_parent: bool = True, + instrumentation: str | None = None, **event: Any, ) -> Span: parent_args = _start_span_parent_args( @@ -4437,6 +4446,7 @@ def _start_span_impl( set_current=set_current, event=event, state=self.state, + instrumentation=instrumentation, ) def __enter__(self) -> "Experiment": @@ -4525,6 +4535,7 @@ def __init__( root_span_id: str | None = None, state: BraintrustState | None = None, lookup_span_parent: bool = True, + instrumentation: str | None = None, ): if span_attributes is None: span_attributes = SpanAttributes() @@ -4594,7 +4605,9 @@ def __init__( caller_location or {}, "braintrust-python-logger", self.state.span_origin_environment, + override_instrumentation_name=instrumentation, ) + self._instrumentation = instrumentation # TODO: can be simplified after `event` is typed. id = event.pop("id", None) @@ -4719,6 +4732,7 @@ def start_span( set_current: bool | None = None, parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, + instrumentation: str | None = None, **event: Any, ) -> Span: if parent: @@ -4749,6 +4763,7 @@ def start_span( event=event, lookup_span_parent=lookup_span_parent, state=self.state, + instrumentation=instrumentation, ) def end(self, end_time: float | None = None) -> float: @@ -5738,6 +5753,7 @@ def start_span( propagated_event: dict[str, Any] | None = None, span_id: str | None = None, root_span_id: str | None = None, + instrumentation: str | None = None, **event: Any, ) -> Span: """Create a new toplevel span underneath the logger. The name defaults to "root" and the span type to "task". @@ -5755,6 +5771,7 @@ def start_span( propagated_event=propagated_event, span_id=span_id, root_span_id=root_span_id, + instrumentation=instrumentation, **event, ) @@ -5789,6 +5806,7 @@ def _start_span_impl( span_id: str | None = None, root_span_id: str | None = None, lookup_span_parent: bool = True, + instrumentation: str | None = None, **event: Any, ) -> Span: parent_args = _start_span_parent_args( @@ -5812,6 +5830,7 @@ def _start_span_impl( root_span_id=root_span_id, lookup_span_parent=lookup_span_parent, state=self.state, + instrumentation=instrumentation, ) def export(self) -> str: diff --git a/py/src/braintrust/span_origin.py b/py/src/braintrust/span_origin.py index fa0237ad..22973c91 100644 --- a/py/src/braintrust/span_origin.py +++ b/py/src/braintrust/span_origin.py @@ -84,13 +84,15 @@ def merge_span_origin_context( context: dict[str, Any] | None, instrumentation_name: str, environment: SpanOriginEnvironment | None, + override_instrumentation_name: str | None = None, ) -> dict[str, Any]: merged = dict(context or {}) span_origin = dict(merged.get("span_origin") or {}) + resolved_instrumentation = override_instrumentation_name or instrumentation_name span_origin = { "name": "braintrust.sdk.python", "version": _sdk_version(), - "instrumentation": {"name": instrumentation_name}, + "instrumentation": {"name": resolved_instrumentation}, **({"environment": environment} if environment else {}), **span_origin, } diff --git a/py/src/braintrust/test_otel.py b/py/src/braintrust/test_otel.py index 01faf0a6..b6b82caf 100644 --- a/py/src/braintrust/test_otel.py +++ b/py/src/braintrust/test_otel.py @@ -146,6 +146,56 @@ def test_braintrust_span_processor_merges_span_origin_with_context_json_set_afte provider.shutdown() +def test_merge_span_origin_context_default_instrumentation(): + from braintrust.span_origin import merge_span_origin_context + + merged = merge_span_origin_context({}, "braintrust-python-logger", None) + assert merged["span_origin"]["instrumentation"] == {"name": "braintrust-python-logger"} + + +def test_merge_span_origin_context_override_instrumentation(): + from braintrust.span_origin import merge_span_origin_context + + merged = merge_span_origin_context( + {}, + "braintrust-python-logger", + None, + override_instrumentation_name="openai-auto", + ) + assert merged["span_origin"]["instrumentation"] == {"name": "openai-auto"} + + +def test_merge_span_origin_context_empty_override_falls_back_to_default(): + from braintrust.span_origin import merge_span_origin_context + + for value in (None, ""): + merged = merge_span_origin_context( + {}, + "braintrust-python-logger", + None, + override_instrumentation_name=value, + ) + assert merged["span_origin"]["instrumentation"] == {"name": "braintrust-python-logger"} + + +def test_span_impl_records_instrumentation_kwarg(monkeypatch): + monkeypatch.setenv("BRAINTRUST_API_KEY", "test-key") + monkeypatch.delenv("BRAINTRUST_ENVIRONMENT_TYPE", raising=False) + monkeypatch.delenv("BRAINTRUST_ENVIRONMENT_NAME", raising=False) + + from braintrust.logger import init_logger, start_span + + init_logger(project="test_instrumentation_kwarg") + + with start_span(name="parent", instrumentation="openai-auto") as parent: + with start_span(name="child") as child: + pass + + assert parent._instrumentation == "openai-auto" + # Child spans do NOT inherit the parent's instrumentation name. + assert child._instrumentation is None + + def test_detect_environment_classifies_aws_ecs_before_lambda(monkeypatch): from braintrust.span_origin import detect_environment From d09b84aa3b0b3594d03eb5c49ef46e996c7e3c8f Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 14:14:42 +0000 Subject: [PATCH 2/9] chore: fix ruff format/check violations from shadow-migration --- py/src/braintrust/integrations/adk/tracing.py | 2 ++ .../braintrust/integrations/agentscope/tracing.py | 2 ++ py/src/braintrust/integrations/agno/tracing.py | 2 ++ py/src/braintrust/integrations/anthropic/tracing.py | 5 ++++- py/src/braintrust/integrations/autogen/tracing.py | 2 ++ .../integrations/bedrock_runtime/tracing.py | 2 ++ .../integrations/claude_agent_sdk/tracing.py | 2 ++ py/src/braintrust/integrations/cohere/tracing.py | 2 ++ py/src/braintrust/integrations/crewai/tracing.py | 5 ++++- py/src/braintrust/integrations/dspy/tracing.py | 5 ++++- .../braintrust/integrations/google_genai/tracing.py | 13 ++++++++++++- .../integrations/huggingface_hub/tracing.py | 2 ++ .../braintrust/integrations/instructor/tracing.py | 2 ++ .../braintrust/integrations/langchain/callbacks.py | 5 ++++- py/src/braintrust/integrations/litellm/tracing.py | 5 ++++- .../integrations/livekit_agents/tracing.py | 2 ++ .../braintrust/integrations/llamaindex/tracing.py | 5 ++++- py/src/braintrust/integrations/mistral/tracing.py | 2 ++ py/src/braintrust/integrations/openai/tracing.py | 5 ++++- .../integrations/openai_agents/tracing.py | 5 ++++- .../braintrust/integrations/openrouter/tracing.py | 2 ++ .../braintrust/integrations/pydantic_ai/tracing.py | 5 ++++- py/src/braintrust/integrations/test_span_origin.py | 4 ++-- 23 files changed, 74 insertions(+), 12 deletions(-) diff --git a/py/src/braintrust/integrations/adk/tracing.py b/py/src/braintrust/integrations/adk/tracing.py index 7d74e426..126706e9 100644 --- a/py/src/braintrust/integrations/adk/tracing.py +++ b/py/src/braintrust/integrations/adk/tracing.py @@ -14,6 +14,7 @@ from braintrust.integrations.utils import _materialize_attachment from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "adk-auto" # instrumentation shadow: do not edit @@ -21,6 +22,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/agentscope/tracing.py b/py/src/braintrust/integrations/agentscope/tracing.py index f8f9e742..11e49cac 100644 --- a/py/src/braintrust/integrations/agentscope/tracing.py +++ b/py/src/braintrust/integrations/agentscope/tracing.py @@ -7,6 +7,7 @@ from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "agentscope-auto" # instrumentation shadow: do not edit @@ -14,6 +15,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py index 8761f581..bbbd2e68 100644 --- a/py/src/braintrust/integrations/agno/tracing.py +++ b/py/src/braintrust/integrations/agno/tracing.py @@ -5,6 +5,7 @@ from braintrust.integrations.utils import _try_to_dict from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "agno-auto" # instrumentation shadow: do not edit @@ -12,6 +13,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index 388968c0..3beca34b 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -6,7 +6,9 @@ from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.anthropic._utils import Wrapper, _try_to_dict, extract_anthropic_usage from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import log_exc_info_to_span, start_span as _bt_start_span +from braintrust.logger import log_exc_info_to_span +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "anthropic-auto" # instrumentation shadow: do not edit @@ -15,6 +17,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/autogen/tracing.py b/py/src/braintrust/integrations/autogen/tracing.py index 586003c3..9206d528 100644 --- a/py/src/braintrust/integrations/autogen/tracing.py +++ b/py/src/braintrust/integrations/autogen/tracing.py @@ -6,6 +6,7 @@ from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "autogen-auto" # instrumentation shadow: do not edit @@ -13,6 +14,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/bedrock_runtime/tracing.py b/py/src/braintrust/integrations/bedrock_runtime/tracing.py index 151f6caf..a9b0c5a7 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/tracing.py +++ b/py/src/braintrust/integrations/bedrock_runtime/tracing.py @@ -13,6 +13,7 @@ ) from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "bedrock-runtime-auto" # instrumentation shadow: do not edit @@ -20,6 +21,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py index b9a79d50..737cde56 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py @@ -25,6 +25,7 @@ ) from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "claude-agent-sdk-auto" # instrumentation shadow: do not edit @@ -32,6 +33,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/cohere/tracing.py b/py/src/braintrust/integrations/cohere/tracing.py index 499b530c..fe6570a6 100644 --- a/py/src/braintrust/integrations/cohere/tracing.py +++ b/py/src/braintrust/integrations/cohere/tracing.py @@ -23,6 +23,7 @@ ) from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "cohere-auto" # instrumentation shadow: do not edit @@ -30,6 +31,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/crewai/tracing.py b/py/src/braintrust/integrations/crewai/tracing.py index b261160c..f09513fb 100644 --- a/py/src/braintrust/integrations/crewai/tracing.py +++ b/py/src/braintrust/integrations/crewai/tracing.py @@ -47,7 +47,9 @@ _parse_openai_usage_metrics, _try_to_dict, ) -from braintrust.logger import NOOP_SPAN, Span, current_span, start_span as _bt_start_span +from braintrust.logger import NOOP_SPAN, Span, current_span +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "crewai-auto" # instrumentation shadow: do not edit @@ -56,6 +58,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index 7a5de914..eaf4d0db 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -2,7 +2,9 @@ from typing import Any -from braintrust.logger import current_span, start_span as _bt_start_span +from braintrust.logger import current_span +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "dspy-auto" # instrumentation shadow: do not edit @@ -11,6 +13,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/google_genai/tracing.py b/py/src/braintrust/integrations/google_genai/tracing.py index 4e02adb0..e99fc7df 100644 --- a/py/src/braintrust/integrations/google_genai/tracing.py +++ b/py/src/braintrust/integrations/google_genai/tracing.py @@ -10,7 +10,17 @@ from braintrust.bt_json import bt_safe_deep_copy from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import NOOP_SPAN, _state, current_logger, current_span, parent_context, start_span as _bt_start_span +from braintrust.logger import ( + NOOP_SPAN, + _state, + current_logger, + current_span, + parent_context, +) +from braintrust.logger import ( + start_span as _bt_start_span, +) + _INSTRUMENTATION = "google-genai-auto" # instrumentation shadow: do not edit @@ -19,6 +29,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/huggingface_hub/tracing.py b/py/src/braintrust/integrations/huggingface_hub/tracing.py index 435a5e93..70e5ae85 100644 --- a/py/src/braintrust/integrations/huggingface_hub/tracing.py +++ b/py/src/braintrust/integrations/huggingface_hub/tracing.py @@ -36,6 +36,7 @@ ) from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "huggingface-hub-auto" # instrumentation shadow: do not edit @@ -43,6 +44,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import is_numeric diff --git a/py/src/braintrust/integrations/instructor/tracing.py b/py/src/braintrust/integrations/instructor/tracing.py index e2f2bace..cb30640b 100644 --- a/py/src/braintrust/integrations/instructor/tracing.py +++ b/py/src/braintrust/integrations/instructor/tracing.py @@ -30,6 +30,7 @@ from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "instructor-auto" # instrumentation shadow: do not edit @@ -37,6 +38,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/langchain/callbacks.py b/py/src/braintrust/integrations/langchain/callbacks.py index 376da3c9..8830dd13 100644 --- a/py/src/braintrust/integrations/langchain/callbacks.py +++ b/py/src/braintrust/integrations/langchain/callbacks.py @@ -11,7 +11,9 @@ from uuid import UUID from braintrust.generated_types import SpanAttributes -from braintrust.logger import NOOP_SPAN, Logger, Span, current_span, init_logger, start_span as _bt_start_span +from braintrust.logger import NOOP_SPAN, Logger, Span, current_span, init_logger +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "langchain-auto" # instrumentation shadow: do not edit @@ -20,6 +22,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.version import VERSION as sdk_version from langchain_core.agents import AgentAction, AgentFinish diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index 8a9352c3..16c63517 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -14,7 +14,9 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import Span, start_span as _bt_start_span +from braintrust.logger import Span +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "litellm-auto" # instrumentation shadow: do not edit @@ -23,6 +25,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones, is_numeric, merge_dicts diff --git a/py/src/braintrust/integrations/livekit_agents/tracing.py b/py/src/braintrust/integrations/livekit_agents/tracing.py index def4fabb..bbfbab71 100644 --- a/py/src/braintrust/integrations/livekit_agents/tracing.py +++ b/py/src/braintrust/integrations/livekit_agents/tracing.py @@ -16,6 +16,8 @@ _state, current_span, parent_context, +) +from braintrust.logger import ( start_span as _bt_start_span, ) diff --git a/py/src/braintrust/integrations/llamaindex/tracing.py b/py/src/braintrust/integrations/llamaindex/tracing.py index 151594d2..583cddf4 100644 --- a/py/src/braintrust/integrations/llamaindex/tracing.py +++ b/py/src/braintrust/integrations/llamaindex/tracing.py @@ -4,7 +4,9 @@ import time from typing import Any -from braintrust.logger import NOOP_SPAN, Span, current_span, start_span as _bt_start_span +from braintrust.logger import NOOP_SPAN, Span, current_span +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "llamaindex-auto" # instrumentation shadow: do not edit @@ -13,6 +15,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/mistral/tracing.py b/py/src/braintrust/integrations/mistral/tracing.py index f57d9987..8ca0f1b5 100644 --- a/py/src/braintrust/integrations/mistral/tracing.py +++ b/py/src/braintrust/integrations/mistral/tracing.py @@ -18,6 +18,7 @@ ) from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "mistral-auto" # instrumentation shadow: do not edit @@ -25,6 +26,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index ead44b30..653e4764 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -19,7 +19,9 @@ _timing_metrics, _try_to_dict, ) -from braintrust.logger import Span, start_span as _bt_start_span +from braintrust.logger import Span +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "openai-auto" # instrumentation shadow: do not edit @@ -28,6 +30,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from braintrust.util import clean_nones, merge_dicts from wrapt import FunctionWrapper diff --git a/py/src/braintrust/integrations/openai_agents/tracing.py b/py/src/braintrust/integrations/openai_agents/tracing.py index 1641daa7..a04291e8 100644 --- a/py/src/braintrust/integrations/openai_agents/tracing.py +++ b/py/src/braintrust/integrations/openai_agents/tracing.py @@ -4,7 +4,9 @@ from typing import Any from agents import tracing -from braintrust.logger import NOOP_SPAN, Experiment, Logger, Span, current_span, flush, start_span as _bt_start_span +from braintrust.logger import NOOP_SPAN, Experiment, Logger, Span, current_span, flush +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "openai-agents-auto" # instrumentation shadow: do not edit @@ -13,6 +15,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/openrouter/tracing.py b/py/src/braintrust/integrations/openrouter/tracing.py index 8474b905..90a4203f 100644 --- a/py/src/braintrust/integrations/openrouter/tracing.py +++ b/py/src/braintrust/integrations/openrouter/tracing.py @@ -15,6 +15,7 @@ ) from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "openrouter-auto" # instrumentation shadow: do not edit @@ -22,6 +23,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index 0c979b2b..34d02afc 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -8,7 +8,9 @@ from typing import Any from braintrust.integrations.utils import _materialize_attachment -from braintrust.logger import _internal_get_global_state, start_span as _bt_start_span +from braintrust.logger import _internal_get_global_state +from braintrust.logger import start_span as _bt_start_span + _INSTRUMENTATION = "pydantic-ai-auto" # instrumentation shadow: do not edit @@ -17,6 +19,7 @@ def start_span(*args, **kwargs): kwargs.setdefault("instrumentation", _INSTRUMENTATION) return _bt_start_span(*args, **kwargs) + from braintrust.span_types import SpanTypeAttribute from wrapt import wrap_function_wrapper diff --git a/py/src/braintrust/integrations/test_span_origin.py b/py/src/braintrust/integrations/test_span_origin.py index d477c016..7d07a357 100644 --- a/py/src/braintrust/integrations/test_span_origin.py +++ b/py/src/braintrust/integrations/test_span_origin.py @@ -14,10 +14,10 @@ from typing import Any import pytest - from braintrust import logger from braintrust.test_helpers import init_test_logger + # Each entry: (module path, expected `-auto` name). _SHADOW_TARGETS: list[tuple[str, str]] = [ ("braintrust.integrations.adk.tracing", "adk-auto"), @@ -107,7 +107,7 @@ def test_temporal_plugin_declares_instrumentation() -> None: stamped = src.count('instrumentation="temporal-auto"') assert stamped == logger_calls, ( f"temporal/plugin.py has {logger_calls} logger.start_span() sites but only " - f"{stamped} carry instrumentation=\"temporal-auto\"" + f'{stamped} carry instrumentation="temporal-auto"' ) From f2f2d2df17b5406b7e194a764b70104119c121e2 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 14:20:40 +0000 Subject: [PATCH 3/9] chore: drop migration-only shadow marker comment --- py/src/braintrust/integrations/adk/tracing.py | 2 +- py/src/braintrust/integrations/agentscope/tracing.py | 2 +- py/src/braintrust/integrations/agno/tracing.py | 2 +- py/src/braintrust/integrations/anthropic/tracing.py | 2 +- py/src/braintrust/integrations/autogen/tracing.py | 2 +- py/src/braintrust/integrations/bedrock_runtime/tracing.py | 2 +- py/src/braintrust/integrations/claude_agent_sdk/tracing.py | 2 +- py/src/braintrust/integrations/cohere/tracing.py | 2 +- py/src/braintrust/integrations/crewai/tracing.py | 2 +- py/src/braintrust/integrations/dspy/tracing.py | 2 +- py/src/braintrust/integrations/google_genai/tracing.py | 2 +- py/src/braintrust/integrations/huggingface_hub/tracing.py | 2 +- py/src/braintrust/integrations/instructor/tracing.py | 2 +- py/src/braintrust/integrations/langchain/callbacks.py | 2 +- py/src/braintrust/integrations/litellm/tracing.py | 2 +- py/src/braintrust/integrations/livekit_agents/tracing.py | 2 +- py/src/braintrust/integrations/llamaindex/tracing.py | 2 +- py/src/braintrust/integrations/mistral/tracing.py | 2 +- py/src/braintrust/integrations/openai/tracing.py | 2 +- py/src/braintrust/integrations/openai_agents/tracing.py | 2 +- py/src/braintrust/integrations/openrouter/tracing.py | 2 +- py/src/braintrust/integrations/pydantic_ai/tracing.py | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/py/src/braintrust/integrations/adk/tracing.py b/py/src/braintrust/integrations/adk/tracing.py index 126706e9..029d22cb 100644 --- a/py/src/braintrust/integrations/adk/tracing.py +++ b/py/src/braintrust/integrations/adk/tracing.py @@ -15,7 +15,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "adk-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "adk-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/agentscope/tracing.py b/py/src/braintrust/integrations/agentscope/tracing.py index 11e49cac..be82d07b 100644 --- a/py/src/braintrust/integrations/agentscope/tracing.py +++ b/py/src/braintrust/integrations/agentscope/tracing.py @@ -8,7 +8,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "agentscope-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "agentscope-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py index bbbd2e68..f91bd09b 100644 --- a/py/src/braintrust/integrations/agno/tracing.py +++ b/py/src/braintrust/integrations/agno/tracing.py @@ -6,7 +6,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "agno-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "agno-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index 3beca34b..2d4020d0 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -10,7 +10,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "anthropic-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "anthropic-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/autogen/tracing.py b/py/src/braintrust/integrations/autogen/tracing.py index 9206d528..0de63b19 100644 --- a/py/src/braintrust/integrations/autogen/tracing.py +++ b/py/src/braintrust/integrations/autogen/tracing.py @@ -7,7 +7,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "autogen-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "autogen-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/bedrock_runtime/tracing.py b/py/src/braintrust/integrations/bedrock_runtime/tracing.py index a9b0c5a7..d7d2802f 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/tracing.py +++ b/py/src/braintrust/integrations/bedrock_runtime/tracing.py @@ -14,7 +14,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "bedrock-runtime-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "bedrock-runtime-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py index 737cde56..93168cea 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py @@ -26,7 +26,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "claude-agent-sdk-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "claude-agent-sdk-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/cohere/tracing.py b/py/src/braintrust/integrations/cohere/tracing.py index fe6570a6..805cafd3 100644 --- a/py/src/braintrust/integrations/cohere/tracing.py +++ b/py/src/braintrust/integrations/cohere/tracing.py @@ -24,7 +24,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "cohere-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "cohere-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/crewai/tracing.py b/py/src/braintrust/integrations/crewai/tracing.py index f09513fb..ed1b3bf7 100644 --- a/py/src/braintrust/integrations/crewai/tracing.py +++ b/py/src/braintrust/integrations/crewai/tracing.py @@ -51,7 +51,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "crewai-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "crewai-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index eaf4d0db..aaf4c696 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -6,7 +6,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "dspy-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "dspy-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/google_genai/tracing.py b/py/src/braintrust/integrations/google_genai/tracing.py index e99fc7df..68dc475d 100644 --- a/py/src/braintrust/integrations/google_genai/tracing.py +++ b/py/src/braintrust/integrations/google_genai/tracing.py @@ -22,7 +22,7 @@ ) -_INSTRUMENTATION = "google-genai-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "google-genai-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/huggingface_hub/tracing.py b/py/src/braintrust/integrations/huggingface_hub/tracing.py index 70e5ae85..c8b1a0c3 100644 --- a/py/src/braintrust/integrations/huggingface_hub/tracing.py +++ b/py/src/braintrust/integrations/huggingface_hub/tracing.py @@ -37,7 +37,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "huggingface-hub-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "huggingface-hub-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/instructor/tracing.py b/py/src/braintrust/integrations/instructor/tracing.py index cb30640b..e4a4a361 100644 --- a/py/src/braintrust/integrations/instructor/tracing.py +++ b/py/src/braintrust/integrations/instructor/tracing.py @@ -31,7 +31,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "instructor-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "instructor-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/langchain/callbacks.py b/py/src/braintrust/integrations/langchain/callbacks.py index 8830dd13..c09f504f 100644 --- a/py/src/braintrust/integrations/langchain/callbacks.py +++ b/py/src/braintrust/integrations/langchain/callbacks.py @@ -15,7 +15,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "langchain-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "langchain-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index 16c63517..b5729512 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -18,7 +18,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "litellm-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "litellm-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/livekit_agents/tracing.py b/py/src/braintrust/integrations/livekit_agents/tracing.py index bbfbab71..b77c2d38 100644 --- a/py/src/braintrust/integrations/livekit_agents/tracing.py +++ b/py/src/braintrust/integrations/livekit_agents/tracing.py @@ -22,7 +22,7 @@ ) -_INSTRUMENTATION = "livekit-agents-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "livekit-agents-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/llamaindex/tracing.py b/py/src/braintrust/integrations/llamaindex/tracing.py index 583cddf4..dce6e383 100644 --- a/py/src/braintrust/integrations/llamaindex/tracing.py +++ b/py/src/braintrust/integrations/llamaindex/tracing.py @@ -8,7 +8,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "llamaindex-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "llamaindex-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/mistral/tracing.py b/py/src/braintrust/integrations/mistral/tracing.py index 8ca0f1b5..f8bffe1b 100644 --- a/py/src/braintrust/integrations/mistral/tracing.py +++ b/py/src/braintrust/integrations/mistral/tracing.py @@ -19,7 +19,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "mistral-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "mistral-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 653e4764..6d897ddc 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -23,7 +23,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "openai-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "openai-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/openai_agents/tracing.py b/py/src/braintrust/integrations/openai_agents/tracing.py index a04291e8..dcdba550 100644 --- a/py/src/braintrust/integrations/openai_agents/tracing.py +++ b/py/src/braintrust/integrations/openai_agents/tracing.py @@ -8,7 +8,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "openai-agents-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "openai-agents-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/openrouter/tracing.py b/py/src/braintrust/integrations/openrouter/tracing.py index 90a4203f..95565ed7 100644 --- a/py/src/braintrust/integrations/openrouter/tracing.py +++ b/py/src/braintrust/integrations/openrouter/tracing.py @@ -16,7 +16,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "openrouter-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "openrouter-auto" def start_span(*args, **kwargs): diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index 34d02afc..60b755b9 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -12,7 +12,7 @@ from braintrust.logger import start_span as _bt_start_span -_INSTRUMENTATION = "pydantic-ai-auto" # instrumentation shadow: do not edit +_INSTRUMENTATION = "pydantic-ai-auto" def start_span(*args, **kwargs): From 9643ca5e8ee5558516edbc727b0671f279e081aa Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 14:29:39 +0000 Subject: [PATCH 4/9] refactor: inline instrumentation.name assertions into per-integration VCR tests Two related cleanups: 1. Simplify merge_span_origin_context API. Drop the override_instrumentation_name kwarg and just pass the single resolved name in through the existing instrumentation_name parameter. SpanImpl.__init__ resolves the fallback with `or`: self._instrumentation = instrumentation or "braintrust-python-logger" Direct callers no longer have to know about two parameters. 2. Replace the consolidated integrations/test_span_origin.py file with one inline assertion in each integration's existing VCR test. The consolidated file re-tested the same shadow shape 22 times without ever exercising a real provider payload; the per-integration assertions live alongside the real provider-facing coverage and fail on the specific integration if the shadow ever regresses. Adds `assert span["context"]["span_origin"]["instrumentation"]["name"] == "-auto"` to one existing VCR test per integration (adk, agentscope, agno, anthropic, autogen, bedrock_runtime, claude_agent_sdk, cohere, crewai, dspy, google_genai, huggingface_hub, instructor, langchain, litellm, livekit_agents, llamaindex, mistral, openai, openai_agents, openrouter, pydantic_ai, temporal). --- .../braintrust/integrations/adk/test_adk.py | 2 + .../agentscope/test_agentscope.py | 1 + .../braintrust/integrations/agno/test_agno.py | 1 + .../integrations/anthropic/test_anthropic.py | 1 + .../integrations/autogen/test_autogen.py | 1 + .../bedrock_runtime/test_bedrock_runtime.py | 1 + .../claude_agent_sdk/test_claude_agent_sdk.py | 1 + .../integrations/cohere/test_cohere.py | 1 + .../integrations/crewai/test_crewai.py | 1 + .../braintrust/integrations/dspy/test_dspy.py | 1 + .../google_genai/test_google_genai.py | 1 + .../huggingface_hub/test_huggingface_hub.py | 1 + .../instructor/test_instructor.py | 1 + .../integrations/langchain/test_callbacks.py | 2 + .../integrations/litellm/test_litellm.py | 1 + .../livekit_agents/test_livekit_agents.py | 1 + .../llamaindex/test_llamaindex.py | 1 + .../integrations/mistral/test_mistral.py | 1 + .../integrations/openai/test_openai.py | 1 + .../openai_agents/test_openai_agents.py | 1 + .../openrouter/test_openrouter.py | 1 + .../test_pydantic_ai_integration.py | 1 + .../integrations/temporal/test_temporal.py | 4 + .../integrations/test_span_origin.py | 131 ------------------ py/src/braintrust/logger.py | 5 +- py/src/braintrust/span_origin.py | 4 +- py/src/braintrust/test_otel.py | 34 +---- 27 files changed, 36 insertions(+), 166 deletions(-) delete mode 100644 py/src/braintrust/integrations/test_span_origin.py diff --git a/py/src/braintrust/integrations/adk/test_adk.py b/py/src/braintrust/integrations/adk/test_adk.py index afaad199..462f2eba 100644 --- a/py/src/braintrust/integrations/adk/test_adk.py +++ b/py/src/braintrust/integrations/adk/test_adk.py @@ -198,6 +198,8 @@ async def run_message(text: str) -> str: invocation_spans = [row for row in spans if row["span_attributes"]["name"] == f"invocation [{app_name}]"] assert len(invocation_spans) == 2 + for span in invocation_spans: + assert span["context"]["span_origin"]["instrumentation"]["name"] == "adk-auto" assert {span["metadata"]["session_id"] for span in invocation_spans} == {session_id} assert {span["input"]["new_message"]["parts"][0]["text"] for span in invocation_spans} == { "Hi, my name is Alice.", diff --git a/py/src/braintrust/integrations/agentscope/test_agentscope.py b/py/src/braintrust/integrations/agentscope/test_agentscope.py index 455a9ab4..372d675d 100644 --- a/py/src/braintrust/integrations/agentscope/test_agentscope.py +++ b/py/src/braintrust/integrations/agentscope/test_agentscope.py @@ -119,6 +119,7 @@ async def test_agentscope_simple_agent_run(memory_logger): agent_span = next(span for span in spans if span["span_attributes"]["name"] == "Friday.reply") llm_spans = [span for span in spans if _span_type(span) == SpanTypeAttribute.LLM] + assert agent_span["context"]["span_origin"]["instrumentation"]["name"] == "agentscope-auto" assert _span_type(agent_span) == "task" assert llm_spans assert llm_spans[0]["metadata"]["model"] == "gpt-4o-mini" diff --git a/py/src/braintrust/integrations/agno/test_agno.py b/py/src/braintrust/integrations/agno/test_agno.py index 59549747..0d17262d 100644 --- a/py/src/braintrust/integrations/agno/test_agno.py +++ b/py/src/braintrust/integrations/agno/test_agno.py @@ -62,6 +62,7 @@ def test_agno_simple_agent_execution(memory_logger): assert len(spans) == 2, f"Expected 2 spans, got {len(spans)}" root_span = spans[0] + assert root_span["context"]["span_origin"]["instrumentation"]["name"] == "agno-auto" assert root_span["span_attributes"]["name"] == "Author Agent.run" assert root_span["span_attributes"]["type"].value == "task" root_input = root_span["input"] diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index a2b7fca9..08d5c20b 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -482,6 +482,7 @@ def test_anthropic_messages_create_with_image_attachment_input(memory_logger): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "anthropic-auto" content = span["input"][0]["content"] image_block = content[1] diff --git a/py/src/braintrust/integrations/autogen/test_autogen.py b/py/src/braintrust/integrations/autogen/test_autogen.py index 355227ac..70630150 100644 --- a/py/src/braintrust/integrations/autogen/test_autogen.py +++ b/py/src/braintrust/integrations/autogen/test_autogen.py @@ -47,6 +47,7 @@ async def test_autogen_agent_run_creates_braintrust_spans(memory_logger): assert result.messages[-1].content spans = memory_logger.pop() agent_span = next(span for span in spans if span["span_attributes"]["name"] == "assistant.run") + assert agent_span["context"]["span_origin"]["instrumentation"]["name"] == "autogen-auto" assert _span_type(agent_span) == "task" assert agent_span["input"]["task"] == "Say hello in exactly two words." assert agent_span["metadata"]["component"] == "agent" diff --git a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py index 73e19249..583b9f86 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py +++ b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py @@ -222,6 +222,7 @@ def test_wrap_bedrock_converse_stream(memory_logger): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "bedrock-runtime-auto" _assert_converse_span(span, name="bedrock.converse-stream", start=start, end=end, model_id=CONVERSE_STREAM_MODEL) assert span["metadata"]["endpoint"] == "converse-stream" assert span["metadata"]["stream"] is True diff --git a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py index 1ff40395..258a6a46 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/test_claude_agent_sdk.py @@ -162,6 +162,7 @@ async def calculator_handler(args): assert len(task_spans) == 1, f"Should have exactly one task span, got {len(task_spans)}" task_span = task_spans[0] + assert task_span["context"]["span_origin"]["instrumentation"]["name"] == "claude-agent-sdk-auto" assert task_span["span_attributes"]["name"] == "Claude Agent" assert "15 multiplied by 7" in task_span["input"] assert task_span["output"] is not None diff --git a/py/src/braintrust/integrations/cohere/test_cohere.py b/py/src/braintrust/integrations/cohere/test_cohere.py index d131039b..0a99b639 100644 --- a/py/src/braintrust/integrations/cohere/test_cohere.py +++ b/py/src/braintrust/integrations/cohere/test_cohere.py @@ -247,6 +247,7 @@ def test_wrap_cohere_chat_v2_sync(memory_logger): assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "cohere-auto" assert span["span_attributes"]["name"] == "cohere.chat" assert span["span_attributes"]["type"] == "llm" assert span["metadata"]["provider"] == "cohere" diff --git a/py/src/braintrust/integrations/crewai/test_crewai.py b/py/src/braintrust/integrations/crewai/test_crewai.py index 4e15fb9a..86f4d4a7 100644 --- a/py/src/braintrust/integrations/crewai/test_crewai.py +++ b/py/src/braintrust/integrations/crewai/test_crewai.py @@ -262,6 +262,7 @@ def test_kickoff_llm_event_tree_parents_and_shape(memory_logger): kickoff_span = by_name["crewai.kickoff"][0] llm_span = by_name["crewai.llm"][0] + assert kickoff_span["context"]["span_origin"]["instrumentation"]["name"] == "crewai-auto" assert kickoff_span["span_attributes"]["type"] == "task" assert llm_span["span_attributes"]["type"] == "llm" diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index e9ccef9d..9fa1b52c 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -43,6 +43,7 @@ def test_dspy_callback(memory_logger): spans_by_name = {span["span_attributes"]["name"]: span for span in spans} lm_span = spans_by_name["dspy.lm"] + assert lm_span["context"]["span_origin"]["instrumentation"]["name"] == "dspy-auto" assert "metadata" in lm_span assert "model" in lm_span["metadata"] assert MODEL in lm_span["metadata"]["model"] diff --git a/py/src/braintrust/integrations/google_genai/test_google_genai.py b/py/src/braintrust/integrations/google_genai/test_google_genai.py index a0cf1b41..dc9f71ea 100644 --- a/py/src/braintrust/integrations/google_genai/test_google_genai.py +++ b/py/src/braintrust/integrations/google_genai/test_google_genai.py @@ -215,6 +215,7 @@ def test_basic_completion(memory_logger, mode): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "google-genai-auto" assert span["metadata"]["model"] == MODEL assert "What is the capital of France?" in str(span["input"]) assert span["output"] diff --git a/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py b/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py index 49f42252..b1ae4985 100644 --- a/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py +++ b/py/src/braintrust/integrations/huggingface_hub/test_huggingface_hub.py @@ -244,6 +244,7 @@ def test_wrap_huggingface_hub_chat_completion_sync(memory_logger): assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "huggingface-hub-auto" assert span["span_attributes"]["name"] == "huggingface.chat_completion" assert span["span_attributes"]["type"] == "llm" # With no parent span on the stack, the LLM span is its own root and has diff --git a/py/src/braintrust/integrations/instructor/test_instructor.py b/py/src/braintrust/integrations/instructor/test_instructor.py index 0450e3e6..4f024c44 100644 --- a/py/src/braintrust/integrations/instructor/test_instructor.py +++ b/py/src/braintrust/integrations/instructor/test_instructor.py @@ -121,6 +121,7 @@ def test_instructor_openai_single_success(self, setup_logger, memory_logger): assert len(llm_spans) == 1, f"Expected 1 llm span, got {len(llm_spans)}: names={_names(spans)}" parent = task_spans[0] + assert parent["context"]["span_origin"]["instrumentation"]["name"] == "instructor-auto" assert parent["span_attributes"]["name"] == "instructor.create" meta = parent.get("metadata", {}) assert meta.get("response_model") == "Person" diff --git a/py/src/braintrust/integrations/langchain/test_callbacks.py b/py/src/braintrust/integrations/langchain/test_callbacks.py index b5fa5e81..1543c1fd 100644 --- a/py/src/braintrust/integrations/langchain/test_callbacks.py +++ b/py/src/braintrust/integrations/langchain/test_callbacks.py @@ -61,6 +61,8 @@ def test_llm_calls(logger_memory_logger): spans = memory_logger.pop() assert len(spans) == 3 + for span in spans: + assert span["context"]["span_origin"]["instrumentation"]["name"] == "langchain-auto" # ``root_span_id`` is the root span's own span_id (the parent reference for # its children); ``trace_root_id`` is the trace shared by every span. diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 549d3e33..33e97ca5 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -68,6 +68,7 @@ def test_litellm_completion_metrics(memory_logger) -> None: assert len(spans) == 1 span = spans[0] assert span + assert span["context"]["span_origin"]["instrumentation"]["name"] == "litellm-auto" metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert span["metadata"]["model"] == TEST_MODEL diff --git a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py index afa48881..a27740fb 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py +++ b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py @@ -159,6 +159,7 @@ async def cancelled_start(): logs = memory_logger.pop() session_logs = _spans_named(logs, "livekit_agent_session") assert len(session_logs) == 1 + assert session_logs[0]["context"]["span_origin"]["instrumentation"]["name"] == "livekit-agents-auto" @pytest.mark.asyncio diff --git a/py/src/braintrust/integrations/llamaindex/test_llamaindex.py b/py/src/braintrust/integrations/llamaindex/test_llamaindex.py index 315a8ce0..2c27cfc6 100644 --- a/py/src/braintrust/integrations/llamaindex/test_llamaindex.py +++ b/py/src/braintrust/integrations/llamaindex/test_llamaindex.py @@ -114,6 +114,7 @@ def test_llm_complete(logger_memory_logger): assert len(llm_spans) >= 1 llm_span = llm_spans[0] + assert llm_span["context"]["span_origin"]["instrumentation"]["name"] == "llamaindex-auto" assert llm_span["span_attributes"]["name"] == "OpenAI" assert llm_span["input"] is not None assert llm_span["output"] is not None diff --git a/py/src/braintrust/integrations/mistral/test_mistral.py b/py/src/braintrust/integrations/mistral/test_mistral.py index 6833bbd5..e8a9f41f 100644 --- a/py/src/braintrust/integrations/mistral/test_mistral.py +++ b/py/src/braintrust/integrations/mistral/test_mistral.py @@ -246,6 +246,7 @@ def test_wrap_mistral_chat_complete_sync(memory_logger): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "mistral-auto" assert span["input"] == [{"role": "user", "content": "What is 2+2? Reply with just the number."}] assert span["metadata"]["provider"] == "mistral" assert span["metadata"]["model"] == CHAT_MODEL diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 84418d53..b95ce084 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -119,6 +119,7 @@ def test_openai_chat_metrics(memory_logger): assert len(spans) == 1 span = spans[0] assert span + assert span["context"]["span_origin"]["instrumentation"]["name"] == "openai-auto" metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert TEST_MODEL in span["metadata"]["model"] diff --git a/py/src/braintrust/integrations/openai_agents/test_openai_agents.py b/py/src/braintrust/integrations/openai_agents/test_openai_agents.py index 7a6db120..51f442cd 100644 --- a/py/src/braintrust/integrations/openai_agents/test_openai_agents.py +++ b/py/src/braintrust/integrations/openai_agents/test_openai_agents.py @@ -90,6 +90,7 @@ def export(self): spans = memory_logger.pop() root_span = spans[0] + assert root_span["context"]["span_origin"]["instrumentation"]["name"] == "openai-agents-auto" assert root_span["metadata"]["conversation_id"] == "test-12345" diff --git a/py/src/braintrust/integrations/openrouter/test_openrouter.py b/py/src/braintrust/integrations/openrouter/test_openrouter.py index 0839c230..fa1ec892 100644 --- a/py/src/braintrust/integrations/openrouter/test_openrouter.py +++ b/py/src/braintrust/integrations/openrouter/test_openrouter.py @@ -50,6 +50,7 @@ def test_wrap_openrouter_chat_send_sync(memory_logger): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "openrouter-auto" assert span["input"] == [{"role": "user", "content": "What is 2+2? Reply with just the number."}] assert span["metadata"]["provider"] == "openai" assert span["metadata"]["model"] == "gpt-4o-mini" diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index 60462e5c..814aeaf3 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -131,6 +131,7 @@ async def test_direct_model_request_creates_nested_chat_span_without_class_scan( assert direct_span is not None, "model_request span not found" assert chat_span is not None, "chat span not found" + assert direct_span["context"]["span_origin"]["instrumentation"]["name"] == "pydantic-ai-auto" assert chat_span["span_parents"] == [direct_span["span_id"]] assert chat_span["metadata"]["model"] == "gpt-4o-mini" assert chat_span["metadata"]["provider"] == "openai" diff --git a/py/src/braintrust/integrations/temporal/test_temporal.py b/py/src/braintrust/integrations/temporal/test_temporal.py index da2c4bba..c726e865 100644 --- a/py/src/braintrust/integrations/temporal/test_temporal.py +++ b/py/src/braintrust/integrations/temporal/test_temporal.py @@ -526,6 +526,10 @@ async def test_plugin_activity_after_replay_stays_under_workflow_span( assert len(workflow_spans) == 1 assert len(activity_spans) == 2 workflow_span = workflow_spans[0] + assert workflow_span["context"]["span_origin"]["instrumentation"]["name"] == "temporal-auto" + assert all( + span["context"]["span_origin"]["instrumentation"]["name"] == "temporal-auto" for span in activity_spans + ) assert all(workflow_span["span_id"] in span.get("span_parents", []) for span in activity_spans) assert all(workflow_span["root_span_id"] == span["root_span_id"] for span in activity_spans) if with_client_parent: diff --git a/py/src/braintrust/integrations/test_span_origin.py b/py/src/braintrust/integrations/test_span_origin.py deleted file mode 100644 index 7d07a357..00000000 --- a/py/src/braintrust/integrations/test_span_origin.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Cross-integration assertions for `context.span_origin.instrumentation.name`. - -Each integration's `tracing.py` (or `callbacks.py`/`plugin.py`) shadows the -module-level `start_span` to stamp the integration's identifier. This test -suite opens a span through each shadow and confirms the emitted context -carries the expected `-auto` name. Provider packages are NOT -imported here — only the local shadow — so the file runs cleanly under -`test_core`. -""" - -from __future__ import annotations - -import importlib -from typing import Any - -import pytest -from braintrust import logger -from braintrust.test_helpers import init_test_logger - - -# Each entry: (module path, expected `-auto` name). -_SHADOW_TARGETS: list[tuple[str, str]] = [ - ("braintrust.integrations.adk.tracing", "adk-auto"), - ("braintrust.integrations.agentscope.tracing", "agentscope-auto"), - ("braintrust.integrations.agno.tracing", "agno-auto"), - ("braintrust.integrations.anthropic.tracing", "anthropic-auto"), - ("braintrust.integrations.autogen.tracing", "autogen-auto"), - ("braintrust.integrations.bedrock_runtime.tracing", "bedrock-runtime-auto"), - ("braintrust.integrations.claude_agent_sdk.tracing", "claude-agent-sdk-auto"), - ("braintrust.integrations.cohere.tracing", "cohere-auto"), - ("braintrust.integrations.crewai.tracing", "crewai-auto"), - ("braintrust.integrations.dspy.tracing", "dspy-auto"), - ("braintrust.integrations.google_genai.tracing", "google-genai-auto"), - ("braintrust.integrations.huggingface_hub.tracing", "huggingface-hub-auto"), - ("braintrust.integrations.instructor.tracing", "instructor-auto"), - ("braintrust.integrations.langchain.callbacks", "langchain-auto"), - ("braintrust.integrations.litellm.tracing", "litellm-auto"), - ("braintrust.integrations.livekit_agents.tracing", "livekit-agents-auto"), - ("braintrust.integrations.llamaindex.tracing", "llamaindex-auto"), - ("braintrust.integrations.mistral.tracing", "mistral-auto"), - ("braintrust.integrations.openai.tracing", "openai-auto"), - ("braintrust.integrations.openai_agents.tracing", "openai-agents-auto"), - ("braintrust.integrations.openrouter.tracing", "openrouter-auto"), - ("braintrust.integrations.pydantic_ai.tracing", "pydantic-ai-auto"), -] - -_PROJECT = "test-span-origin" - - -@pytest.fixture -def memory_logger() -> Any: - init_test_logger(_PROJECT) - with logger._internal_with_memory_background_logger() as bgl: - yield bgl - logger._state.current_experiment = None - logger._state.reset_parent_state() - - -def _try_import(module_path: str): - try: - return importlib.import_module(module_path) - except Exception: - return None - - -@pytest.mark.parametrize(("module_path", "expected"), _SHADOW_TARGETS) -def test_integration_shadow_declares_instrumentation(module_path: str, expected: str) -> None: - """Every integration exposes `_INSTRUMENTATION = "-auto"` at module scope.""" - module = _try_import(module_path) - if module is None: - pytest.skip(f"{module_path}: optional provider dep not installed") - assert getattr(module, "_INSTRUMENTATION", None) == expected, ( - f"{module_path} should declare _INSTRUMENTATION = {expected!r}" - ) - - -@pytest.mark.parametrize(("module_path", "expected"), _SHADOW_TARGETS) -def test_integration_shadow_stamps_span_origin(module_path: str, expected: str, memory_logger) -> None: - """A span opened via the integration's local `start_span` carries the expected instrumentation name.""" - module = _try_import(module_path) - if module is None: - pytest.skip(f"{module_path}: optional provider dep not installed") - start_span = getattr(module, "start_span", None) - if start_span is None: - pytest.skip(f"{module_path}: no module-level start_span shadow") - - with start_span(name="span-origin-check") as span: - # SpanImpl exposes the resolved instrumentation name as `_instrumentation`. - assert getattr(span, "_instrumentation", None) == expected - - spans = memory_logger.pop() - assert spans, f"{module_path}: expected at least one logged span" - origin = spans[0]["context"]["span_origin"] - assert origin["instrumentation"]["name"] == expected, ( - f"{module_path} emitted {origin['instrumentation']['name']!r}, expected {expected!r}" - ) - - -def test_temporal_plugin_declares_instrumentation() -> None: - """Temporal passes `instrumentation="temporal-auto"` at every `logger.start_span(...)` site.""" - from pathlib import Path - - src = Path(__file__).parent.joinpath("temporal", "plugin.py").read_text() - # Every logger.start_span( occurrence should be followed nearby by - # instrumentation="temporal-auto". - logger_calls = src.count("logger.start_span(") - stamped = src.count('instrumentation="temporal-auto"') - assert stamped == logger_calls, ( - f"temporal/plugin.py has {logger_calls} logger.start_span() sites but only " - f'{stamped} carry instrumentation="temporal-auto"' - ) - - -def test_openai_agents_stamps_instrumentation_on_method_start_span() -> None: - """openai_agents opens spans via `parent.start_span(...)` and `logger.start_span(...)`; - every such site must pass `instrumentation=_INSTRUMENTATION` explicitly.""" - from pathlib import Path - - src = Path(__file__).parent.joinpath("openai_agents", "tracing.py").read_text() - # Both `current_context.start_span(...)` and `self._logger.start_span(...)` - # and `parent.start_span(...)` should include instrumentation=_INSTRUMENTATION. - method_calls = ( - src.count("current_context.start_span(") - + src.count("self._logger.start_span(") - + src.count("parent.start_span(") - ) - stamped = src.count("instrumentation=_INSTRUMENTATION") - assert stamped == method_calls, ( - f"openai_agents/tracing.py has {method_calls} span-method call sites but only " - f"{stamped} carry instrumentation=_INSTRUMENTATION" - ) diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index c95b1bc1..94796d9f 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -4601,13 +4601,12 @@ def __init__( span_attributes=dict(**{"type": type, "name": name, **span_attributes}, exec_counter=exec_counter), created=datetime.datetime.now(datetime.timezone.utc).isoformat(), ) + self._instrumentation = instrumentation or "braintrust-python-logger" internal_data["context"] = merge_span_origin_context( caller_location or {}, - "braintrust-python-logger", + self._instrumentation, self.state.span_origin_environment, - override_instrumentation_name=instrumentation, ) - self._instrumentation = instrumentation # TODO: can be simplified after `event` is typed. id = event.pop("id", None) diff --git a/py/src/braintrust/span_origin.py b/py/src/braintrust/span_origin.py index 22973c91..fa0237ad 100644 --- a/py/src/braintrust/span_origin.py +++ b/py/src/braintrust/span_origin.py @@ -84,15 +84,13 @@ def merge_span_origin_context( context: dict[str, Any] | None, instrumentation_name: str, environment: SpanOriginEnvironment | None, - override_instrumentation_name: str | None = None, ) -> dict[str, Any]: merged = dict(context or {}) span_origin = dict(merged.get("span_origin") or {}) - resolved_instrumentation = override_instrumentation_name or instrumentation_name span_origin = { "name": "braintrust.sdk.python", "version": _sdk_version(), - "instrumentation": {"name": resolved_instrumentation}, + "instrumentation": {"name": instrumentation_name}, **({"environment": environment} if environment else {}), **span_origin, } diff --git a/py/src/braintrust/test_otel.py b/py/src/braintrust/test_otel.py index b6b82caf..970e3478 100644 --- a/py/src/braintrust/test_otel.py +++ b/py/src/braintrust/test_otel.py @@ -146,38 +146,13 @@ def test_braintrust_span_processor_merges_span_origin_with_context_json_set_afte provider.shutdown() -def test_merge_span_origin_context_default_instrumentation(): +def test_merge_span_origin_context_uses_passed_instrumentation_name(): from braintrust.span_origin import merge_span_origin_context - merged = merge_span_origin_context({}, "braintrust-python-logger", None) - assert merged["span_origin"]["instrumentation"] == {"name": "braintrust-python-logger"} - - -def test_merge_span_origin_context_override_instrumentation(): - from braintrust.span_origin import merge_span_origin_context - - merged = merge_span_origin_context( - {}, - "braintrust-python-logger", - None, - override_instrumentation_name="openai-auto", - ) + merged = merge_span_origin_context({}, "openai-auto", None) assert merged["span_origin"]["instrumentation"] == {"name": "openai-auto"} -def test_merge_span_origin_context_empty_override_falls_back_to_default(): - from braintrust.span_origin import merge_span_origin_context - - for value in (None, ""): - merged = merge_span_origin_context( - {}, - "braintrust-python-logger", - None, - override_instrumentation_name=value, - ) - assert merged["span_origin"]["instrumentation"] == {"name": "braintrust-python-logger"} - - def test_span_impl_records_instrumentation_kwarg(monkeypatch): monkeypatch.setenv("BRAINTRUST_API_KEY", "test-key") monkeypatch.delenv("BRAINTRUST_ENVIRONMENT_TYPE", raising=False) @@ -192,8 +167,9 @@ def test_span_impl_records_instrumentation_kwarg(monkeypatch): pass assert parent._instrumentation == "openai-auto" - # Child spans do NOT inherit the parent's instrumentation name. - assert child._instrumentation is None + # Child spans do NOT inherit the parent's instrumentation name; + # they fall back to the channel default. + assert child._instrumentation == "braintrust-python-logger" def test_detect_environment_classifies_aws_ecs_before_lambda(monkeypatch): From 9ea3a88aaeff749bc14ac3328761dad1b3781a1a Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 14:33:45 +0000 Subject: [PATCH 5/9] docs: link the instrumentation spec by URL --- .agents/skills/sdk-integrations/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/skills/sdk-integrations/SKILL.md b/.agents/skills/sdk-integrations/SKILL.md index 18d35496..9a14247d 100644 --- a/.agents/skills/sdk-integrations/SKILL.md +++ b/.agents/skills/sdk-integrations/SKILL.md @@ -27,7 +27,7 @@ Do not design a new integration shape from scratch if an existing provider alrea The authoritative source for what integrations must capture, how spans must be structured, and which fields are allowed is the Braintrust instrumentation guide: -- `braintrustdata/braintrust-spec` -> `docs/instrumentation-guide.md` +- https://github.com/braintrustdata/braintrust-spec/blob/main/docs/instrumentation-guide.md Read the spec first when the task involves: From f6b9c6a0152cef5037eb5b6a0b41e1bb3513c71d Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 14:38:19 +0000 Subject: [PATCH 6/9] refactor: nest instrumentation under internal kwarg on start_span MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the `instrumentation` parameter behind an `internal: dict` kwarg on every span-creation surface. The `internal` dict is opaque to external callers by design — the name signals \"reserved for Braintrust SDK internals\" and the docstring says the keys inside may change without notice. Before: start_span(name=\"my-op\", instrumentation=\"openai-auto\") After: start_span(name=\"my-op\", internal={\"instrumentation\": \"openai-auto\"}) Integration shadow helpers become: def start_span(*args, **kwargs): internal = dict(kwargs.get(\"internal\") or {}) internal.setdefault(\"instrumentation\", _INSTRUMENTATION) kwargs[\"internal\"] = internal return _bt_start_span(*args, **kwargs) temporal and openai_agents (which open spans through Logger / parent-Span methods rather than the module-level `start_span`) pass `internal={\"instrumentation\": \"-auto\"}` explicitly. SKILL.md updated accordingly. --- .agents/skills/sdk-integrations/SKILL.md | 10 +++--- py/src/braintrust/integrations/adk/tracing.py | 4 ++- .../integrations/agentscope/tracing.py | 4 ++- .../braintrust/integrations/agno/tracing.py | 4 ++- .../integrations/anthropic/tracing.py | 4 ++- .../integrations/autogen/tracing.py | 4 ++- .../integrations/bedrock_runtime/tracing.py | 4 ++- .../integrations/claude_agent_sdk/tracing.py | 4 ++- .../braintrust/integrations/cohere/tracing.py | 4 ++- .../braintrust/integrations/crewai/tracing.py | 4 ++- .../braintrust/integrations/dspy/tracing.py | 4 ++- .../integrations/google_genai/tracing.py | 4 ++- .../integrations/huggingface_hub/tracing.py | 4 ++- .../integrations/instructor/tracing.py | 4 ++- .../integrations/langchain/callbacks.py | 4 ++- .../integrations/litellm/tracing.py | 4 ++- .../integrations/livekit_agents/tracing.py | 4 ++- .../integrations/llamaindex/tracing.py | 4 ++- .../integrations/mistral/tracing.py | 4 ++- .../braintrust/integrations/openai/tracing.py | 4 ++- .../integrations/openai_agents/tracing.py | 10 +++--- .../integrations/openrouter/tracing.py | 4 ++- .../integrations/pydantic_ai/tracing.py | 4 ++- .../integrations/temporal/plugin.py | 6 ++-- py/src/braintrust/logger.py | 36 +++++++++---------- py/src/braintrust/test_otel.py | 4 +-- 26 files changed, 98 insertions(+), 52 deletions(-) diff --git a/.agents/skills/sdk-integrations/SKILL.md b/.agents/skills/sdk-integrations/SKILL.md index 9a14247d..50560986 100644 --- a/.agents/skills/sdk-integrations/SKILL.md +++ b/.agents/skills/sdk-integrations/SKILL.md @@ -236,11 +236,11 @@ Preserve provider behavior. Tracing code must not change return values, control Every span an integration creates MUST carry `context.span_origin.instrumentation.name` identifying which integration produced it. Naming convention: `-auto` (e.g. `openai-auto`, `anthropic-auto`, `adk-auto`, `google-genai-auto`). Match the JS SDK exactly for cross-SDK dashboards to work. -Braintrust's `start_span(...)` (and every provider-level `start_span` method) accepts an `instrumentation: str | None` kwarg. When set, the SDK stamps it into `context.span_origin.instrumentation.name`. When unset, spans fall back to the channel default (`braintrust-python-logger` for direct logging, `braintrust-python-otel` for the OTel processor). +Braintrust's `start_span(...)` (and every provider-level `start_span` method) accepts an `internal: dict | None` kwarg reserved for SDK internals. Integrations pass `internal={"instrumentation": "-auto"}` to stamp `context.span_origin.instrumentation.name`. When unset, spans fall back to the channel default (`braintrust-python-logger` for direct logging, `braintrust-python-otel` for the OTel processor). The `internal` dict is intentionally opaque to signal that external callers should not use it — the keys inside are unstable. **Only integration-created spans should carry the integration's name.** User-owned spans nested inside a wrapped provider call — for example a `@traced` scorer running inside a wrapped agent turn — must NOT inherit the integration's `instrumentation.name`. The value does not propagate through the span parent/child edge; each `start_span` call independently decides its own value. -The standard per-integration pattern is a two-line shadow at the top of the integration's `tracing.py` (or `callbacks.py`/`plugin.py`): +The standard per-integration pattern is a small shadow at the top of the integration's `tracing.py` (or `callbacks.py`/`plugin.py`): ```python from braintrust.logger import start_span as _bt_start_span @@ -249,11 +249,13 @@ _INSTRUMENTATION = "-auto" def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) ``` -Every direct `start_span(...)` call site inside the module then flows through the shadow automatically. If the integration opens spans through a `Logger`, `Experiment`, or existing `Span` instance (i.e. `logger.start_span(...)`, `parent.start_span(...)`), pass `instrumentation=_INSTRUMENTATION` explicitly at those sites — the shadow only catches module-level `start_span` calls. +Every direct `start_span(...)` call site inside the module then flows through the shadow automatically. If the integration opens spans through a `Logger`, `Experiment`, or existing `Span` instance (i.e. `logger.start_span(...)`, `parent.start_span(...)`), pass `internal={"instrumentation": _INSTRUMENTATION}` explicitly at those sites — the shadow only catches module-level `start_span` calls. Tests should assert on the emitted span's `context.span_origin.instrumentation.name`. `SpanImpl` also exposes the resolved value as `span._instrumentation` for direct inspection in tests. diff --git a/py/src/braintrust/integrations/adk/tracing.py b/py/src/braintrust/integrations/adk/tracing.py index 029d22cb..c6367e8e 100644 --- a/py/src/braintrust/integrations/adk/tracing.py +++ b/py/src/braintrust/integrations/adk/tracing.py @@ -19,7 +19,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/agentscope/tracing.py b/py/src/braintrust/integrations/agentscope/tracing.py index be82d07b..f96e3cfd 100644 --- a/py/src/braintrust/integrations/agentscope/tracing.py +++ b/py/src/braintrust/integrations/agentscope/tracing.py @@ -12,7 +12,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py index f91bd09b..5c8983d4 100644 --- a/py/src/braintrust/integrations/agno/tracing.py +++ b/py/src/braintrust/integrations/agno/tracing.py @@ -10,7 +10,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index 2d4020d0..c316ea9a 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -14,7 +14,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/autogen/tracing.py b/py/src/braintrust/integrations/autogen/tracing.py index 0de63b19..3fa798be 100644 --- a/py/src/braintrust/integrations/autogen/tracing.py +++ b/py/src/braintrust/integrations/autogen/tracing.py @@ -11,7 +11,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/bedrock_runtime/tracing.py b/py/src/braintrust/integrations/bedrock_runtime/tracing.py index d7d2802f..fee3f1f1 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/tracing.py +++ b/py/src/braintrust/integrations/bedrock_runtime/tracing.py @@ -18,7 +18,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py index 93168cea..ed0f8d3c 100644 --- a/py/src/braintrust/integrations/claude_agent_sdk/tracing.py +++ b/py/src/braintrust/integrations/claude_agent_sdk/tracing.py @@ -30,7 +30,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/cohere/tracing.py b/py/src/braintrust/integrations/cohere/tracing.py index 805cafd3..5bf72cd0 100644 --- a/py/src/braintrust/integrations/cohere/tracing.py +++ b/py/src/braintrust/integrations/cohere/tracing.py @@ -28,7 +28,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/crewai/tracing.py b/py/src/braintrust/integrations/crewai/tracing.py index ed1b3bf7..baf4b046 100644 --- a/py/src/braintrust/integrations/crewai/tracing.py +++ b/py/src/braintrust/integrations/crewai/tracing.py @@ -55,7 +55,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index aaf4c696..e44b70f1 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -10,7 +10,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/google_genai/tracing.py b/py/src/braintrust/integrations/google_genai/tracing.py index 68dc475d..6b8c0382 100644 --- a/py/src/braintrust/integrations/google_genai/tracing.py +++ b/py/src/braintrust/integrations/google_genai/tracing.py @@ -26,7 +26,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/huggingface_hub/tracing.py b/py/src/braintrust/integrations/huggingface_hub/tracing.py index c8b1a0c3..839d3345 100644 --- a/py/src/braintrust/integrations/huggingface_hub/tracing.py +++ b/py/src/braintrust/integrations/huggingface_hub/tracing.py @@ -41,7 +41,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/instructor/tracing.py b/py/src/braintrust/integrations/instructor/tracing.py index e4a4a361..48b52b3b 100644 --- a/py/src/braintrust/integrations/instructor/tracing.py +++ b/py/src/braintrust/integrations/instructor/tracing.py @@ -35,7 +35,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/langchain/callbacks.py b/py/src/braintrust/integrations/langchain/callbacks.py index c09f504f..4ca0fdb8 100644 --- a/py/src/braintrust/integrations/langchain/callbacks.py +++ b/py/src/braintrust/integrations/langchain/callbacks.py @@ -19,7 +19,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index b5729512..750ad2d3 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -22,7 +22,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/livekit_agents/tracing.py b/py/src/braintrust/integrations/livekit_agents/tracing.py index b77c2d38..98e5fd29 100644 --- a/py/src/braintrust/integrations/livekit_agents/tracing.py +++ b/py/src/braintrust/integrations/livekit_agents/tracing.py @@ -26,7 +26,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/llamaindex/tracing.py b/py/src/braintrust/integrations/llamaindex/tracing.py index dce6e383..a91f1ee9 100644 --- a/py/src/braintrust/integrations/llamaindex/tracing.py +++ b/py/src/braintrust/integrations/llamaindex/tracing.py @@ -12,7 +12,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/mistral/tracing.py b/py/src/braintrust/integrations/mistral/tracing.py index f8bffe1b..7a1ed296 100644 --- a/py/src/braintrust/integrations/mistral/tracing.py +++ b/py/src/braintrust/integrations/mistral/tracing.py @@ -23,7 +23,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 6d897ddc..6348f481 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -27,7 +27,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/openai_agents/tracing.py b/py/src/braintrust/integrations/openai_agents/tracing.py index dcdba550..bc939504 100644 --- a/py/src/braintrust/integrations/openai_agents/tracing.py +++ b/py/src/braintrust/integrations/openai_agents/tracing.py @@ -12,7 +12,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) @@ -143,7 +145,7 @@ def on_trace_start(self, trace: tracing.Trace) -> None: name=trace.name, span_attributes={"type": "task", "name": trace.name}, metadata=metadata, - instrumentation=_INSTRUMENTATION, + internal={"instrumentation": _INSTRUMENTATION}, ) elif self._logger is not None: span = self._logger.start_span( @@ -151,7 +153,7 @@ def on_trace_start(self, trace: tracing.Trace) -> None: span_id=trace.trace_id, root_span_id=trace.trace_id, metadata=metadata, - instrumentation=_INSTRUMENTATION, + internal={"instrumentation": _INSTRUMENTATION}, ) else: span = start_span( @@ -349,7 +351,7 @@ def on_span_start(self, span: tracing.Span[tracing.SpanData]) -> None: name=_span_name(span), type=_span_type(span), start_time=_timestamp_from_maybe_iso(span.started_at), - instrumentation=_INSTRUMENTATION, + internal={"instrumentation": _INSTRUMENTATION}, ) self._spans[span.span_id] = created_span created_span.set_current() diff --git a/py/src/braintrust/integrations/openrouter/tracing.py b/py/src/braintrust/integrations/openrouter/tracing.py index 95565ed7..a425e1fc 100644 --- a/py/src/braintrust/integrations/openrouter/tracing.py +++ b/py/src/braintrust/integrations/openrouter/tracing.py @@ -20,7 +20,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index 60b755b9..8374d508 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -16,7 +16,9 @@ def start_span(*args, **kwargs): - kwargs.setdefault("instrumentation", _INSTRUMENTATION) + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal return _bt_start_span(*args, **kwargs) diff --git a/py/src/braintrust/integrations/temporal/plugin.py b/py/src/braintrust/integrations/temporal/plugin.py index 52f82ce0..53961c7a 100644 --- a/py/src/braintrust/integrations/temporal/plugin.py +++ b/py/src/braintrust/integrations/temporal/plugin.py @@ -247,7 +247,7 @@ async def execute_activity(self, input: temporalio.worker.ExecuteActivityInput) # Create Braintrust span for activity execution, linked to workflow span span = logger.start_span( - instrumentation="temporal-auto", + internal={"instrumentation": "temporal-auto"}, name=f"temporal.activity.{info.activity_type}", type="task", parent=parent_span_context or None, @@ -318,7 +318,7 @@ async def execute_workflow(self, input: temporalio.worker.ExecuteWorkflowInput) self._parent_span_context = _workflow_span_context(parent, ids) else: span = logger.start_span( - instrumentation="temporal-auto", + internal={"instrumentation": "temporal-auto"}, name=f"temporal.workflow.{info.workflow_type}", type="task", parent=parent, @@ -335,7 +335,7 @@ async def execute_workflow(self, input: temporalio.worker.ExecuteWorkflowInput) self._parent_span_context = _workflow_span_context(parent, ids) elif not temporalio.workflow.unsafe.is_replaying(): span = logger.start_span( - instrumentation="temporal-auto", + internal={"instrumentation": "temporal-auto"}, name=f"temporal.workflow.{info.workflow_type}", type="task", parent=parent_span_context or None, diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index 94796d9f..bd3ce163 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -217,7 +217,7 @@ def start_span( start_time: float | None = None, set_current: bool | None = None, parent: str | dict | None = None, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, **event: Any, ) -> "Span": """Create a new span. This is useful if you want to log more detailed trace information beyond the scope of a single log event. Data logged over several calls to `Span.log` will be merged into one logical row. @@ -231,7 +231,7 @@ def start_span( :param start_time: Optional start time of the span, as a timestamp in seconds. :param set_current: If true (the default), the span will be marked as the currently-active span for the duration of the context manager. :param parent: Optional parent info string for the span. The string can be generated from `[Span,Experiment,Logger].export`. If not provided, the current span will be used (depending on context). This is useful for adding spans to an existing trace. - :param instrumentation: Optional identifier for the instrumentation code creating this span. Used by SDK integrations to stamp `context.span_origin.instrumentation.name` (e.g. `"openai-auto"`). Leave unset for manual tracing. + :param internal: Reserved for Braintrust SDK internals. Not for external callers; the keys inside this dict are unstable and may change without notice. :param **event: Data to be logged. See `Experiment.log` for full details. :returns: The newly-created `Span` """ @@ -362,7 +362,7 @@ def start_span( start_time: float | None = None, set_current: bool | None = None, parent: str | dict | None = None, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, **event: Any, ): return self @@ -2921,7 +2921,7 @@ def start_span( parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, state: BraintrustState | None = None, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, **event: Any, ) -> Span: """Lower-level alternative to `@traced` for starting a span at the toplevel. It creates a span under the first active object (using the same precedence order as `@traced`), or if `parent` is specified, under the specified parent row, or returns a no-op span object. @@ -2961,7 +2961,7 @@ def start_span( event=event, state=state, lookup_span_parent=False, - instrumentation=instrumentation, + internal=internal, ) else: return parent_obj.start_span( @@ -2972,7 +2972,7 @@ def start_span( set_current=set_current, parent=parent, propagated_event=propagated_event, - instrumentation=instrumentation, + internal=internal, **event, ) @@ -4268,7 +4268,7 @@ def start_span( set_current: bool | None = None, parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, **event: Any, ) -> Span: """Create a new toplevel span underneath the experiment. The name defaults to "root" and the span type to "eval". @@ -4284,7 +4284,7 @@ def start_span( set_current=set_current, parent=parent, propagated_event=propagated_event, - instrumentation=instrumentation, + internal=internal, **event, ) @@ -4424,7 +4424,7 @@ def _start_span_impl( parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, lookup_span_parent: bool = True, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, **event: Any, ) -> Span: parent_args = _start_span_parent_args( @@ -4446,7 +4446,7 @@ def _start_span_impl( set_current=set_current, event=event, state=self.state, - instrumentation=instrumentation, + internal=internal, ) def __enter__(self) -> "Experiment": @@ -4535,7 +4535,7 @@ def __init__( root_span_id: str | None = None, state: BraintrustState | None = None, lookup_span_parent: bool = True, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, ): if span_attributes is None: span_attributes = SpanAttributes() @@ -4601,7 +4601,7 @@ def __init__( span_attributes=dict(**{"type": type, "name": name, **span_attributes}, exec_counter=exec_counter), created=datetime.datetime.now(datetime.timezone.utc).isoformat(), ) - self._instrumentation = instrumentation or "braintrust-python-logger" + self._instrumentation = (internal or {}).get("instrumentation") or "braintrust-python-logger" internal_data["context"] = merge_span_origin_context( caller_location or {}, self._instrumentation, @@ -4731,7 +4731,7 @@ def start_span( set_current: bool | None = None, parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, **event: Any, ) -> Span: if parent: @@ -4762,7 +4762,7 @@ def start_span( event=event, lookup_span_parent=lookup_span_parent, state=self.state, - instrumentation=instrumentation, + internal=internal, ) def end(self, end_time: float | None = None) -> float: @@ -5752,7 +5752,7 @@ def start_span( propagated_event: dict[str, Any] | None = None, span_id: str | None = None, root_span_id: str | None = None, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, **event: Any, ) -> Span: """Create a new toplevel span underneath the logger. The name defaults to "root" and the span type to "task". @@ -5770,7 +5770,7 @@ def start_span( propagated_event=propagated_event, span_id=span_id, root_span_id=root_span_id, - instrumentation=instrumentation, + internal=internal, **event, ) @@ -5805,7 +5805,7 @@ def _start_span_impl( span_id: str | None = None, root_span_id: str | None = None, lookup_span_parent: bool = True, - instrumentation: str | None = None, + internal: dict[str, Any] | None = None, **event: Any, ) -> Span: parent_args = _start_span_parent_args( @@ -5829,7 +5829,7 @@ def _start_span_impl( root_span_id=root_span_id, lookup_span_parent=lookup_span_parent, state=self.state, - instrumentation=instrumentation, + internal=internal, ) def export(self) -> str: diff --git a/py/src/braintrust/test_otel.py b/py/src/braintrust/test_otel.py index 970e3478..bb4adcbb 100644 --- a/py/src/braintrust/test_otel.py +++ b/py/src/braintrust/test_otel.py @@ -153,7 +153,7 @@ def test_merge_span_origin_context_uses_passed_instrumentation_name(): assert merged["span_origin"]["instrumentation"] == {"name": "openai-auto"} -def test_span_impl_records_instrumentation_kwarg(monkeypatch): +def test_span_impl_records_internal_instrumentation(monkeypatch): monkeypatch.setenv("BRAINTRUST_API_KEY", "test-key") monkeypatch.delenv("BRAINTRUST_ENVIRONMENT_TYPE", raising=False) monkeypatch.delenv("BRAINTRUST_ENVIRONMENT_NAME", raising=False) @@ -162,7 +162,7 @@ def test_span_impl_records_instrumentation_kwarg(monkeypatch): init_logger(project="test_instrumentation_kwarg") - with start_span(name="parent", instrumentation="openai-auto") as parent: + with start_span(name="parent", internal={"instrumentation": "openai-auto"}) as parent: with start_span(name="child") as child: pass From 46d712d10fe4b547278607890ee491efd4790e2f Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 14:41:28 +0000 Subject: [PATCH 7/9] refactor: type SpanInternalOptions as a TypedDict Adds a `SpanInternalOptions(TypedDict, total=False)` in logger.py so IDEs still get autocompletion / type-checking on the opaque `internal={...}` kwarg. Currently declares one field: class SpanInternalOptions(TypedDict, total=False): instrumentation: str Every span-creation surface (top-level `start_span`, `Span.start_span`, `SpanImpl.__init__` / `.start_span`, `Experiment.start_span`, `Dataset.start_span`, `Logger.start_span`, `_NoopSpan.start_span`) now types the kwarg as `internal: SpanInternalOptions | None = None`. pyright + mypy pass. --- py/src/braintrust/logger.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index bd3ce163..7c89d51a 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -150,6 +150,20 @@ class ParametersRef(TypedDict, total=False): version: str +class SpanInternalOptions(TypedDict, total=False): + """Options reserved for Braintrust SDK internals. + + Passed as the `internal` kwarg on `start_span` and its variants. External + callers should not use these fields; keys and semantics may change without + notice. + """ + + instrumentation: str + """Identifier for the instrumentation code creating the span. Stamped into + `context.span_origin.instrumentation.name`. Set by SDK integrations + (`openai-auto`, `anthropic-auto`, etc.).""" + + T = TypeVar("T") TMapping = TypeVar("TMapping", bound=Mapping[str, Any]) TMutableMapping = TypeVar("TMutableMapping", bound=MutableMapping[str, Any]) @@ -217,7 +231,7 @@ def start_span( start_time: float | None = None, set_current: bool | None = None, parent: str | dict | None = None, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> "Span": """Create a new span. This is useful if you want to log more detailed trace information beyond the scope of a single log event. Data logged over several calls to `Span.log` will be merged into one logical row. @@ -231,7 +245,7 @@ def start_span( :param start_time: Optional start time of the span, as a timestamp in seconds. :param set_current: If true (the default), the span will be marked as the currently-active span for the duration of the context manager. :param parent: Optional parent info string for the span. The string can be generated from `[Span,Experiment,Logger].export`. If not provided, the current span will be used (depending on context). This is useful for adding spans to an existing trace. - :param internal: Reserved for Braintrust SDK internals. Not for external callers; the keys inside this dict are unstable and may change without notice. + :param internal: Reserved for Braintrust SDK internals (see `SpanInternalOptions`). Not for external callers; the keys inside this dict are unstable and may change without notice. :param **event: Data to be logged. See `Experiment.log` for full details. :returns: The newly-created `Span` """ @@ -362,7 +376,7 @@ def start_span( start_time: float | None = None, set_current: bool | None = None, parent: str | dict | None = None, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ): return self @@ -2921,7 +2935,7 @@ def start_span( parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, state: BraintrustState | None = None, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: """Lower-level alternative to `@traced` for starting a span at the toplevel. It creates a span under the first active object (using the same precedence order as `@traced`), or if `parent` is specified, under the specified parent row, or returns a no-op span object. @@ -4268,7 +4282,7 @@ def start_span( set_current: bool | None = None, parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: """Create a new toplevel span underneath the experiment. The name defaults to "root" and the span type to "eval". @@ -4424,7 +4438,7 @@ def _start_span_impl( parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, lookup_span_parent: bool = True, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: parent_args = _start_span_parent_args( @@ -4535,7 +4549,7 @@ def __init__( root_span_id: str | None = None, state: BraintrustState | None = None, lookup_span_parent: bool = True, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, ): if span_attributes is None: span_attributes = SpanAttributes() @@ -4731,7 +4745,7 @@ def start_span( set_current: bool | None = None, parent: str | dict | None = None, propagated_event: dict[str, Any] | None = None, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: if parent: @@ -5752,7 +5766,7 @@ def start_span( propagated_event: dict[str, Any] | None = None, span_id: str | None = None, root_span_id: str | None = None, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: """Create a new toplevel span underneath the logger. The name defaults to "root" and the span type to "task". @@ -5805,7 +5819,7 @@ def _start_span_impl( span_id: str | None = None, root_span_id: str | None = None, lookup_span_parent: bool = True, - internal: dict[str, Any] | None = None, + internal: SpanInternalOptions | None = None, **event: Any, ) -> Span: parent_args = _start_span_parent_args( From 5287c69c6010f25a8ca0f7fb1a029dfe0437b47d Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 14:44:37 +0000 Subject: [PATCH 8/9] docs: tighten sdk-integrations SKILL.md --- .agents/skills/sdk-integrations/SKILL.md | 559 +++++++---------------- 1 file changed, 175 insertions(+), 384 deletions(-) diff --git a/.agents/skills/sdk-integrations/SKILL.md b/.agents/skills/sdk-integrations/SKILL.md index 50560986..54f1d406 100644 --- a/.agents/skills/sdk-integrations/SKILL.md +++ b/.agents/skills/sdk-integrations/SKILL.md @@ -5,466 +5,262 @@ description: Create or update Braintrust Python SDK integrations built on the in # SDK Integrations -Use this skill for integration work under `py/src/braintrust/integrations/`. - -Use `sdk-wrapper-migrations` instead when the provider already has a real implementation under `py/src/braintrust/wrappers//` and the task is to move that implementation into the integrations API. - -## Quick Start - -Before editing: - -1. Read the Braintrust instrumentation spec (see [Spec](#spec)). -2. Read the shared integration primitives. -3. Read the target provider package. -4. Pick the nearest existing integration as a reference. -5. Confirm provider versions and nox sessions from source files, not memory. -6. Decide the span shape before writing patchers. -7. Run the narrowest provider nox session first. - -Do not design a new integration shape from scratch if an existing provider already matches the problem. Do not invent span fields, metrics, or metadata keys that are not already in the spec. +Work under `py/src/braintrust/integrations/`. Use `sdk-wrapper-migrations` instead if the task is moving a real implementation from `py/src/braintrust/wrappers//` into the integrations API. ## Spec -The authoritative source for what integrations must capture, how spans must be structured, and which fields are allowed is the Braintrust instrumentation guide: - -- https://github.com/braintrustdata/braintrust-spec/blob/main/docs/instrumentation-guide.md +Authoritative source for span shape, allowed fields, and instrumentation rules: https://github.com/braintrustdata/braintrust-spec/blob/main/docs/instrumentation-guide.md -Read the spec first when the task involves: - -- span type, name, or hierarchy decisions -- what to put in `input`, `output`, `metadata`, `metrics`, `error`, or `context` -- tool calls, tool results, or available tool definitions -- prompt provenance metadata -- streaming, reasoning models, or prompt caching -- multimodal or attachment payloads -- distinguishing completion-style vs agentic-style APIs -- naming or namespacing new attributes - -Instrumentation MUST NOT capture data or emit metric/metadata keys that are not explicitly allowed by the spec. If the task needs a new field, update the spec before shipping the SDK change. +Read it first for anything involving span type/name, `input`/`output`/`metadata`/`metrics`/`error`/`context` shape, tool calls, prompt provenance, streaming, reasoning models, prompt caching, multimodal, completion-vs-agentic, or new attribute namespacing. Do not invent new top-level span fields, metric keys, or span types; update the spec first. ## Read First -Always read: - -- `py/src/braintrust/integrations/base.py` -- `py/src/braintrust/integrations/versioning.py` -- `py/src/braintrust/integrations/__init__.py` -- `py/src/braintrust/integrations/utils.py` -- `py/pyproject.toml` for provider matrix pins and cassette directory mappings -- `py/noxfile.py` - -Read these when working on an existing integration: - -- `py/src/braintrust/integrations//__init__.py` -- `py/src/braintrust/integrations//integration.py` -- `py/src/braintrust/integrations//patchers.py` -- `py/src/braintrust/integrations//tracing.py` -- `py/src/braintrust/integrations//test_*.py` +Always: `integrations/base.py`, `integrations/versioning.py`, `integrations/__init__.py`, `integrations/utils.py`, `pyproject.toml`, `noxfile.py`. -Read these when relevant: +Existing integration: `integrations//{__init__,integration,patchers,tracing,test_*}.py`. -- `py/src/braintrust/auto.py` for `auto_instrument()` changes -- `py/src/braintrust/conftest.py` for VCR behavior -- `py/src/braintrust/integrations/conftest.py` for per-version cassette directory resolution -- `py/src/braintrust/integrations/auto_test_scripts/` for subprocess auto-instrument coverage -- `py/src/braintrust/integrations/test_utils.py` when touching shared attachment materialization or multimodal payload shaping -- `py/src/braintrust/integrations/adk/test_adk.py`, `py/src/braintrust/integrations/anthropic/test_anthropic.py`, and `py/src/braintrust/integrations/google_genai/test_google_genai.py` for attachment-focused test layout patterns -- `py/src/braintrust/integrations/adk/tracing.py`, `py/src/braintrust/integrations/anthropic/tracing.py`, and `py/src/braintrust/integrations/google_genai/tracing.py` when handling multimodal content, binary inputs, generated media, or attachment materialization behavior +Situational: +- `auto.py` + `integrations/auto_test_scripts/` — anything touching `auto_instrument()` or import timing +- `conftest.py`, `integrations/conftest.py` — VCR / per-version cassette resolution +- `integrations/test_utils.py` — shared attachment / multimodal helpers +- `integrations/{adk,anthropic,google_genai}/tracing.py` — attachment + multimodal reference implementations -Do not forget `auto.py` and `auto_test_scripts/`. Import-order and subprocess regressions often only show up there. +Import-order and subprocess regressions usually only surface via `auto_test_scripts/`. Do not skip it. ## Version And CI Routing -Do not guess which provider versions or sessions apply. +Never guess versions or session names. The routing chain: -Use these files as the routing chain: +- `pyproject.toml` `[tool.braintrust.matrix]` — supported provider versions, what `latest` resolves to +- `pyproject.toml` `[tool.braintrust.cassette-dirs]` — versioned cassette directory ownership +- `integrations/versioning.py` — supported version helpers, gates +- `noxfile.py` — session names, package install, `BRAINTRUST_TEST_PACKAGE_VERSION` +- `.github/workflows/checks.yaml` — CI matrix + shard membership -- `py/pyproject.toml` `[tool.braintrust.matrix]`: supported provider versions and what `latest` resolves to -- `py/pyproject.toml` `[tool.braintrust.cassette-dirs]`: versioned cassette directory ownership -- `py/src/braintrust/integrations/versioning.py`: supported version helpers and gates -- `py/noxfile.py`: actual session names, package installation, and `BRAINTRUST_TEST_PACKAGE_VERSION` -- `.github/workflows/checks.yaml`: CI matrix and which sessions run in shards or static checks - -When changing version-gated behavior: - -1. Identify every matrix version for the provider. -2. Check whether the integration has `min_version`, `max_version`, `superseded_by`, or feature-detection branches. -3. Test the narrowest affected version first. -4. Add or update cassettes only for versions whose observable provider behavior intentionally changed. +When changing version-gated behavior: identify every matrix version, check `min_version`/`max_version`/`superseded_by`/feature-detect branches, test the narrowest affected version first, re-record cassettes only for versions whose observable provider behavior intentionally changed. ## Pick A Reference -Start from the nearest current integration: - -- ADK: direct method patching, `target_module`, `CompositeFunctionWrapperPatcher`, manual `wrap_*()` helpers, context propagation, inline data to `Attachment` -- Agno: multi-target patching, several related patchers, version-conditional fallbacks with `superseded_by` -- Anthropic: compact constructor patching, a small public surface, and multimodal request blocks that distinguish image vs document attachment payloads -- Google GenAI: multimodal tracing, generated media, output-side `Attachment` handling, and nested attachment materialization while preserving non-attachment values +Start from the nearest existing integration: -Choose the reference based on the hardest part of the task: - -- patcher topology -- tracing shape -- streaming behavior -- multimodal or binary payload handling +- **ADK** — direct method patching, `target_module`, `CompositeFunctionWrapperPatcher`, manual `wrap_*()`, context propagation, inline data → `Attachment` +- **Agno** — multi-target patching, related patchers, version-conditional fallbacks with `superseded_by` +- **Anthropic** — compact constructor patching, small public surface, multimodal image-vs-document attachment payloads +- **Google GenAI** — multimodal tracing, generated media, output-side attachments, nested materialization that preserves non-attachment values ## Default Workflow -Use this order unless the task is clearly narrower: - -1. Read the spec sections that cover the affected span shape. -2. Read shared primitives and the provider package. -3. Decide whether the target API is completion-style or agentic-style (see [Span Design Rules](#span-design-rules)). -4. Decide which public surface is being patched. Prefer stable public API entry points over internal helpers; internal targets break more often across provider versions. -5. Define the span shape: - - span `type` and `name` - - `input` - - `output` - - `metadata` - - `metrics` - - `error` when failures matter -6. Implement or update patchers. -7. Implement or update tracing helpers. -8. Add or update focused tests. -9. For provider-behavior bugs, make the primary regression test cassette-backed when practical, even if the implementation change is in local tracing/span post-processing of the provider response. -10. Be suspicious of mock/fake coverage for integrations. Do not choose mocks because they are convenient, faster, or easier to control. -11. Run the narrowest nox session first, then expand only if shared code changed. - -Do not start by wiring wrappers and only later decide what the span should contain. - -For behavior changes, prefer adding or adjusting a failing test first, then implementing until it passes. +1. Read the spec section(s) covering the affected span shape. +2. Read the provider package + shared primitives. +3. Decide completion-style vs agentic-style (see [Span Design](#span-design)). +4. Pick the public API surface to patch — prefer stable public entry points over internal helpers. +5. Define span shape (`type`, `name`, `input`, `output`, `metadata`, `metrics`, `error`) before writing patchers. +6. Implement patchers, then tracing helpers. +7. For behavior changes: add/update a failing test first, then fix (red → green). Cassette-backed if the provider payload triggers the bug. +8. Run the narrowest nox session first. Expand only if shared code changed. + +Do not start by wiring patchers and defer span-shape decisions. ## Route The Task -### New provider integration - -1. Create `py/src/braintrust/integrations//`. -2. Use this layout unless the provider is exceptionally small: - - `__init__.py` - - `integration.py` - - `patchers.py` - - `tracing.py` - - `test_.py` - - `cassettes//` when the provider uses HTTP (one subdirectory per version in the nox matrix, plus `latest/`) -3. Export the integration from `py/src/braintrust/integrations/__init__.py`. -4. Add or update the provider session in `py/noxfile.py`. -5. Update `py/src/braintrust/auto.py` only if the integration should participate in `auto_instrument()`. -6. Add subprocess coverage in `py/src/braintrust/integrations/auto_test_scripts/` when `auto_instrument()` changes. -7. Add the `_INSTRUMENTATION = "-auto"` shadow described in [Span origin](#span-origin-instrumentationname) at the top of `tracing.py`, and include an assertion in the integration's test suite that `context.span_origin.instrumentation.name` matches. +### New integration + +1. Create `integrations//` with `__init__.py`, `integration.py`, `patchers.py`, `tracing.py`, `test_.py`, and `cassettes//` (one dir per matrix version + `latest/`). +2. Export from `integrations/__init__.py`. +3. Add the provider session in `noxfile.py`. +4. Update `auto.py` only if it should participate in `auto_instrument()`; add a subprocess test in `auto_test_scripts/` when it does. +5. Add the `_INSTRUMENTATION = "-auto"` shadow (see [Span origin](#span-origin)) and assert it in the test suite. ### Existing integration update -1. Read the current provider package before editing. -2. Change only the affected patchers, tracing helpers, exports, tests, and cassettes. -3. Preserve the provider's public setup and `wrap_*()` surface unless the task explicitly changes it. -4. Do not touch `auto.py`, `integrations/__init__.py`, or `py/noxfile.py` unless the task requires it. -5. Even if `auto.py` does not change, check whether the behavior change also needs an auto-instrument subprocess test update. -6. Preserve existing span shape conventions unless the task is intentionally correcting them to match the spec. +Change only affected patchers, tracing, exports, tests, cassettes. Preserve the public `setup_*()` / `wrap_*()` surface. Don't touch `auto.py` / `integrations/__init__.py` / `noxfile.py` unless required. Check whether the change also needs a subprocess auto-instrument test update. ### `auto_instrument()` only -1. Update `py/src/braintrust/auto.py`. -2. Prefer `_instrument_integration(...)` over a custom `_instrument_*` helper when the standard pattern fits. -3. Add the integration import near the other integration imports. -4. Add or update the relevant subprocess auto-instrument test. - -### Setup, manual wrapping, and auto-instrument - -Treat these as distinct entry points: - -- `setup_()`: explicit package-level patching -- public `wrap_*()` helpers: manual wrapping of a provided class, function, or client -- `auto_instrument()`: import-order-sensitive discovery and setup - -When changing one entry point, check whether the other two should keep equivalent span behavior. Auto-instrument and manual paths should emit equivalent spans; if a provider-facing behavior change goes into one path, it usually needs to go into the other. If `auto_instrument()` changes or could be affected by import timing, validate it with a subprocess test instead of only calling the integration in-process. +Update `auto.py`. Prefer `_instrument_integration(...)` over a custom `_instrument_*` helper. Add/update the subprocess auto-instrument test. -## Package Layout Rules +### Entry-point parity -Keep provider-specific behavior in `py/src/braintrust/integrations//`. +The three entry points must emit equivalent spans: +- `setup_()` — package-level patching +- `wrap_*()` helpers — manual wrapping +- `auto_instrument()` — import-order-sensitive discovery -Typical ownership: +If a behavior change lands in one, check the other two. Validate `auto_instrument()` with a subprocess test, not just in-process. -- `__init__.py`: public exports, `setup_()`, public `wrap_*()` helpers -- `integration.py`: `BaseIntegration` subclass and patcher registration -- `patchers.py`: patchers and manual `wrap_*()` helpers -- `tracing.py`: request/response normalization, metadata extraction, stream handling, error logging -- `test_*.py`: provider behavior tests -- `cassettes/`: VCR recordings for provider HTTP traffic +## Package Layout -Keep `integration.py` thin. +Ownership: +- `__init__.py` — public exports, `setup_()`, `wrap_*()` helpers +- `integration.py` — `BaseIntegration` subclass + patcher registration (keep thin) +- `patchers.py` — patchers and manual wrapping helpers +- `tracing.py` — request/response normalization, metadata extraction, stream handling, error logging +- `test_*.py` — provider behavior tests +- `cassettes/` — VCR recordings -If logic is genuinely shared across integrations, move it to `py/src/braintrust/integrations/utils.py` instead of copying it into multiple providers. Before adding a local helper, check `utils.py` and neighboring integrations for an existing one. +If logic is shared across integrations, put it in `integrations/utils.py`, not copies. Check `utils.py` and neighboring integrations before writing a local helper. ## Integration Rules -Set up the integration declaratively: - -- set `name` -- set `import_names` -- set `patchers` -- set `min_version` or `max_version` only when feature detection is not enough - -Prefer feature detection first and version checks second. Use: - -- `detect_module_version(...)` -- `version_satisfies(...)` -- `make_specifier(...)` - -Let `BaseIntegration.resolve_patchers()` reject duplicate patcher ids. Do not hide duplicates. - -### Non-invasiveness and safety - -Instrumentation must not change user-visible behavior: - -- Errors from the provider MUST propagate to the caller. Do not swallow or rewrap them. -- Return values, iterator/async-iterator semantics, and generator/subclass behavior MUST be preserved. -- Sync and async traced schemas MUST stay aligned when the provider exposes both. -- Setup, teardown, and patching MUST be idempotent. Applying setup twice, tearing down twice, or wrapping twice must remain safe. Rely on the base patcher's idempotence marker. -- Contain instrumentation failures. Errors in extraction, normalization, or logging code must be logged or ignored, not raised into the user's call path. Only intentional provider errors should surface. -- Treat provider inputs, results, events, headers, and metadata as untrusted. Avoid attribute access that could trigger arbitrary code, and avoid mutating third-party objects. -- Limit instrumentation to AI-generation-relevant operations (LLM calls, embeddings, tool executions, media generation, agent runs). Do not instrument unrelated CRUD or platform-management APIs. - -### Scope of instrumentation - -Follow the spec closely for spec-defined fields — `input`, `output`, `metrics`, `error`, tool-call structure, prompt provenance, attachments. Do not invent new top-level fields, metric keys, or span types; update the spec first if you genuinely need one. - -`metadata` is treated slightly more loosely: use an **allowlist per provider**, not a denylist. Capture the spec-defined keys (`model`, `provider`, `tools`, `tool_choice`, `parallel_tool_calls`, `max_tool_calls`, `prompt`, and the permitted request-config subset) plus provider-specific detail fields you deliberately choose to include — provider request/response ids, safety data, model-family flags, and similar. Do not dump entire raw request/response objects into metadata; each captured key should be an intentional decision. Redact obvious secrets. If a provider-specific field ends up broadly useful across SDKs, promote it to the spec. - -Preserve provider behavior. Tracing code must not change return values, control flow, or error behavior unless the task explicitly requires it. - -### Span origin (`instrumentation.name`) - -Every span an integration creates MUST carry `context.span_origin.instrumentation.name` identifying which integration produced it. Naming convention: `-auto` (e.g. `openai-auto`, `anthropic-auto`, `adk-auto`, `google-genai-auto`). Match the JS SDK exactly for cross-SDK dashboards to work. - -Braintrust's `start_span(...)` (and every provider-level `start_span` method) accepts an `internal: dict | None` kwarg reserved for SDK internals. Integrations pass `internal={"instrumentation": "-auto"}` to stamp `context.span_origin.instrumentation.name`. When unset, spans fall back to the channel default (`braintrust-python-logger` for direct logging, `braintrust-python-otel` for the OTel processor). The `internal` dict is intentionally opaque to signal that external callers should not use it — the keys inside are unstable. - -**Only integration-created spans should carry the integration's name.** User-owned spans nested inside a wrapped provider call — for example a `@traced` scorer running inside a wrapped agent turn — must NOT inherit the integration's `instrumentation.name`. The value does not propagate through the span parent/child edge; each `start_span` call independently decides its own value. - -The standard per-integration pattern is a small shadow at the top of the integration's `tracing.py` (or `callbacks.py`/`plugin.py`): - -```python -from braintrust.logger import start_span as _bt_start_span - -_INSTRUMENTATION = "-auto" - +Declarative setup on the `BaseIntegration` subclass: `name`, `import_names`, `patchers`. Use `min_version`/`max_version` only when feature detection isn't enough. Prefer `detect_module_version(...)` / `version_satisfies(...)` / `make_specifier(...)`. -def start_span(*args, **kwargs): - internal = dict(kwargs.get("internal") or {}) - internal.setdefault("instrumentation", _INSTRUMENTATION) - kwargs["internal"] = internal - return _bt_start_span(*args, **kwargs) -``` - -Every direct `start_span(...)` call site inside the module then flows through the shadow automatically. If the integration opens spans through a `Logger`, `Experiment`, or existing `Span` instance (i.e. `logger.start_span(...)`, `parent.start_span(...)`), pass `internal={"instrumentation": _INSTRUMENTATION}` explicitly at those sites — the shadow only catches module-level `start_span` calls. - -Tests should assert on the emitted span's `context.span_origin.instrumentation.name`. `SpanImpl` also exposes the resolved value as `span._instrumentation` for direct inspection in tests. +Let `BaseIntegration.resolve_patchers()` reject duplicate patcher ids; don't hide duplicates. -## Span Design Rules +**Non-invasiveness:** +- Provider errors MUST propagate. Never swallow or rewrap. +- Preserve return types, iterator/async-iterator semantics, generator/subclass behavior. +- Sync and async traced schemas MUST stay aligned when both exist. +- Setup / teardown / patching MUST be idempotent (rely on the base patcher marker). +- Contain instrumentation failures. Extraction/normalization/logging errors must be logged or ignored, never raised into the user's call path. +- Treat provider inputs/results/events/headers as untrusted. Avoid arbitrary attribute access; don't mutate third-party objects. +- Only instrument AI-generation-relevant ops (LLM calls, embeddings, tool exec, media gen, agent runs). No unrelated CRUD. -### Span type and name +## Span Design -Every span must set the correct `type` and a stable `name`: +### Type + name -- `llm`: a single provider API call -- `task`: a parent span for an agent run, pipeline step, or named operation -- `tool`: a model-initiated tool/function execution +- `llm` — one provider API call +- `task` — parent for an agent run, pipeline step, or named operation +- `tool` — model-initiated tool/function execution -Use spec-recommended names where they exist (for example `Chat Completion` for OpenAI, `anthropic.messages.create` for Anthropic, `generate_content` for Google). For new providers, pick a stable, provider-specific name. +Use spec-recommended names (`Chat Completion`, `anthropic.messages.create`, `generate_content`). For new providers, pick a stable provider-specific name. ### Completion-style vs agentic-style -Decide the span shape before writing patchers: - -- Completion-style APIs (single request/response, user executes any tool calls): one `llm` span per API call, no children. Tool calls the model requests appear in the span's `output`; the SDK does not create `tool` spans. -- Agentic-style APIs (the SDK/framework runs the tool loop internally): one parent `task` span wrapping child `llm` spans (one per underlying model call) and child `tool` spans (one per tool execution). Preserve execution order. - -For frameworks: completion-style frameworks (for example LiteLLM) must produce provider-shaped `llm` spans and set `metadata.provider` to the underlying provider (`openai`, `anthropic`, etc.), not the framework name. Agentic-style frameworks (Vercel AI SDK with tools, LangChain agents, OpenAI Agents SDK, Claude Agent SDK, ADK, Agno) must produce the full parent-plus-children tree. +- **Completion** (single request/response, user runs any tool calls): one `llm` span, no children. Tool calls appear in `output`. +- **Agentic** (SDK/framework runs the tool loop): one parent `task` wrapping child `llm` (per model call) + `tool` (per tool exec) spans in execution order. -### Canonical payload format +Completion-style *frameworks* (LiteLLM, OpenRouter, etc.) still emit `llm` spans with `metadata.provider` set to the underlying provider, NOT the framework name. Agentic frameworks (Vercel AI SDK w/ tools, LangChain agents, OpenAI Agents SDK, Claude Agent SDK, ADK, Agno) emit the full parent+children tree. -Braintrust uses the OpenAI Chat Completions message format as the canonical representation for LLM `input` and `output`. The UI parses, displays, and diffs spans assuming this format. +### Payload format -- For providers that have a dedicated Braintrust UI normalizer (currently OpenAI, Anthropic, Google), instrumentation MAY preserve the provider-native payload. Set `metadata.provider` correctly so the UI can normalize. -- For any other provider, instrumentation MUST convert payloads into the OpenAI Chat Completions format so the UI can render them. If the provider exposes a system prompt as a separate parameter, insert it into the messages array as a `role: "system"` entry. +Canonical shape is OpenAI Chat Completions format. Providers with dedicated UI normalizers (OpenAI, Anthropic, Google) MAY preserve provider-native payloads if `metadata.provider` is set correctly; all other providers MUST convert to OpenAI shape. If a provider passes system prompt separately, insert it as a `role: "system"` entry. -### Input, output, metadata, metrics, error +### Fields -Build readable spans. Do not dump raw `args` and `kwargs` unless the provider API already exposes a clean schema. +- `input` — meaningful user request (messages / prompt / provider-native input) +- `output` — meaningful provider result (normalized, not opaque SDK instances) +- `metadata` — see below +- `metrics` — spec-listed keys only (see [Metrics](#metrics)) +- `error` — pass the `Exception` instance directly to span logging; do not pre-format -Use this rubric: +**Every `llm` span MUST include `metadata.model` and `metadata.provider`** (`provider` = whose pricing applies, even when going through a gateway or framework). -- `input`: the meaningful user request (messages, prompt, or provider-native input structure) -- `output`: the meaningful provider result (choices, message, or provider-native response structure) -- `metadata`: only spec-allowed fields such as `model`, `provider`, `tools`, `tool_choice`, `parallel_tool_calls`, `max_tool_calls`, `prompt`, and the permitted request-config subset (`temperature`, `top_p`, `max_tokens`, `frequency_penalty`, `presence_penalty`, `stop`, `response_format`) -- `metrics`: only spec-allowed keys (see [Metrics](#metrics)) -- `error`: exceptions or failure information; pass the `Exception` instance directly to span logging rather than pre-formatting it into strings +**Tool definitions go in `metadata.tools`** (OpenAI-shaped regardless of underlying provider), NOT in `input` messages. Preserve provider-native built-in tool types (Anthropic `computer_use`, OpenAI `web_search`) as opaque JSON-serializable entries. Never log executable tool handlers. -Every `llm` span MUST include `metadata.model` (the resolved model id from the response, including any version suffix when the API returns one) and `metadata.provider` (the provider whose pricing applies, even when the caller went through a gateway or framework). +**Prompt provenance goes in `metadata.prompt`** (`id`, `project_id`, `version`, `variables`, plus `prompt_session_id` for playground calls). Not in `input`, request payloads, or `metadata.tools`. Strip carrier fields like `span_info` before logging. -Tool definitions are request configuration, not conversation content: the schemas passed to the model MUST go in `metadata.tools` (OpenAI-shaped, regardless of the underlying provider), not in the `input` messages. Preserve provider-native built-in tool types (for example Anthropic `computer_use`, OpenAI `web_search`) as opaque JSON-serializable entries in `metadata.tools`. Do not log executable tool handlers or closures. - -Prompt provenance for Braintrust-managed prompts MUST go in `metadata.prompt` (`id`, `project_id`, `version`, `variables`, and `prompt_session_id` for playground/prompt-session calls). Do not put prompt provenance in `input`, request payloads, or `metadata.tools`. Wrapper-only carrier fields such as `span_info` are internal plumbing; strip them from the provider request and do not log them. +**`metadata` allowlist-per-provider:** capture the spec-defined keys plus provider-specific detail fields (request/response ids, safety data, model-family flags) chosen deliberately. Do not dump entire raw request/response objects. Redact secrets. If a provider-specific field ends up broadly useful, promote it to the spec. ### Token metrics -Avoid double-counting token metrics: - -- the integration that directly owns the model/provider API response should own token accounting -- orchestration/framework integrations should usually not log token metrics when underlying provider integrations can create leaf spans with usage metrics; agentic parent `task` spans MAY aggregate token counts across children when the framework does not delegate to a separately instrumented provider client -- do not add fragile provider-specific ownership checks such as "if OpenAI is patched, skip metrics"; prefer a clear span ownership rule instead +The integration that directly owns the model/provider API response owns token accounting. Orchestration/framework integrations should not log token metrics when underlying provider integrations create leaf spans with usage. Agentic parent `task` spans MAY aggregate across children when the framework doesn't delegate to a separately instrumented provider client. Do not add "if OpenAI is patched, skip metrics" checks — define clear ownership instead. ### Shaping guidance -Good span shaping usually means: +- Flatten positional args into named fields; normalize SDK objects into dicts/lists/scalars; drop noisy transport fields. +- Aggregate streaming chunks into one final `output` + stream-specific `metrics`. One `llm` span per API call, not per chunk. +- Do not over-serialize. Braintrust serializes at send/log time. Integration tracing only needs readable Python dicts/lists/scalars and materialized attachments. +- Keep wrapper bodies thin: prepare traced input, open span, call provider, normalize result, log. +- Do not wrap `span.log(...)` / `span.set_attributes(...)` in broad try/except. Braintrust span methods are boundary-safe. -- flatten positional arguments into named fields -- normalize provider SDK objects into dicts, lists, or scalars when that improves readability -- drop duplicate or noisy transport fields -- aggregate streaming chunks into one final `output` plus stream-specific `metrics` +### Span origin -Do not over-serialize in integration code. Braintrust handles serialization when sending/logging spans, so integration tracing helpers usually only need to shape readable Python dicts/lists/scalars and materialize attachments where appropriate. Avoid unnecessary JSON dumps/loads, recursive conversion, or stringification just to make values serializable. +Every span an integration creates MUST carry `context.span_origin.instrumentation.name = "-auto"` (e.g. `openai-auto`, `anthropic-auto`, `adk-auto`, `google-genai-auto`). Match the JS SDK exactly for cross-SDK filtering. -Keep wrapper bodies thin: prepare traced input, open the span, call the provider, normalize the result, and log `output`/`metadata`/`metrics`. +`start_span(...)` and all provider-level `start_span` methods accept `internal: SpanInternalOptions | None`, a TypedDict reserved for SDK internals — external callers should not use it. Integrations pass `internal={"instrumentation": "-auto"}`. When unset, spans fall back to the channel default (`braintrust-python-logger` / `braintrust-python-otel`). -Braintrust span logging methods are boundary-safe and should not throw during normal integration use. Do not wrap `span.log(...)`, `span.set_attributes(...)`, or similar Braintrust span methods in broad `try`/`except` blocks. Only catch exceptions around provider calls or around integration-owned conversion code when there is a specific expected failure mode and a clear fallback. +The name does NOT propagate through the parent/child edge — each `start_span` call decides independently. User-owned spans nested inside a wrapped call (e.g. a `@traced` scorer) must NOT inherit the integration's name. -Prefer provider-local helpers in `tracing.py`, for example: +Standard pattern at the top of `tracing.py` (or `callbacks.py` / `plugin.py`): ```python -def _prepare_traced_call(args: list[Any], kwargs: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: - ... +from braintrust.logger import start_span as _bt_start_span +_INSTRUMENTATION = "-auto" -def _process_result(result: Any, start: float) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: - ... -``` -## Metrics +def start_span(*args, **kwargs): + internal = dict(kwargs.get("internal") or {}) + internal.setdefault("instrumentation", _INSTRUMENTATION) + kwargs["internal"] = internal + return _bt_start_span(*args, **kwargs) +``` -Only emit spec-listed metric keys. Do not invent metric keys. +Module-level `start_span(...)` calls flow through automatically. Calls that go through a `Logger` / `Experiment` / parent-`Span` instance (`logger.start_span(...)`, `parent.start_span(...)`) need `internal={"instrumentation": _INSTRUMENTATION}` explicitly. Tests assert `span["context"]["span_origin"]["instrumentation"]["name"] == "-auto"`; `SpanImpl._instrumentation` exposes the resolved value. -Standard LLM metrics: +## Metrics -- `tokens`, `prompt_tokens`, `completion_tokens`: required for LLM spans (values MUST be non-negative) -- `time_to_first_token`: required for streaming spans; measured by the SDK from request start to the first chunk -- `completion_reasoning_tokens`: required when the provider reports reasoning tokens (for example OpenAI o-series) -- `prompt_cached_tokens`, `prompt_cache_creation_tokens`, `prompt_cache_creation_5m_tokens`, `prompt_cache_creation_1h_tokens`: capture when the provider reports prompt caching -- `prompt_audio_tokens`, `completion_audio_tokens`, `completion_image_tokens`: capture when the provider reports them for audio/image models -- `start`, `end`: standard span timing +Only emit spec-listed keys: +- `tokens`, `prompt_tokens`, `completion_tokens` — required for LLM spans; values MUST be non-negative +- `time_to_first_token` — required for streaming; SDK measures from request start to first chunk +- `completion_reasoning_tokens` — required when provider reports reasoning tokens (o-series etc.) +- `prompt_cached_tokens`, `prompt_cache_creation_tokens`, `prompt_cache_creation_5m_tokens`, `prompt_cache_creation_1h_tokens` — provider prompt caching +- `prompt_audio_tokens`, `completion_audio_tokens`, `completion_image_tokens` — audio/image models when provider reports +- `start`, `end` — standard span timing -Streaming spans still produce a single `llm` span per API call, with `input`/`output` accumulated from chunks. Capture token counts from stream usage metadata when the provider surfaces it (for example OpenAI `stream_options.include_usage`). +For streaming, still produce one span per API call with accumulated `input`/`output`. Capture usage from stream metadata (e.g. OpenAI `stream_options.include_usage`) when surfaced. -For reasoning models, capture the full output structure (reasoning summaries plus assistant message blocks) when the provider exposes them, and include prior reasoning blocks in the `input` for multi-turn calls. +For reasoning models, capture the full output structure (reasoning summaries + message blocks) and include prior reasoning in `input` for multi-turn calls. ## Multimodal And Attachments -When a request or response contains inline binary media (images, PDFs, audio, video, and similar), replace the raw media leaf in the span payload with a Braintrust attachment. Do not add a separate top-level `attachments` list. +Inline binary media (images, PDFs, audio, video) MUST be replaced by a Braintrust attachment at the leaf position. Do not add a separate top-level `attachments` list. -Treat binary payloads as attachments, not logged bytes: - -- prefer the shared `_materialize_attachment(...)` helper in `py/src/braintrust/integrations/utils.py` over provider-local base64 or file-decoding code -- convert provider-owned raw `bytes`, base64 payloads, data URLs, file inputs, and generated media into `braintrust.logger.Attachment` objects when Braintrust should upload the content (the SDK converts these into `braintrust_attachment` references on the wire) -- preserve normal remote URLs as strings; do not fetch remote URLs solely to create an attachment -- use the repo's existing multimodal payload shapes after materialization: - - images -> `{"image_url": {"url": attachment}}` - - non-image media/documents/files -> `{"file": {"file_data": attachment, "filename": resolved.filename}}` -- do not force non-image payloads through `image_url` shims -- if attachment materialization or upload fails, keep the original value instead of dropping it or replacing it with `None`, and do not raise into the user's call path -- preserve non-attachment values while walking nested payloads unless you are intentionally normalizing them for readability -- keep useful metadata such as MIME type, size, safety data, filenames, or provider ids next to the attachment -- when a provider-native payload has a dedicated UI normalizer (OpenAI, Anthropic, Google), you may preserve the provider-native structure and replace only the raw media leaf - -For generated media on the output side, log the attachment inside `output` (not `metadata`). For streaming media outputs, aggregate chunks into the final `output` attachment; do not log raw chunks as separate blobs. +- Prefer `_materialize_attachment(...)` in `integrations/utils.py`. Don't reimplement base64/file decoding. +- Convert raw `bytes` / base64 / data URLs / file inputs / generated media → `braintrust.logger.Attachment`. Preserve remote URLs as strings — do not fetch to create an attachment. +- Payload shape after materialization: + - images → `{"image_url": {"url": attachment}}` + - non-image media/documents/files → `{"file": {"file_data": attachment, "filename": resolved.filename}}` +- Do not force non-image payloads through `image_url`. +- If materialization fails, keep the original value; never drop, null it out, or raise. +- Preserve non-attachment values while walking nested payloads unless intentionally normalizing. +- Keep MIME type, size, safety data, filenames, provider ids next to the attachment. +- Providers with UI normalizers (OpenAI, Anthropic, Google) — MAY preserve the provider-native structure and only replace the raw media leaf. +- Generated media → log the attachment in `output`, not `metadata`. For streaming, aggregate into one final `output` attachment. ## Patcher Rules -Create one patcher per coherent patch target. - -Prefer: +One patcher per coherent target. Prefer: -- `FunctionWrapperPatcher` for one import path or one constructor/method surface -- `CompositeFunctionWrapperPatcher` for one logical surface spread across multiple related targets -- `CallbackPatcher` for setup side effects after applicability succeeds +- `FunctionWrapperPatcher` — one import path / one constructor / one method surface +- `CompositeFunctionWrapperPatcher` — one logical surface across multiple related targets +- `CallbackPatcher` — setup side effects after applicability succeeds -Use `target_module` when the patch target lives outside the module named by `import_names`, especially for optional or deep submodules. +Use `target_module` for patch targets outside the module named by `import_names` (deep or optional submodules). Use `superseded_by` for version-conditional fallbacks — not custom target-selection logic. Use lower `priority` only when ordering matters (e.g. context propagation before tracing). -Use `superseded_by` for version-conditional fallbacks instead of custom target-selection logic. - -Use lower `priority` only when patch ordering really matters, such as context propagation before tracing. - -Manual wrapping helpers should be thin: +Manual wrapping helpers stay thin: ```python def wrap_agent(Agent: Any) -> Any: return AgentPatcher.wrap_target(Agent) ``` -Require every patcher to have: - -- a stable `name` -- clean existence checks -- version gating only when necessary -- idempotence through the base patcher marker - -Prefer patching stable public API surfaces (documented client methods, top-level constructors). Reach into internal helpers only when the public surface is genuinely insufficient, and expect those patches to need version-specific maintenance. +Every patcher needs: stable `name`, clean existence checks, version gating only when necessary, idempotence via the base patcher marker. Prefer patching stable public API surfaces; internal helpers need version-specific maintenance. ## Testing Rules -Keep tests in the provider package. - -Default bug-fix workflow: red -> green. +Tests live in the provider package. Default bug-fix workflow: red → green — add/update a failing test first, then fix. -- First add or update a focused test that reproduces the integration bug. -- Then implement the fix. -- Only skip this when the task explicitly asks for a different approach. - -Prefer VCR-backed real provider coverage with `@pytest.mark.vcr`. This includes span-shaping/tracing bugs where the bad behavior is triggered by a real provider response payload; do not treat those as mock-first just because the code path is local. - -Default stance: if the behavior is provider-facing, assume mocks/fakes are the wrong tool until proven otherwise. A mock should need justification, not the other way around. - -Use mocks or fakes only for cases that are hard to drive through recordings, such as: - -- narrow error injection -- purely local version-routing logic -- patcher existence checks -- provider-independent helper logic where the provider response shape is not part of the contract being validated - -Do not replace or skip a cassette-backed regression with a mock/fake test merely because the implementation change lives in `tracing.py`, a serializer, or another local post-processing layer. If a real provider payload is what triggers the bug, the main regression test should reflect that real payload. - -Test emitted spans, not just provider return values. In particular, assert on: +**Prefer VCR-backed real provider coverage with `@pytest.mark.vcr`.** The burden of proof is on mocks: use them only for narrow error injection, purely local version-routing logic, patcher existence checks, or provider-independent helpers where the response shape is not part of the contract. Do not swap a cassette-backed regression for a mock just because the fix lives in `tracing.py` or a serializer — if a real payload triggers the bug, the primary regression stays cassette-backed. +Assert on emitted spans (not just provider return values): - span `type` and `name` -- `input` shape (messages/prompt/config fields the spec requires) -- `output` shape (normalized provider result, not opaque SDK instances) -- `metadata.model`, `metadata.provider`, and any `metadata.tools` / `metadata.tool_choice` / `metadata.prompt` present -- required `metrics` keys (token counts for LLM spans; `time_to_first_token` for streaming) -- parent/child structure for agentic APIs (parent `task`, child `llm` and `tool` spans in execution order) -- `context.span_origin.instrumentation.name` equals the integration's `-auto` identifier -- attachment conversion for binary inputs or generated media, including that images land under `image_url.url`, non-image payloads land under `file.file_data`, and traced payloads contain `Attachment` objects rather than raw bytes or base64 blobs -- error propagation and error logging behavior -- idempotence of setup/teardown/wrapping -- patcher resolution and duplicate detection when relevant - -Cover the surfaces that changed: - -- direct `wrap_*()` behavior -- setup-time patching -- sync behavior -- async behavior -- streaming behavior -- idempotence -- failure and error logging - -For streaming changes, verify both: - -- the provider still returns the expected iterator or async iterator -- the final logged span contains the aggregated `output` and stream-specific `metrics` +- `input` shape (spec-required messages/prompt/config fields) +- `output` shape (normalized, not opaque SDK instances) +- `metadata.model`, `metadata.provider`, `metadata.tools`/`tool_choice`/`prompt` when present +- required `metrics` keys (tokens for LLM; `time_to_first_token` for streaming) +- parent/child structure for agentic APIs (parent `task`, child `llm`+`tool` in execution order) +- `context.span_origin.instrumentation.name == "-auto"` +- attachments: images under `image_url.url`, non-images under `file.file_data`, `Attachment` objects (not raw bytes/base64) +- error propagation, error logging +- setup/teardown/wrapping idempotence, patcher resolution when relevant -Keep VCR cassettes in `py/src/braintrust/integrations//cassettes//` (e.g. `cassettes/latest/`, `cassettes/0.48.0/`). Nox sessions set `BRAINTRUST_TEST_PACKAGE_VERSION` automatically so cassettes land in the correct version subdirectory. Do not add per-test `vcr_cassette_dir` or `cassette_library_dir` fixtures; the shared `py/src/braintrust/integrations/conftest.py` handles version routing. Re-record only when behavior intentionally changes. +For streaming, assert both the provider iterator/async-iterator still works AND the final span has aggregated `output` + stream-specific `metrics`. -When the provider returns binary HTTP responses or generated media, sanitize cassettes as needed so fixtures do not store raw file bytes. +Cassettes live in `integrations//cassettes//` (e.g. `cassettes/latest/`, `cassettes/0.48.0/`). Nox sets `BRAINTRUST_TEST_PACKAGE_VERSION` so cassettes land correctly. Do not add per-test `vcr_cassette_dir` / `cassette_library_dir` fixtures — `integrations/conftest.py` handles it. Re-record only when behavior intentionally changed. Sanitize cassettes when the provider returns binary bodies. -When choosing test commands, confirm the actual session name in `py/noxfile.py` instead of assuming it matches the provider folder. +Confirm the exact session name from `noxfile.py` — don't assume it matches the folder. ## Commands @@ -478,34 +274,29 @@ cd py && make lint ## Validation Checklist -- Run the narrowest provider session first. -- If the change touches patchers, setup behavior, import timing, or anything that could affect `auto_instrument()`, run the relevant subprocess auto-instrument test from `py/src/braintrust/integrations/auto_test_scripts/`. -- Run the relevant auto-instrument subprocess test if `auto.py` changed. -- Run `cd py && make test-core` if shared integration code changed. -- Run `cd py && make lint` before handoff when shared files or repo-level wiring changed. +- Narrowest provider session first. +- Subprocess auto-instrument test if patchers / setup / import timing / `auto.py` changed. +- `make test-core` if shared integration code changed. +- `make lint` before handoff if shared files or repo wiring changed. ## Common Mistakes -Avoid these failures: - -- treating a wrapper migration as fresh integration work -- changing shared integration primitives when provider-local code should own the behavior -- combining unrelated patch targets into one patcher -- forgetting repo-level wiring for new providers: `integrations/__init__.py`, `py/noxfile.py`, and sometimes `auto.py` -- forgetting the subprocess auto-instrument tests -- forgetting async or streaming coverage -- re-recording cassettes when behavior did not intentionally change -- adding a custom `_instrument_*` helper where `_instrument_integration()` already fits -- forgetting `target_module` for deep or optional patch targets -- inventing new top-level span fields, metric keys, or span types beyond what the spec allows (metadata is treated more loosely — see the metadata allowlist guidance above) -- putting tool definitions in `input` instead of `metadata.tools`, or including prompt provenance anywhere other than `metadata.prompt` -- forgetting to set `instrumentation=-auto` on spans an integration opens, or leaking that identifier onto user-owned spans nested inside a wrapped call -- using the framework name as `metadata.provider` for a completion-style framework wrapper instead of the underlying provider -- producing per-chunk spans for a streamed call instead of one accumulated span per API call -- letting instrumentation errors surface into the user's call path, or swallowing real provider errors -- double-counting token metrics in both orchestration/framework spans and provider leaf spans -- adding provider-specific token ownership detection instead of defining clear metric ownership for the integration -- doing excessive serialization/stringification in tracing code even though Braintrust serializes span payloads at send/log time -- wrapping Braintrust span logging methods in broad `try`/`except` blocks even though those methods are designed not to throw -- forcing non-image attachments through `image_url` shims, dropping unrecognized file inputs, or re-serializing non-attachment values while materializing payloads -- patching internal helpers when a stable public API surface would work +- Treating a wrapper migration as fresh integration work (use `sdk-wrapper-migrations`). +- Changing shared primitives when provider-local code should own the behavior. +- Combining unrelated patch targets into one patcher. +- Forgetting repo wiring for new providers (`integrations/__init__.py`, `noxfile.py`, sometimes `auto.py`). +- Forgetting subprocess auto-instrument tests, or async/streaming coverage. +- Re-recording cassettes when behavior didn't intentionally change. +- Custom `_instrument_*` helper where `_instrument_integration()` fits. +- Missing `target_module` for deep/optional patch targets. +- Inventing new top-level span fields / metric keys / span types (metadata is looser — see allowlist-per-provider). +- Tool definitions in `input` instead of `metadata.tools`; prompt provenance outside `metadata.prompt`. +- Missing `internal={"instrumentation": "-auto"}` on integration spans, or leaking it onto user spans nested inside. +- Framework name in `metadata.provider` for a completion-style framework wrapper. +- Per-chunk spans for streamed calls (should be one accumulated span per API call). +- Instrumentation errors escaping into the user's call path, or swallowing real provider errors. +- Double-counting tokens across orchestration + provider spans, or provider-specific ownership checks instead of clear rules. +- Over-serializing (JSON dumps/loads, recursive conversion) in tracing code. +- `try`/`except` around Braintrust span-logging methods — they're boundary-safe. +- Forcing non-image attachments through `image_url` shims; dropping unrecognized file inputs; re-serializing non-attachment values. +- Patching internal helpers when a stable public API surface would work. From 297e85f07c43a740965f5571fa5f54ea96062e08 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 14:55:40 +0000 Subject: [PATCH 9/9] fix: stamp instrumentation on non-shadow span-open sites Three integrations open child spans through Logger/Span method calls that bypass the module-level `start_span` shadow, so those spans were falling back to `braintrust-python-logger`: - langchain/callbacks.py: `parent_span.start_span(...)` and `init_logger().start_span(...)` in `_start_span` - llamaindex/tracing.py: `parent_bt_span.start_span(...)` in `on_start` - google_genai/tracing.py: `logger._start_span_impl(...)` in the streaming context helper Each site now passes `internal={"instrumentation": _INSTRUMENTATION}` explicitly. Matches the pattern already in place for temporal/plugin.py and openai_agents/tracing.py. Caught by the per-integration span_origin assertions in each integration's test file. --- py/src/braintrust/integrations/google_genai/tracing.py | 1 + py/src/braintrust/integrations/langchain/callbacks.py | 2 ++ py/src/braintrust/integrations/llamaindex/tracing.py | 8 +++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/py/src/braintrust/integrations/google_genai/tracing.py b/py/src/braintrust/integrations/google_genai/tracing.py index 6b8c0382..af75d83d 100644 --- a/py/src/braintrust/integrations/google_genai/tracing.py +++ b/py/src/braintrust/integrations/google_genai/tracing.py @@ -1179,6 +1179,7 @@ def _stream_span_context( input=input, metadata=metadata, lookup_span_parent=False, + internal={"instrumentation": _INSTRUMENTATION}, ) as span: yield span return diff --git a/py/src/braintrust/integrations/langchain/callbacks.py b/py/src/braintrust/integrations/langchain/callbacks.py index 4ca0fdb8..9d9517c9 100644 --- a/py/src/braintrust/integrations/langchain/callbacks.py +++ b/py/src/braintrust/integrations/langchain/callbacks.py @@ -145,6 +145,7 @@ def _start_span( start_time=start_time, set_current=set_current, parent=parent, + internal={"instrumentation": _INSTRUMENTATION}, **event, ) @@ -159,6 +160,7 @@ def _start_span( start_time=start_time, set_current=set_current, parent=parent, + internal={"instrumentation": _INSTRUMENTATION}, **event, ) diff --git a/py/src/braintrust/integrations/llamaindex/tracing.py b/py/src/braintrust/integrations/llamaindex/tracing.py index a91f1ee9..f63a60bb 100644 --- a/py/src/braintrust/integrations/llamaindex/tracing.py +++ b/py/src/braintrust/integrations/llamaindex/tracing.py @@ -214,7 +214,13 @@ def new_span( event["input"] = input_data if parent_bt_span is not None: - bt_span = parent_bt_span.start_span(name=span_name, type=span_type, start_time=start_time, **event) + bt_span = parent_bt_span.start_span( + name=span_name, + type=span_type, + start_time=start_time, + internal={"instrumentation": _INSTRUMENTATION}, + **event, + ) else: bt_span = start_span(name=span_name, type=span_type, start_time=start_time, **event)