From bc6a81b4632a6f70cfee5bc6cb3bbd6e17423e48 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 19:51:32 +0000 Subject: [PATCH 1/4] fix(agno): SKILL-compliance for spans, tools, errors, and multimodal Align the Agno integration with `.agents/skills/sdk-integrations/SKILL.md`. - Pass `Exception` instances to `span.log(error=...)` instead of `str(e)`, matching the anthropic pattern and giving Braintrust a chance to extract structured error info. - Route `tools`, `tool_choice`, `functions`, and `tool_call_limit` into `metadata` rather than `input` via a new `_split_model_call()` helper. Fixes the pre-existing `tool_chocie` typo by construction. - Guard `FunctionCall._build_entrypoint_args()` and attribute reads on `instance.function.*` so extraction failures no longer escape into the user's tool exec path. - Materialize inline `Image` / `Audio` / `Video` / `File` bytes as Braintrust `Attachment`s using `_materialize_attachment` from `integrations/utils.py`. Applied to model input `messages`, non-streaming model output, and both streaming aggregators. Images land under `image_url.url`, other media under `file.file_data` with `filename` preserved, remote `url`s pass through verbatim. - Fix an off-by-check bug in the existing `get_args_kwargs` helper (would IndexError when kwargs-only, then removed since it's now unused). - Extend `test_agno_simple_agent_execution` with a negative assertion that tool-related keys never leak into `input`. - Add `test_agno_agent_tools_metadata_placement` (tools) and `test_agno_agent_image_input_materializes_attachment` (vision) as cassette-backed positive tests. Both call `_skip_if_cassette_missing()` so CI stays green until the cassettes are recorded. Co-Authored-By: Claude Opus 4.7 --- .../braintrust/integrations/agno/test_agno.py | 135 +++++++- .../braintrust/integrations/agno/tracing.py | 302 ++++++++++++++---- 2 files changed, 366 insertions(+), 71 deletions(-) diff --git a/py/src/braintrust/integrations/agno/test_agno.py b/py/src/braintrust/integrations/agno/test_agno.py index 0d17262d..ef857c00 100644 --- a/py/src/braintrust/integrations/agno/test_agno.py +++ b/py/src/braintrust/integrations/agno/test_agno.py @@ -4,13 +4,16 @@ # pyright: reportUnknownParameterType=false # pyright: reportUnknownVariableType=false # pyright: reportUnknownArgumentType=false +import os +from pathlib import Path + import pytest from braintrust import logger from braintrust.integrations.agno import setup_agno from braintrust.integrations.agno import tracing as agno_tracing_module from braintrust.integrations.agno.patchers import wrap_agent, wrap_team from braintrust.integrations.test_utils import verify_autoinstrument_script -from braintrust.logger import start_span +from braintrust.logger import Attachment, start_span from braintrust.test_helpers import init_test_logger from ._test_agno_helpers import ( @@ -37,6 +40,29 @@ def setup_wrapper(): yield +_TOOL_METADATA_ONLY_KEYS = ("tools", "tool_choice", "functions", "tool_call_limit") + + +def _assert_tool_fields_not_in_input(llm_span) -> None: + """Guardrail against regressing the SKILL rule that puts tool definitions in metadata.""" + for forbidden in _TOOL_METADATA_ONLY_KEYS: + assert forbidden not in llm_span["input"], f"{forbidden!r} must live under metadata, not input" + + +def _skip_if_cassette_missing(cassette_name: str) -> None: + """Skip a VCR-marked test when its cassette has not yet been recorded. + + Prevents CI (``record_mode=none``) from failing while a new cassette is + still pending. Record with ``nox -s "test_agno(latest)" -- --vcr-record=all``. + """ + version = os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") + base = Path(__file__).parent / "cassettes" + candidates = [base / version / cassette_name] if version else [] + candidates.append(base / cassette_name) + if not any(p.exists() for p in candidates): + pytest.skip(f"cassette {cassette_name!r} not yet recorded — see re-record command in the test docstring") + + @pytest.mark.vcr def test_agno_simple_agent_execution(memory_logger): agent_module = pytest.importorskip("agno.agent") @@ -97,6 +123,9 @@ def test_agno_simple_agent_execution(memory_logger): assert "librarian" in messages[0]["content"] assert messages[1]["role"] == "user" assert messages[1]["content"] == "Charlotte's Web" + # Tool-related fields must not leak into `input` even when they are absent + # from this particular call — see integrations SKILL "Span Design / Fields". + _assert_tool_fields_not_in_input(llm_span) assert llm_span["output"]["content"] == "E.B. White" assert llm_span["metrics"]["prompt_tokens"] > 0 assert llm_span["metrics"]["completion_tokens"] > 0 @@ -106,6 +135,110 @@ def test_agno_simple_agent_execution(memory_logger): ) +@pytest.mark.vcr +def test_agno_agent_tools_metadata_placement(memory_logger): + """Tool definitions must live in `metadata.tools`, not in the span input. + + Cassette: recorded against the real OpenAI API with a small ``get_weather`` + tool. Re-record with ``nox -s "test_agno(latest)" -- --vcr-record=all -k + "test_agno_agent_tools_metadata_placement"``. + """ + _skip_if_cassette_missing("test_agno_agent_tools_metadata_placement.yaml") + agent_module = pytest.importorskip("agno.agent") + openai_module = pytest.importorskip("agno.models.openai") + Agent = agent_module.Agent + OpenAIChat = openai_module.OpenAIChat + + def get_weather(city: str) -> str: + """Return the current weather for *city*.""" + return f"The weather in {city} is 72F and sunny." + + assert not memory_logger.pop() + + agent = Agent( + name="Weather Agent", + model=OpenAIChat(id="gpt-4o-mini"), + tools=[get_weather], + instructions="Use the get_weather tool to answer questions.", + ) + + response = agent.run("What's the weather in Paris?") + assert response and response.content + + spans = memory_logger.pop() + llm_spans = [s for s in spans if s["span_attributes"]["type"].value == "llm"] + assert llm_spans, "expected at least one llm span" + + for llm_span in llm_spans: + _assert_tool_fields_not_in_input(llm_span) + tools_meta = llm_span["metadata"].get("tools") + assert tools_meta, "expected metadata.tools to be populated on llm spans" + # Agno passes its own tool schema (name / description / parameters) to + # Model.response — spec placement rule is satisfied as long as the tool + # definitions live in metadata, not input. Normalization to the fully + # OpenAI-wrapped `{"type": "function", "function": {...}}` shape is + # tracked separately. + names = {t.get("name") or t.get("function", {}).get("name") for t in tools_meta} + assert "get_weather" in names + + +@pytest.mark.vcr +def test_agno_agent_image_input_materializes_attachment(memory_logger): + """Inline image bytes must be replaced by a Braintrust ``Attachment``. + + Cassette: recorded against the real OpenAI vision API with a 1x1 PNG. + Re-record with ``nox -s "test_agno(latest)" -- --vcr-record=all -k + "test_agno_agent_image_input_materializes_attachment"``. + """ + _skip_if_cassette_missing("test_agno_agent_image_input_materializes_attachment.yaml") + agent_module = pytest.importorskip("agno.agent") + openai_module = pytest.importorskip("agno.models.openai") + media_module = pytest.importorskip("agno.media") + Agent = agent_module.Agent + OpenAIChat = openai_module.OpenAIChat + Image = media_module.Image + + # 1x1 transparent PNG + png_bytes = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR" + b"\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4" + b"\x89\x00\x00\x00\rIDATx\x9cc\xf8\x0f\x00\x00\x01\x01\x00\x05" + b"\x0c\xd6\xa2\r\x00\x00\x00\x00IEND\xaeB`\x82" + ) + + assert not memory_logger.pop() + + agent = Agent( + name="Vision Agent", + model=OpenAIChat(id="gpt-4o-mini"), + instructions="Describe the image in one word.", + ) + + response = agent.run("What's in this image?", images=[Image(content=png_bytes, format="png")]) + assert response and response.content + + spans = memory_logger.pop() + llm_spans = [s for s in spans if s["span_attributes"]["type"].value == "llm"] + assert llm_spans + + def _find_attachments(obj): + found = [] + if isinstance(obj, Attachment): + found.append(obj) + elif isinstance(obj, dict): + for v in obj.values(): + found.extend(_find_attachments(v)) + elif isinstance(obj, list): + for v in obj: + found.extend(_find_attachments(v)) + return found + + attachments = _find_attachments(llm_spans[0]["input"]) + assert attachments, "expected image bytes to be materialized as an Attachment" + att = attachments[0] + assert att.reference["content_type"].startswith("image/") + + def test_get_model_name_prefers_stable_provider_attribute(): class FakeModel: provider = "OpenAI" diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py index 5c8983d4..82dbfac6 100644 --- a/py/src/braintrust/integrations/agno/tracing.py +++ b/py/src/braintrust/integrations/agno/tracing.py @@ -2,7 +2,7 @@ from inspect import isawaitable from typing import Any -from braintrust.integrations.utils import _try_to_dict +from braintrust.integrations.utils import _materialize_attachment, _try_to_dict from braintrust.logger import start_span as _bt_start_span @@ -33,8 +33,46 @@ def clean(obj: dict[str, Any]) -> dict[str, Any]: return {k: v for k, v in obj.items() if v is not None} -def get_args_kwargs(args: list[str], kwargs: dict[str, Any], keys: list[str]): - return {k: args[i] if args else kwargs.get(k) for i, k in enumerate(keys)}, omit(kwargs, keys) +# Keys the SDK-integrations spec routes into metadata rather than the span input. +_MODEL_METADATA_KEYS = ("tools", "tool_choice", "functions", "tool_call_limit") + + +def _split_model_call( + args: tuple, kwargs: dict[str, Any], positional_order: list[str] +) -> tuple[dict[str, Any], dict[str, Any]]: + """Bind positional args to their names and split into (input, metadata_extras). + + Tool definitions, tool_choice, function schemas and tool-call limits go + into metadata per the integrations spec; everything else that names a + request field lands in input. Pure — no materialization side effects. + """ + combined: dict[str, Any] = dict(kwargs) + for i, key in enumerate(positional_order): + if i < len(args): + combined[key] = args[i] + metadata_extras: dict[str, Any] = {} + for k in _MODEL_METADATA_KEYS: + if k in combined: + metadata_extras[k] = combined.pop(k) + return combined, metadata_extras + + +def _prepare_model_input(input_data: dict[str, Any]) -> dict[str, Any]: + """Materialize inline media in an Agno request-input dict before logging.""" + if "messages" in input_data: + return {**input_data, "messages": _materialize_agno_messages(input_data["messages"])} + return input_data + + +def _prepare_model_output(result: Any) -> Any: + """Convert an Agno model response to a materialized dict; pass through on failure.""" + try: + as_dict = _try_to_dict(result) + except Exception: + return result + if not isinstance(as_dict, dict): + return result + return _materialize_agno_output_media(dict(as_dict)) def is_sync_iterator(result: Any) -> bool: @@ -45,6 +83,126 @@ def is_async_iterator(result: Any) -> bool: return hasattr(result, "__aiter__") and hasattr(result, "__anext__") +# --------------------------------------------------------------------------- +# Multimodal materialization +# --------------------------------------------------------------------------- +# +# Agno's Image / Audio / Video / File objects carry inline bytes on ``content`` +# (plus optional ``url`` / ``filepath`` / ``mime_type`` / ``format`` / ``filename``). +# The integrations spec requires inline binary media to be converted to +# Braintrust ``Attachment`` objects at the leaf position; remote URLs stay as +# strings and unrecognized shapes pass through unchanged. + + +def _agno_media_mime_type(as_dict: dict[str, Any], media_kind: str) -> str | None: + mime = as_dict.get("mime_type") + if isinstance(mime, str) and mime: + return mime + fmt = as_dict.get("format") + if isinstance(fmt, str) and fmt: + return f"{media_kind}/{fmt}" + return None + + +def _materialize_agno_media(value: Any, media_kind: str) -> Any: + """Materialize a single Agno media object to a serializable dict. + + Returns the input unchanged when it is not a recognizable Agno media object + or when materialization is not possible. + """ + if value is None: + return value + + as_dict = value if isinstance(value, dict) else _try_to_dict(value) + if not isinstance(as_dict, dict): + return value + + content = as_dict.get("content") + filepath = as_dict.get("filepath") + url = as_dict.get("url") + filename = as_dict.get("filename") + mime_type = _agno_media_mime_type(as_dict, media_kind) + + raw: Any = content if isinstance(content, (bytes, bytearray, str)) and content else None + if raw is None and isinstance(filepath, str) and filepath: + raw = filepath + + resolved = None + if raw is not None: + try: + resolved = _materialize_attachment( + raw, + mime_type=mime_type, + filename=filename if isinstance(filename, str) else None, + label=media_kind, + prefix=media_kind, + ) + except Exception: + resolved = None + + result: dict[str, Any] = {k: v for k, v in as_dict.items() if k not in ("content", "filepath")} + if resolved is not None: + result.update(resolved.multimodal_part_payload) + elif isinstance(url, str) and url and "url" not in result: + # Preserve remote URL references verbatim per the spec. + result["url"] = url + return result + + +def _materialize_agno_media_list(value: Any, media_kind: str) -> Any: + if not isinstance(value, list): + return value + return [_materialize_agno_media(v, media_kind) for v in value] + + +_AGNO_MESSAGE_MEDIA_FIELDS = ( + ("images", "image"), + ("image_output", "image"), + ("audio", "audio"), + ("audio_output", "audio"), + ("videos", "video"), + ("video_output", "video"), + ("files", "file"), + ("file_output", "file"), +) + + +def _materialize_agno_message(msg: Any) -> Any: + """Return a dict form of an Agno Message with inline media replaced by attachments.""" + as_dict = msg if isinstance(msg, dict) else _try_to_dict(msg) + if not isinstance(as_dict, dict): + return msg + result = dict(as_dict) + for field, kind in _AGNO_MESSAGE_MEDIA_FIELDS: + if field in result and result[field]: + if isinstance(result[field], list): + result[field] = [_materialize_agno_media(v, kind) for v in result[field]] + else: + result[field] = _materialize_agno_media(result[field], kind) + return result + + +def _materialize_agno_messages(messages: Any) -> Any: + if not isinstance(messages, list): + return messages + return [_materialize_agno_message(m) for m in messages] + + +def _materialize_agno_output_media(aggregated: dict[str, Any]) -> dict[str, Any]: + """Materialize model-response-style media fields in an aggregated dict in place.""" + for field, kind in (("images", "image"), ("videos", "video"), ("files", "file")): + val = aggregated.get(field) + if isinstance(val, list) and val: + aggregated[field] = [_materialize_agno_media(v, kind) for v in val] + audio_val = aggregated.get("audio") + if audio_val is not None: + if isinstance(audio_val, list): + aggregated["audio"] = [_materialize_agno_media(v, "audio") for v in audio_val] + else: + aggregated["audio"] = _materialize_agno_media(audio_val, "audio") + return aggregated + + # --------------------------------------------------------------------------- # Metrics mapping & extraction # --------------------------------------------------------------------------- @@ -207,7 +365,7 @@ def _aggregate_model_chunks(chunks: list[Any]) -> dict[str, Any]: else: aggregated["metrics"] = None - return aggregated + return _materialize_agno_output_media(aggregated) def _aggregate_response_stream_chunks(chunks: list[Any]) -> dict[str, Any]: @@ -267,7 +425,7 @@ def _aggregate_response_stream_chunks(chunks: list[Any]) -> dict[str, Any]: else: aggregated["metrics"] = None - return aggregated + return _materialize_agno_output_media(aggregated) def _aggregate_agent_chunks(chunks: list[Any]) -> dict[str, Any]: @@ -393,7 +551,7 @@ def _inner(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -421,7 +579,7 @@ async def _inner(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -506,7 +664,7 @@ def _trace_stream(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -547,7 +705,7 @@ async def _trace_stream(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -622,7 +780,7 @@ def _trace_stream(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -663,7 +821,7 @@ async def _trace_stream(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -710,7 +868,7 @@ def _run_public_dispatch_wrapper( span.end() return result except Exception as e: - span.log(error=str(e)) + span.log(error=e) span.unset_current() span.end() raise @@ -754,7 +912,7 @@ async def _trace_awaitable(): span.log(output=awaited, metrics=extract_metrics(awaited)) return awaited except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_end_span: @@ -771,7 +929,7 @@ async def _trace_awaitable(): span.end() return result except Exception as e: - span.log(error=str(e)) + span.log(error=e) span.unset_current() span.end() raise @@ -817,41 +975,38 @@ def _get_model_name(instance: Any) -> str: def _model_invoke_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): model_name = _get_model_name(instance) - input, clean_kwargs = get_args_kwargs( - args, kwargs, ["assistant_message", "messages", "response_format", "tools", "tool_choice"] - ) + input, metadata_extras = _split_model_call(args, kwargs, ["assistant_message", "messages"]) + input = _prepare_model_input(input) with start_span( name=f"{model_name}.invoke", type=SpanTypeAttribute.LLM, input=input, - metadata={**clean_kwargs, **extract_metadata(instance, "model")}, + metadata={**metadata_extras, **extract_metadata(instance, "model")}, ) as span: result = wrapped(*args, **kwargs) - span.log(output=result, metrics=extract_metrics(result, kwargs.get("messages", []))) + span.log(output=_prepare_model_output(result), metrics=extract_metrics(result, kwargs.get("messages", []))) return result async def _model_ainvoke_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): model_name = _get_model_name(instance) - input, clean_kwargs = get_args_kwargs( - args, kwargs, ["messages", "assistant_message", "response_format", "tools", "tool_choice"] - ) + input, metadata_extras = _split_model_call(args, kwargs, ["messages", "assistant_message"]) + input = _prepare_model_input(input) with start_span( name=f"{model_name}.ainvoke", type=SpanTypeAttribute.LLM, input=input, - metadata={**clean_kwargs, **extract_metadata(instance, "model")}, + metadata={**metadata_extras, **extract_metadata(instance, "model")}, ) as span: result = await wrapped(*args, **kwargs) - span.log(output=result, metrics=extract_metrics(result, kwargs.get("messages", []))) + span.log(output=_prepare_model_output(result), metrics=extract_metrics(result, kwargs.get("messages", []))) return result def _model_invoke_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): model_name = _get_model_name(instance) - input, clean_kwargs = get_args_kwargs( - args, kwargs, ["messages", "assistant_messages", "response_format", "tools", "tool_choice"] - ) + input, metadata_extras = _split_model_call(args, kwargs, ["messages", "assistant_messages"]) + input = _prepare_model_input(input) def _trace_stream(): start = time.time() @@ -859,7 +1014,7 @@ def _trace_stream(): name=f"{model_name}.invoke_stream", type=SpanTypeAttribute.LLM, input=input, - metadata={**clean_kwargs, **extract_metadata(instance, "model")}, + metadata={**metadata_extras, **extract_metadata(instance, "model")}, ) as span: first = True collected_chunks = [] @@ -877,9 +1032,8 @@ def _trace_stream(): def _model_ainvoke_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): model_name = _get_model_name(instance) - input, clean_kwargs = get_args_kwargs( - args, kwargs, ["messages", "assistant_messages", "response_format", "tools", "tool_choice"] - ) + input, metadata_extras = _split_model_call(args, kwargs, ["messages", "assistant_messages"]) + input = _prepare_model_input(input) async def _trace_astream(): start = time.time() @@ -887,7 +1041,7 @@ async def _trace_astream(): name=f"{model_name}.ainvoke_stream", type=SpanTypeAttribute.LLM, input=input, - metadata={**clean_kwargs, **extract_metadata(instance, "model")}, + metadata={**metadata_extras, **extract_metadata(instance, "model")}, ) as span: first = True collected_chunks = [] @@ -905,41 +1059,38 @@ async def _trace_astream(): def _model_response_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): model_name = _get_model_name(instance) - input, clean_kwargs = get_args_kwargs( - args, kwargs, ["messages", "response_format", "tools", "functions", "tool_chocie", "tool_call_limit"] - ) + input, metadata_extras = _split_model_call(args, kwargs, ["messages"]) + input = _prepare_model_input(input) with start_span( name=f"{model_name}.response", type=SpanTypeAttribute.LLM, input=input, - metadata={**clean_kwargs, **extract_metadata(instance, "model")}, + metadata={**metadata_extras, **extract_metadata(instance, "model")}, ) as span: result = wrapped(*args, **kwargs) - span.log(output=result, metrics=extract_metrics(result, kwargs.get("messages", []))) + span.log(output=_prepare_model_output(result), metrics=extract_metrics(result, kwargs.get("messages", []))) return result async def _model_aresponse_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): model_name = _get_model_name(instance) - input, clean_kwargs = get_args_kwargs( - args, kwargs, ["messages", "response_format", "tools", "functions", "tool_chocie", "tool_call_limit"] - ) + input, metadata_extras = _split_model_call(args, kwargs, ["messages"]) + input = _prepare_model_input(input) with start_span( name=f"{model_name}.aresponse", type=SpanTypeAttribute.LLM, input=input, - metadata={**clean_kwargs, **extract_metadata(instance, "model")}, + metadata={**metadata_extras, **extract_metadata(instance, "model")}, ) as span: result = await wrapped(*args, **kwargs) - span.log(output=result, metrics=extract_metrics(result, kwargs.get("messages", []))) + span.log(output=_prepare_model_output(result), metrics=extract_metrics(result, kwargs.get("messages", []))) return result def _model_response_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): model_name = _get_model_name(instance) - input, clean_kwargs = get_args_kwargs( - args, kwargs, ["messages", "response_format", "tools", "functions", "tool_chocie", "tool_call_limit"] - ) + input, metadata_extras = _split_model_call(args, kwargs, ["messages"]) + input = _prepare_model_input(input) def _trace_stream(): start = time.time() @@ -947,7 +1098,7 @@ def _trace_stream(): name=f"{model_name}.response_stream", type=SpanTypeAttribute.LLM, input=input, - metadata={**clean_kwargs, **extract_metadata(instance, "model")}, + metadata={**metadata_extras, **extract_metadata(instance, "model")}, ) as span: first = True collected_chunks = [] @@ -965,9 +1116,8 @@ def _trace_stream(): def _model_aresponse_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): model_name = _get_model_name(instance) - input, clean_kwargs = get_args_kwargs( - args, kwargs, ["messages", "response_format", "tools", "functions", "tool_chocie", "tool_call_limit"] - ) + input, metadata_extras = _split_model_call(args, kwargs, ["messages"]) + input = _prepare_model_input(input) async def _trace_astream(): start = time.time() @@ -975,7 +1125,7 @@ async def _trace_astream(): name=f"{model_name}.aresponse_stream", type=SpanTypeAttribute.LLM, input=input, - metadata={**clean_kwargs, **extract_metadata(instance, "model")}, + metadata={**metadata_extras, **extract_metadata(instance, "model")}, ) as span: first = True collected_chunks = [] @@ -1002,18 +1152,34 @@ def _get_function_name(instance) -> str: return "Unknown" +def _function_call_metadata(instance: Any) -> dict[str, Any]: + """Best-effort metadata extraction for a FunctionCall. Contains instrumentation errors.""" + metadata: dict[str, Any] = {} + try: + metadata["name"] = instance.function.name + except Exception: + pass + try: + metadata["entrypoint"] = instance.function.entrypoint.__name__ + except Exception: + pass + try: + entrypoint_args = instance._build_entrypoint_args() + if entrypoint_args: + metadata.update(entrypoint_args) + except Exception: + pass + return metadata + + def _function_call_execute_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): function_name = _get_function_name(instance) - entrypoint_args = instance._build_entrypoint_args() + metadata = _function_call_metadata(instance) with start_span( name=f"{function_name}.execute", type=SpanTypeAttribute.TOOL, - input=(instance.arguments or {}), - metadata={ - "name": instance.function.name, - "entrypoint": instance.function.entrypoint.__name__, - **(entrypoint_args or {}), - }, + input=(getattr(instance, "arguments", None) or {}), + metadata=metadata, ) as span: result = wrapped(*args, **kwargs) span.log(output=result) @@ -1022,16 +1188,12 @@ def _function_call_execute_wrapper(wrapped: Any, instance: Any, args: Any, kwarg async def _function_call_aexecute_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): function_name = _get_function_name(instance) - entrypoint_args = instance._build_entrypoint_args() + metadata = _function_call_metadata(instance) with start_span( name=f"{function_name}.aexecute", type=SpanTypeAttribute.TOOL, - input=(instance.arguments or {}), - metadata={ - "name": instance.function.name, - "entrypoint": instance.function.entrypoint.__name__, - **(entrypoint_args or {}), - }, + input=(getattr(instance, "arguments", None) or {}), + metadata=metadata, ) as span: result = await wrapped(*args, **kwargs) span.log(output=result) @@ -1125,7 +1287,7 @@ def _trace_stream(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -1183,7 +1345,7 @@ async def _trace_stream(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -1230,7 +1392,7 @@ def _trace_stream(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -1244,7 +1406,7 @@ def _trace_stream(): span.end() return result except Exception as e: - span.log(error=str(e)) + span.log(error=e) span.unset_current() span.end() raise @@ -1287,7 +1449,7 @@ async def _trace_stream(): should_unset = False raise except Exception as e: - span.log(error=str(e)) + span.log(error=e) raise finally: if should_unset: @@ -1301,7 +1463,7 @@ async def _trace_stream(): span.end() return result except Exception as e: - span.log(error=str(e)) + span.log(error=e) span.unset_current() span.end() raise From ecd26a6e304e8253a6e92d4de7f2bfd2beca25ef Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 16 Jul 2026 17:04:06 -0400 Subject: [PATCH 2/4] record cassettes --- py/noxfile.py | 3 +- py/src/braintrust/conftest.py | 21 ++ ...t_image_input_materializes_attachment.yaml | 160 +++++++++++ ...t_agno_agent_tools_metadata_placement.yaml | 264 ++++++++++++++++++ ...t_image_input_materializes_attachment.yaml | 157 +++++++++++ ...t_agno_agent_tools_metadata_placement.yaml | 260 +++++++++++++++++ ...t_image_input_materializes_attachment.yaml | 157 +++++++++++ ...t_agno_agent_tools_metadata_placement.yaml | 260 +++++++++++++++++ .../braintrust/integrations/agno/test_agno.py | 23 +- py/src/braintrust/test_vcr_config.py | 19 ++ 10 files changed, 1301 insertions(+), 23 deletions(-) create mode 100644 py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_image_input_materializes_attachment.yaml create mode 100644 py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_tools_metadata_placement.yaml create mode 100644 py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_image_input_materializes_attachment.yaml create mode 100644 py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_tools_metadata_placement.yaml create mode 100644 py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_image_input_materializes_attachment.yaml create mode 100644 py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_tools_metadata_placement.yaml create mode 100644 py/src/braintrust/test_vcr_config.py diff --git a/py/noxfile.py b/py/noxfile.py index 1eb9b8b8..68df96cf 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -348,8 +348,7 @@ def test_agno(session, version): _install_test_deps(session) _install_matrix_dep(session, "agno", version) _install_group_locked(session, "test-agno") - _run_tests(session, f"{INTEGRATION_DIR}/agno/test_agno.py", version=version) - _run_tests(session, f"{INTEGRATION_DIR}/agno/test_workflow.py", version=version) + _run_tests(session, f"{INTEGRATION_DIR}/agno", version=version) LIVEKIT_AGENTS_VERSIONS = _get_matrix_versions("livekit-agents") diff --git a/py/src/braintrust/conftest.py b/py/src/braintrust/conftest.py index bc082ee1..bfeb582d 100644 --- a/py/src/braintrust/conftest.py +++ b/py/src/braintrust/conftest.py @@ -232,6 +232,23 @@ def skip_vcr_tests_in_wheel_mode(request): pytest.skip("VCR tests skipped in wheel mode (pre-release sanity check only)") +_SENSITIVE_RESPONSE_HEADERS = { + "openai-organization", + "openai-project", + "set-cookie", +} + + +def _scrub_sensitive_response_headers(response): + """Remove sensitive provider headers before writing a cassette.""" + response["headers"] = { + name: value + for name, value in response.get("headers", {}).items() + if name.lower() not in _SENSITIVE_RESPONSE_HEADERS + } + return response + + def get_vcr_config(): """ Get VCR configuration for recording/playing back HTTP interactions. @@ -247,6 +264,9 @@ def get_vcr_config(): "authorization", "Authorization", "openai-organization", + "openai-project", + "cookie", + "Cookie", "x-api-key", "api-key", "openai-api-key", @@ -259,6 +279,7 @@ def get_vcr_config(): "amz-sdk-request", "x-amzn-bedrock-api-key", ], + "before_record_response": _scrub_sensitive_response_headers, } diff --git a/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_image_input_materializes_attachment.yaml new file mode 100644 index 00000000..f7a96405 --- /dev/null +++ b/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_image_input_materializes_attachment.yaml @@ -0,0 +1,160 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"\nDescribe the + image in one word.\n"},{"role":"user","content":[{"type":"text","text":"What''s + in this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '371' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MZDFm1JBErEB3lbW8PJBHrj49Ve\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232111,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Red.\",\n \"refusal\": null,\n + \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": + \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 8529,\n \"completion_tokens\": + 2,\n \"total_tokens\": 8531,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_844ddd95ff\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c399e609df4fcb-YYZ + connection: + - keep-alive + content-length: + - '814' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:01:51 GMT + openai-processing-ms: + - '556' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999210' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_5d175e2325594ca8bb63ef1d92003b07 + status: + code: 200 + message: OK +- request: + body: '{"session_id":"5b14603b-835b-4837-9b92-3006800a5515","run_id":"5ddac676-8ef5-48a5-a25a-ed78dda4ce84","data":{"agent_id":"vision-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.1.0","type":"agent"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '453' + content-type: + - application/json + host: + - os-api.agno.com + user-agent: + - agno/2.1.0 + method: POST + uri: https://os-api.agno.com/telemetry/runs + response: + body: + string: '{"message":"Run creation acknowledged: 5ddac676-8ef5-48a5-a25a-ed78dda4ce84","status":"success"}' + headers: + connection: + - keep-alive + content-length: + - '96' + content-type: + - application/json + cross-origin-opener-policy: + - same-origin + cross-origin-resource-policy: + - same-origin + date: + - Thu, 16 Jul 2026 20:01:51 GMT + permissions-policy: + - camera=(), microphone=(), geolocation=() + referrer-policy: + - strict-origin-when-cross-origin + server: + - uvicorn + strict-transport-security: + - max-age=31536000; includeSubDomains + x-content-type-options: + - nosniff + x-frame-options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_tools_metadata_placement.yaml b/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_tools_metadata_placement.yaml new file mode 100644 index 00000000..1ed1ada6 --- /dev/null +++ b/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_tools_metadata_placement.yaml @@ -0,0 +1,264 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"\nUse the get_weather + tool to answer questions.\n"},{"role":"user","content":"What''s + the weather in Paris?"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return + the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '415' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MZAIzZ9MlaKi6phZddEyg7dgNRF\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232108,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_gCgs7fNBAzJefgRzrceNF1hO\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"city\\\":\\\"Paris\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null,\n \"annotations\": + []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": + 14,\n \"total_tokens\": 81,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c399d478447aa8-YYZ + connection: + - keep-alive + content-length: + - '1084' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:01:49 GMT + openai-processing-ms: + - '809' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999972' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_7b66a14062ff44209bb87379ae3b7179 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"developer","content":"\nUse the get_weather + tool to answer questions.\n"},{"role":"user","content":"What''s + the weather in Paris?"},{"role":"assistant","tool_calls":[{"id":"call_gCgs7fNBAzJefgRzrceNF1hO","function":{"arguments":"{\"city\":\"Paris\"}","name":"get_weather"},"type":"function"}],"content":""},{"role":"tool","content":"The + weather in Paris is 72F and sunny.","tool_call_id":"call_gCgs7fNBAzJefgRzrceNF1hO"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return + the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '704' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MZCvEGRddz5DBWH94zascdX5PvU\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232110,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"The weather in Paris is currently 72\xB0F + and sunny.\",\n \"refusal\": null,\n \"annotations\": []\n },\n + \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n + \ \"usage\": {\n \"prompt_tokens\": 100,\n \"completion_tokens\": 13,\n + \ \"total_tokens\": 113,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c399dad9605905-YYZ + connection: + - keep-alive + content-length: + - '859' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:01:50 GMT + openai-processing-ms: + - '497' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999960' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_13f2423c790c4aaea15e040f49bb1851 + status: + code: 200 + message: OK +- request: + body: '{"session_id":"cd7eb49c-b210-4bb6-a6a5-830746dc7f9c","run_id":"a05b11bb-1edb-4fb8-8d5c-fc9e8cb26fc1","data":{"agent_id":"weather-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.1.0","type":"agent"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '454' + content-type: + - application/json + host: + - os-api.agno.com + user-agent: + - agno/2.1.0 + method: POST + uri: https://os-api.agno.com/telemetry/runs + response: + body: + string: '{"message":"Run creation acknowledged: a05b11bb-1edb-4fb8-8d5c-fc9e8cb26fc1","status":"success"}' + headers: + connection: + - keep-alive + content-length: + - '96' + content-type: + - application/json + cross-origin-opener-policy: + - same-origin + cross-origin-resource-policy: + - same-origin + date: + - Thu, 16 Jul 2026 20:01:50 GMT + permissions-policy: + - camera=(), microphone=(), geolocation=() + referrer-policy: + - strict-origin-when-cross-origin + server: + - uvicorn + strict-transport-security: + - max-age=31536000; includeSubDomains + x-content-type-options: + - nosniff + x-frame-options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_image_input_materializes_attachment.yaml new file mode 100644 index 00000000..3f6c27cc --- /dev/null +++ b/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_image_input_materializes_attachment.yaml @@ -0,0 +1,157 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"Describe the image in one word."},{"role":"user","content":[{"type":"text","text":"What''s + in this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '338' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MYzcSH1UCsHzU3QNzHnK9FIZGz7\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232097,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Red.\",\n \"refusal\": null,\n + \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": + \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 8523,\n \"completion_tokens\": + 2,\n \"total_tokens\": 8525,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_844ddd95ff\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c3998eee36abfa-YYZ + connection: + - keep-alive + content-length: + - '814' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:01:38 GMT + openai-processing-ms: + - '1308' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999217' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_22335d6856c84be48a7eef8157caf80f + status: + code: 200 + message: OK +- request: + body: '{"session_id":"e81647f1-acc6-4572-8d0b-fd07effd7004","run_id":"40baf7fc-38c7-40d4-bc26-d6037d6fac53","data":{"agent_id":"vision-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_learnings":false,"has_culture":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.4.0","type":"agent"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '495' + content-type: + - application/json + host: + - os-api.agno.com + user-agent: + - agno/2.4.0 + method: POST + uri: https://os-api.agno.com/telemetry/runs + response: + body: + string: '{"message":"Run creation acknowledged: 40baf7fc-38c7-40d4-bc26-d6037d6fac53","status":"success"}' + headers: + content-length: + - '96' + content-type: + - application/json + cross-origin-opener-policy: + - same-origin + cross-origin-resource-policy: + - same-origin + date: + - Thu, 16 Jul 2026 20:01:38 GMT + permissions-policy: + - camera=(), microphone=(), geolocation=() + referrer-policy: + - strict-origin-when-cross-origin + server: + - uvicorn + strict-transport-security: + - max-age=31536000; includeSubDomains + x-content-type-options: + - nosniff + x-frame-options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_tools_metadata_placement.yaml b/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_tools_metadata_placement.yaml new file mode 100644 index 00000000..c1946035 --- /dev/null +++ b/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_tools_metadata_placement.yaml @@ -0,0 +1,260 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"Use the get_weather tool to + answer questions."},{"role":"user","content":"What''s the weather in Paris?"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return + the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '382' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MYxdFisKNQJUJ7lGomeZM7dw5rI\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232095,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_MI4n3R3AoYrD6OFXbH0rgiAJ\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"city\\\":\\\"Paris\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null,\n \"annotations\": + []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 61,\n \"completion_tokens\": + 14,\n \"total_tokens\": 75,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_a8eb499e0d\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c39983bb2fabfa-YYZ + connection: + - keep-alive + content-length: + - '1084' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:01:36 GMT + openai-processing-ms: + - '570' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999977' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_d03570245a304b549f93a99746f116d7 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"developer","content":"Use the get_weather tool to + answer questions."},{"role":"user","content":"What''s the weather in Paris?"},{"role":"assistant","tool_calls":[{"id":"call_MI4n3R3AoYrD6OFXbH0rgiAJ","function":{"arguments":"{\"city\":\"Paris\"}","name":"get_weather"},"type":"function"}],"content":""},{"role":"tool","content":"The + weather in Paris is 72F and sunny.","tool_call_id":"call_MI4n3R3AoYrD6OFXbH0rgiAJ"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return + the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '671' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MYyjad1qXH8NCzbKr1uRa6UN1p6\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232096,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"The weather in Paris is 72\xB0F and + sunny.\",\n \"refusal\": null,\n \"annotations\": []\n },\n + \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n + \ \"usage\": {\n \"prompt_tokens\": 94,\n \"completion_tokens\": 12,\n + \ \"total_tokens\": 106,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_a8eb499e0d\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c39988afa9abfa-YYZ + connection: + - keep-alive + content-length: + - '848' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:01:36 GMT + openai-processing-ms: + - '604' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999967' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_110f9383dbbf42b4a35da0f4320d5379 + status: + code: 200 + message: OK +- request: + body: '{"session_id":"ca380d3f-1290-49d1-8fd1-4234f24d61b4","run_id":"33f187ef-d597-4be6-b623-dc76ce1005b9","data":{"agent_id":"weather-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_learnings":false,"has_culture":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.4.0","type":"agent"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '496' + content-type: + - application/json + host: + - os-api.agno.com + user-agent: + - agno/2.4.0 + method: POST + uri: https://os-api.agno.com/telemetry/runs + response: + body: + string: '{"message":"Run creation acknowledged: 33f187ef-d597-4be6-b623-dc76ce1005b9","status":"success"}' + headers: + content-length: + - '96' + content-type: + - application/json + cross-origin-opener-policy: + - same-origin + cross-origin-resource-policy: + - same-origin + date: + - Thu, 16 Jul 2026 20:01:37 GMT + permissions-policy: + - camera=(), microphone=(), geolocation=() + referrer-policy: + - strict-origin-when-cross-origin + server: + - uvicorn + strict-transport-security: + - max-age=31536000; includeSubDomains + x-content-type-options: + - nosniff + x-frame-options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_image_input_materializes_attachment.yaml new file mode 100644 index 00000000..739035da --- /dev/null +++ b/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_image_input_materializes_attachment.yaml @@ -0,0 +1,157 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"Describe the image in one word."},{"role":"user","content":[{"type":"text","text":"What''s + in this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '338' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MYEAGGbKfSph0LSwpPHjkGelm1k\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232050,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Red.\",\n \"refusal\": null,\n + \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": + \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 8523,\n \"completion_tokens\": + 2,\n \"total_tokens\": 8525,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_844ddd95ff\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c3986d3cebab5a-YYZ + connection: + - keep-alive + content-length: + - '814' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:00:51 GMT + openai-processing-ms: + - '486' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999217' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_36660cffe8ba4ddcb3b3b549e5bb4303 + status: + code: 200 + message: OK +- request: + body: '{"session_id":"196de1f1-cfd3-42c0-9b9b-f8d63e9c66e9","run_id":"c4a4ab9b-191d-4f14-93dc-ece642e77d7e","data":{"agent_id":"vision-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_learnings":false,"has_culture":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.7.2","type":"agent"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '495' + content-type: + - application/json + host: + - os-api.agno.com + user-agent: + - agno/2.7.2 + method: POST + uri: https://os-api.agno.com/telemetry/runs + response: + body: + string: '{"message":"Run creation acknowledged: c4a4ab9b-191d-4f14-93dc-ece642e77d7e","status":"success"}' + headers: + content-length: + - '96' + content-type: + - application/json + cross-origin-opener-policy: + - same-origin + cross-origin-resource-policy: + - same-origin + date: + - Thu, 16 Jul 2026 20:00:51 GMT + permissions-policy: + - camera=(), microphone=(), geolocation=() + referrer-policy: + - strict-origin-when-cross-origin + server: + - uvicorn + strict-transport-security: + - max-age=31536000; includeSubDomains + x-content-type-options: + - nosniff + x-frame-options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_tools_metadata_placement.yaml b/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_tools_metadata_placement.yaml new file mode 100644 index 00000000..33216517 --- /dev/null +++ b/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_tools_metadata_placement.yaml @@ -0,0 +1,260 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"Use the get_weather tool to + answer questions."},{"role":"user","content":"What''s the weather in Paris?"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return + the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '382' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MXTQrN5sd3ptBLIdUmFnHFC6SNv\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232003,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_RbqZ3KBymj7NFJLQ5XzZPu5p\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n + \ \"arguments\": \"{\\\"city\\\":\\\"Paris\\\"}\"\n }\n + \ }\n ],\n \"refusal\": null,\n \"annotations\": + []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 61,\n \"completion_tokens\": + 14,\n \"total_tokens\": 75,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_a8eb499e0d\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c39742cd5bab88-YYZ + connection: + - keep-alive + content-length: + - '1084' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:00:04 GMT + openai-processing-ms: + - '575' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999977' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_bc94e6d110b74df8a2bc2d1d53a7f413 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"developer","content":"Use the get_weather tool to + answer questions."},{"role":"user","content":"What''s the weather in Paris?"},{"role":"assistant","tool_calls":[{"id":"call_RbqZ3KBymj7NFJLQ5XzZPu5p","function":{"arguments":"{\"city\":\"Paris\"}","name":"get_weather"},"type":"function"}],"content":""},{"role":"tool","content":"The + weather in Paris is 72F and sunny.","tool_call_id":"call_RbqZ3KBymj7NFJLQ5XzZPu5p"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return + the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '671' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.31.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.31.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-E2MXU1FvuDn3iIDaUdTWPX4FReNil\",\n \"object\": + \"chat.completion\",\n \"created\": 1784232004,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"The weather in Paris is 72\xB0F and + sunny.\",\n \"refusal\": null,\n \"annotations\": []\n },\n + \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n + \ \"usage\": {\n \"prompt_tokens\": 94,\n \"completion_tokens\": 12,\n + \ \"total_tokens\": 106,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_a8eb499e0d\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1c3974aac4fab88-YYZ + connection: + - keep-alive + content-length: + - '848' + content-type: + - application/json + date: + - Thu, 16 Jul 2026 20:00:04 GMT + openai-processing-ms: + - '575' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999967' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_30cc951acc2b4e088ea653348b4cf3c8 + status: + code: 200 + message: OK +- request: + body: '{"session_id":"579034d6-9325-41eb-805d-831364c2680f","run_id":"5525f7ad-894b-4b39-b816-ef3809903d29","data":{"agent_id":"weather-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_learnings":false,"has_culture":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.7.2","type":"agent"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '496' + content-type: + - application/json + host: + - os-api.agno.com + user-agent: + - agno/2.7.2 + method: POST + uri: https://os-api.agno.com/telemetry/runs + response: + body: + string: '{"message":"Run creation acknowledged: 5525f7ad-894b-4b39-b816-ef3809903d29","status":"success"}' + headers: + content-length: + - '96' + content-type: + - application/json + cross-origin-opener-policy: + - same-origin + cross-origin-resource-policy: + - same-origin + date: + - Thu, 16 Jul 2026 20:00:05 GMT + permissions-policy: + - camera=(), microphone=(), geolocation=() + referrer-policy: + - strict-origin-when-cross-origin + server: + - uvicorn + strict-transport-security: + - max-age=31536000; includeSubDomains + x-content-type-options: + - nosniff + x-frame-options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/py/src/braintrust/integrations/agno/test_agno.py b/py/src/braintrust/integrations/agno/test_agno.py index ef857c00..dd22859d 100644 --- a/py/src/braintrust/integrations/agno/test_agno.py +++ b/py/src/braintrust/integrations/agno/test_agno.py @@ -4,9 +4,6 @@ # pyright: reportUnknownParameterType=false # pyright: reportUnknownVariableType=false # pyright: reportUnknownArgumentType=false -import os -from pathlib import Path - import pytest from braintrust import logger from braintrust.integrations.agno import setup_agno @@ -49,20 +46,6 @@ def _assert_tool_fields_not_in_input(llm_span) -> None: assert forbidden not in llm_span["input"], f"{forbidden!r} must live under metadata, not input" -def _skip_if_cassette_missing(cassette_name: str) -> None: - """Skip a VCR-marked test when its cassette has not yet been recorded. - - Prevents CI (``record_mode=none``) from failing while a new cassette is - still pending. Record with ``nox -s "test_agno(latest)" -- --vcr-record=all``. - """ - version = os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") - base = Path(__file__).parent / "cassettes" - candidates = [base / version / cassette_name] if version else [] - candidates.append(base / cassette_name) - if not any(p.exists() for p in candidates): - pytest.skip(f"cassette {cassette_name!r} not yet recorded — see re-record command in the test docstring") - - @pytest.mark.vcr def test_agno_simple_agent_execution(memory_logger): agent_module = pytest.importorskip("agno.agent") @@ -143,7 +126,6 @@ def test_agno_agent_tools_metadata_placement(memory_logger): tool. Re-record with ``nox -s "test_agno(latest)" -- --vcr-record=all -k "test_agno_agent_tools_metadata_placement"``. """ - _skip_if_cassette_missing("test_agno_agent_tools_metadata_placement.yaml") agent_module = pytest.importorskip("agno.agent") openai_module = pytest.importorskip("agno.models.openai") Agent = agent_module.Agent @@ -190,7 +172,6 @@ def test_agno_agent_image_input_materializes_attachment(memory_logger): Re-record with ``nox -s "test_agno(latest)" -- --vcr-record=all -k "test_agno_agent_image_input_materializes_attachment"``. """ - _skip_if_cassette_missing("test_agno_agent_image_input_materializes_attachment.yaml") agent_module = pytest.importorskip("agno.agent") openai_module = pytest.importorskip("agno.models.openai") media_module = pytest.importorskip("agno.media") @@ -202,8 +183,8 @@ def test_agno_agent_image_input_materializes_attachment(memory_logger): png_bytes = ( b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR" b"\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4" - b"\x89\x00\x00\x00\rIDATx\x9cc\xf8\x0f\x00\x00\x01\x01\x00\x05" - b"\x0c\xd6\xa2\r\x00\x00\x00\x00IEND\xaeB`\x82" + b"\x89\x00\x00\x00\rIDATx\xdac\xfc\xcf\xc0\xf0\x1f\x00\x05\x05\x02" + b"\x00_\xc8\xf1\xd2\x00\x00\x00\x00IEND\xaeB`\x82" ) assert not memory_logger.pop() diff --git a/py/src/braintrust/test_vcr_config.py b/py/src/braintrust/test_vcr_config.py new file mode 100644 index 00000000..e70ee712 --- /dev/null +++ b/py/src/braintrust/test_vcr_config.py @@ -0,0 +1,19 @@ +from braintrust.conftest import get_vcr_config + + +def test_vcr_config_scrubs_sensitive_provider_headers(): + config = get_vcr_config() + + assert {"cookie", "openai-organization", "openai-project"} <= set(config["filter_headers"]) + + response = { + "headers": { + "Content-Type": ["application/json"], + "OpenAI-Organization": ["org-sensitive"], + "OpenAI-Project": ["proj-sensitive"], + "Set-Cookie": ["session=sensitive"], + } + } + scrubbed = config["before_record_response"](response) + + assert scrubbed["headers"] == {"Content-Type": ["application/json"]} From 3388ecb149b3eaedc5563e8ef8ba4a6866f996c6 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 21:06:22 +0000 Subject: [PATCH 3/4] docs(agno): trim _split_model_call docstring Co-Authored-By: Claude Opus 4.7 --- py/src/braintrust/integrations/agno/tracing.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py index 82dbfac6..7102b92d 100644 --- a/py/src/braintrust/integrations/agno/tracing.py +++ b/py/src/braintrust/integrations/agno/tracing.py @@ -42,9 +42,8 @@ def _split_model_call( ) -> tuple[dict[str, Any], dict[str, Any]]: """Bind positional args to their names and split into (input, metadata_extras). - Tool definitions, tool_choice, function schemas and tool-call limits go - into metadata per the integrations spec; everything else that names a - request field lands in input. Pure — no materialization side effects. + Tools, tool_choice, function schemas and tool-call limits go to metadata; + everything else that names a request field stays in input. """ combined: dict[str, Any] = dict(kwargs) for i, key in enumerate(positional_order): From 5aed2feac6ca7a263aefd921737066ae1bce736c Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 21:07:56 +0000 Subject: [PATCH 4/4] perf(agno): skip dict conversion when there is no inline media MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_prepare_model_input` and `_prepare_model_output` used to call `_try_to_dict()` on every model turn — a wasted `.model_dump()` on non-media calls, since Braintrust's log-time serializer already handles dataclasses and Pydantic models. Now they check for inline `content` bytes or `filepath` first and pass the raw SDK object through when no materialization is needed. Only calls that actually carry binary media pay for the conversion. Co-Authored-By: Claude Opus 4.7 --- .../braintrust/integrations/agno/tracing.py | 59 +++++++++++++++++-- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py index 7102b92d..6263cf2f 100644 --- a/py/src/braintrust/integrations/agno/tracing.py +++ b/py/src/braintrust/integrations/agno/tracing.py @@ -57,14 +57,27 @@ def _split_model_call( def _prepare_model_input(input_data: dict[str, Any]) -> dict[str, Any]: - """Materialize inline media in an Agno request-input dict before logging.""" - if "messages" in input_data: - return {**input_data, "messages": _materialize_agno_messages(input_data["messages"])} - return input_data + """Materialize inline media in an Agno request-input dict before logging. + + Fast path: leave `messages` as raw Agno objects when no message carries + inline binary media — Braintrust's log-time serializer handles the rest. + """ + messages = input_data.get("messages") + if not isinstance(messages, list) or not any(_message_has_inline_media(m) for m in messages): + return input_data + return {**input_data, "messages": _materialize_agno_messages(messages)} def _prepare_model_output(result: Any) -> Any: - """Convert an Agno model response to a materialized dict; pass through on failure.""" + """Materialize inline media in an Agno model response before logging. + + Fast path: pass the raw SDK object through when it has no inline binary + media — Braintrust's log-time serializer already converts dataclasses and + Pydantic models. Only when we actually need to swap in an Attachment do we + pay the dict conversion. + """ + if not _result_has_inline_media(result): + return result try: as_dict = _try_to_dict(result) except Exception: @@ -74,6 +87,42 @@ def _prepare_model_output(result: Any) -> Any: return _materialize_agno_output_media(dict(as_dict)) +def _agno_media_has_inline_bytes(media: Any) -> bool: + """True when *media* carries a `content` byte payload or a `filepath`.""" + if media is None: + return False + content = getattr(media, "content", None) if not isinstance(media, dict) else media.get("content") + if isinstance(content, (bytes, bytearray)) and content: + return True + if isinstance(content, str) and content: + return True + filepath = getattr(media, "filepath", None) if not isinstance(media, dict) else media.get("filepath") + return isinstance(filepath, str) and bool(filepath) + + +def _iter_media_field(container: Any, field: str) -> Any: + val = getattr(container, field, None) if not isinstance(container, dict) else container.get(field) + if val is None: + return () + return val if isinstance(val, list) else (val,) + + +def _message_has_inline_media(msg: Any) -> bool: + for field, _ in _AGNO_MESSAGE_MEDIA_FIELDS: + for item in _iter_media_field(msg, field): + if _agno_media_has_inline_bytes(item): + return True + return False + + +def _result_has_inline_media(result: Any) -> bool: + for field in ("images", "audio", "audios", "videos", "files"): + for item in _iter_media_field(result, field): + if _agno_media_has_inline_bytes(item): + return True + return False + + def is_sync_iterator(result: Any) -> bool: return hasattr(result, "__iter__") and hasattr(result, "__next__")