diff --git a/py/noxfile.py b/py/noxfile.py
index 1eb9b8b8..68df96cf 100644
--- a/py/noxfile.py
+++ b/py/noxfile.py
@@ -348,8 +348,7 @@ def test_agno(session, version):
_install_test_deps(session)
_install_matrix_dep(session, "agno", version)
_install_group_locked(session, "test-agno")
- _run_tests(session, f"{INTEGRATION_DIR}/agno/test_agno.py", version=version)
- _run_tests(session, f"{INTEGRATION_DIR}/agno/test_workflow.py", version=version)
+ _run_tests(session, f"{INTEGRATION_DIR}/agno", version=version)
LIVEKIT_AGENTS_VERSIONS = _get_matrix_versions("livekit-agents")
diff --git a/py/src/braintrust/conftest.py b/py/src/braintrust/conftest.py
index bc082ee1..bfeb582d 100644
--- a/py/src/braintrust/conftest.py
+++ b/py/src/braintrust/conftest.py
@@ -232,6 +232,23 @@ def skip_vcr_tests_in_wheel_mode(request):
pytest.skip("VCR tests skipped in wheel mode (pre-release sanity check only)")
+_SENSITIVE_RESPONSE_HEADERS = {
+ "openai-organization",
+ "openai-project",
+ "set-cookie",
+}
+
+
+def _scrub_sensitive_response_headers(response):
+ """Remove sensitive provider headers before writing a cassette."""
+ response["headers"] = {
+ name: value
+ for name, value in response.get("headers", {}).items()
+ if name.lower() not in _SENSITIVE_RESPONSE_HEADERS
+ }
+ return response
+
+
def get_vcr_config():
"""
Get VCR configuration for recording/playing back HTTP interactions.
@@ -247,6 +264,9 @@ def get_vcr_config():
"authorization",
"Authorization",
"openai-organization",
+ "openai-project",
+ "cookie",
+ "Cookie",
"x-api-key",
"api-key",
"openai-api-key",
@@ -259,6 +279,7 @@ def get_vcr_config():
"amz-sdk-request",
"x-amzn-bedrock-api-key",
],
+ "before_record_response": _scrub_sensitive_response_headers,
}
diff --git a/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_image_input_materializes_attachment.yaml
new file mode 100644
index 00000000..f7a96405
--- /dev/null
+++ b/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_image_input_materializes_attachment.yaml
@@ -0,0 +1,160 @@
+interactions:
+- request:
+ body: '{"messages":[{"role":"developer","content":"\nDescribe the
+ image in one word.\n"},{"role":"user","content":[{"type":"text","text":"What''s
+ in this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"gpt-4o-mini"}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '371'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MZDFm1JBErEB3lbW8PJBHrj49Ve\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232111,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": \"Red.\",\n \"refusal\": null,\n
+ \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\":
+ \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 8529,\n \"completion_tokens\":
+ 2,\n \"total_tokens\": 8531,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_844ddd95ff\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c399e609df4fcb-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '814'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:01:51 GMT
+ openai-processing-ms:
+ - '556'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-input-images:
+ - '50000'
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-input-images:
+ - '49999'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999210'
+ x-ratelimit-reset-input-images:
+ - 1ms
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_5d175e2325594ca8bb63ef1d92003b07
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"session_id":"5b14603b-835b-4837-9b92-3006800a5515","run_id":"5ddac676-8ef5-48a5-a25a-ed78dda4ce84","data":{"agent_id":"vision-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.1.0","type":"agent"}'
+ headers:
+ accept:
+ - '*/*'
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '453'
+ content-type:
+ - application/json
+ host:
+ - os-api.agno.com
+ user-agent:
+ - agno/2.1.0
+ method: POST
+ uri: https://os-api.agno.com/telemetry/runs
+ response:
+ body:
+ string: '{"message":"Run creation acknowledged: 5ddac676-8ef5-48a5-a25a-ed78dda4ce84","status":"success"}'
+ headers:
+ connection:
+ - keep-alive
+ content-length:
+ - '96'
+ content-type:
+ - application/json
+ cross-origin-opener-policy:
+ - same-origin
+ cross-origin-resource-policy:
+ - same-origin
+ date:
+ - Thu, 16 Jul 2026 20:01:51 GMT
+ permissions-policy:
+ - camera=(), microphone=(), geolocation=()
+ referrer-policy:
+ - strict-origin-when-cross-origin
+ server:
+ - uvicorn
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains
+ x-content-type-options:
+ - nosniff
+ x-frame-options:
+ - DENY
+ status:
+ code: 201
+ message: Created
+version: 1
diff --git a/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_tools_metadata_placement.yaml b/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_tools_metadata_placement.yaml
new file mode 100644
index 00000000..1ed1ada6
--- /dev/null
+++ b/py/src/braintrust/integrations/agno/cassettes/2.1.0/test_agno_agent_tools_metadata_placement.yaml
@@ -0,0 +1,264 @@
+interactions:
+- request:
+ body: '{"messages":[{"role":"developer","content":"\nUse the get_weather
+ tool to answer questions.\n"},{"role":"user","content":"What''s
+ the weather in Paris?"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return
+ the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '415'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MZAIzZ9MlaKi6phZddEyg7dgNRF\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232108,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n
+ \ \"id\": \"call_gCgs7fNBAzJefgRzrceNF1hO\",\n \"type\":
+ \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n
+ \ \"arguments\": \"{\\\"city\\\":\\\"Paris\\\"}\"\n }\n
+ \ }\n ],\n \"refusal\": null,\n \"annotations\":
+ []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n
+ \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\":
+ 14,\n \"total_tokens\": 81,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c399d478447aa8-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '1084'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:01:49 GMT
+ openai-processing-ms:
+ - '809'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999972'
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_7b66a14062ff44209bb87379ae3b7179
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"messages":[{"role":"developer","content":"\nUse the get_weather
+ tool to answer questions.\n"},{"role":"user","content":"What''s
+ the weather in Paris?"},{"role":"assistant","tool_calls":[{"id":"call_gCgs7fNBAzJefgRzrceNF1hO","function":{"arguments":"{\"city\":\"Paris\"}","name":"get_weather"},"type":"function"}],"content":""},{"role":"tool","content":"The
+ weather in Paris is 72F and sunny.","tool_call_id":"call_gCgs7fNBAzJefgRzrceNF1hO"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return
+ the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '704'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MZCvEGRddz5DBWH94zascdX5PvU\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232110,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": \"The weather in Paris is currently 72\xB0F
+ and sunny.\",\n \"refusal\": null,\n \"annotations\": []\n },\n
+ \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n
+ \ \"usage\": {\n \"prompt_tokens\": 100,\n \"completion_tokens\": 13,\n
+ \ \"total_tokens\": 113,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c399dad9605905-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '859'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:01:50 GMT
+ openai-processing-ms:
+ - '497'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999960'
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_13f2423c790c4aaea15e040f49bb1851
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"session_id":"cd7eb49c-b210-4bb6-a6a5-830746dc7f9c","run_id":"a05b11bb-1edb-4fb8-8d5c-fc9e8cb26fc1","data":{"agent_id":"weather-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.1.0","type":"agent"}'
+ headers:
+ accept:
+ - '*/*'
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '454'
+ content-type:
+ - application/json
+ host:
+ - os-api.agno.com
+ user-agent:
+ - agno/2.1.0
+ method: POST
+ uri: https://os-api.agno.com/telemetry/runs
+ response:
+ body:
+ string: '{"message":"Run creation acknowledged: a05b11bb-1edb-4fb8-8d5c-fc9e8cb26fc1","status":"success"}'
+ headers:
+ connection:
+ - keep-alive
+ content-length:
+ - '96'
+ content-type:
+ - application/json
+ cross-origin-opener-policy:
+ - same-origin
+ cross-origin-resource-policy:
+ - same-origin
+ date:
+ - Thu, 16 Jul 2026 20:01:50 GMT
+ permissions-policy:
+ - camera=(), microphone=(), geolocation=()
+ referrer-policy:
+ - strict-origin-when-cross-origin
+ server:
+ - uvicorn
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains
+ x-content-type-options:
+ - nosniff
+ x-frame-options:
+ - DENY
+ status:
+ code: 201
+ message: Created
+version: 1
diff --git a/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_image_input_materializes_attachment.yaml
new file mode 100644
index 00000000..3f6c27cc
--- /dev/null
+++ b/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_image_input_materializes_attachment.yaml
@@ -0,0 +1,157 @@
+interactions:
+- request:
+ body: '{"messages":[{"role":"developer","content":"Describe the image in one word."},{"role":"user","content":[{"type":"text","text":"What''s
+ in this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"gpt-4o-mini"}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '338'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MYzcSH1UCsHzU3QNzHnK9FIZGz7\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232097,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": \"Red.\",\n \"refusal\": null,\n
+ \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\":
+ \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 8523,\n \"completion_tokens\":
+ 2,\n \"total_tokens\": 8525,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_844ddd95ff\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c3998eee36abfa-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '814'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:01:38 GMT
+ openai-processing-ms:
+ - '1308'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-input-images:
+ - '50000'
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-input-images:
+ - '49999'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999217'
+ x-ratelimit-reset-input-images:
+ - 1ms
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_22335d6856c84be48a7eef8157caf80f
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"session_id":"e81647f1-acc6-4572-8d0b-fd07effd7004","run_id":"40baf7fc-38c7-40d4-bc26-d6037d6fac53","data":{"agent_id":"vision-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_learnings":false,"has_culture":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.4.0","type":"agent"}'
+ headers:
+ accept:
+ - '*/*'
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '495'
+ content-type:
+ - application/json
+ host:
+ - os-api.agno.com
+ user-agent:
+ - agno/2.4.0
+ method: POST
+ uri: https://os-api.agno.com/telemetry/runs
+ response:
+ body:
+ string: '{"message":"Run creation acknowledged: 40baf7fc-38c7-40d4-bc26-d6037d6fac53","status":"success"}'
+ headers:
+ content-length:
+ - '96'
+ content-type:
+ - application/json
+ cross-origin-opener-policy:
+ - same-origin
+ cross-origin-resource-policy:
+ - same-origin
+ date:
+ - Thu, 16 Jul 2026 20:01:38 GMT
+ permissions-policy:
+ - camera=(), microphone=(), geolocation=()
+ referrer-policy:
+ - strict-origin-when-cross-origin
+ server:
+ - uvicorn
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains
+ x-content-type-options:
+ - nosniff
+ x-frame-options:
+ - DENY
+ status:
+ code: 201
+ message: Created
+version: 1
diff --git a/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_tools_metadata_placement.yaml b/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_tools_metadata_placement.yaml
new file mode 100644
index 00000000..c1946035
--- /dev/null
+++ b/py/src/braintrust/integrations/agno/cassettes/2.4.0/test_agno_agent_tools_metadata_placement.yaml
@@ -0,0 +1,260 @@
+interactions:
+- request:
+ body: '{"messages":[{"role":"developer","content":"Use the get_weather tool to
+ answer questions."},{"role":"user","content":"What''s the weather in Paris?"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return
+ the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '382'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MYxdFisKNQJUJ7lGomeZM7dw5rI\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232095,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n
+ \ \"id\": \"call_MI4n3R3AoYrD6OFXbH0rgiAJ\",\n \"type\":
+ \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n
+ \ \"arguments\": \"{\\\"city\\\":\\\"Paris\\\"}\"\n }\n
+ \ }\n ],\n \"refusal\": null,\n \"annotations\":
+ []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n
+ \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 61,\n \"completion_tokens\":
+ 14,\n \"total_tokens\": 75,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_a8eb499e0d\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c39983bb2fabfa-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '1084'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:01:36 GMT
+ openai-processing-ms:
+ - '570'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999977'
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_d03570245a304b549f93a99746f116d7
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"messages":[{"role":"developer","content":"Use the get_weather tool to
+ answer questions."},{"role":"user","content":"What''s the weather in Paris?"},{"role":"assistant","tool_calls":[{"id":"call_MI4n3R3AoYrD6OFXbH0rgiAJ","function":{"arguments":"{\"city\":\"Paris\"}","name":"get_weather"},"type":"function"}],"content":""},{"role":"tool","content":"The
+ weather in Paris is 72F and sunny.","tool_call_id":"call_MI4n3R3AoYrD6OFXbH0rgiAJ"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return
+ the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '671'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MYyjad1qXH8NCzbKr1uRa6UN1p6\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232096,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": \"The weather in Paris is 72\xB0F and
+ sunny.\",\n \"refusal\": null,\n \"annotations\": []\n },\n
+ \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n
+ \ \"usage\": {\n \"prompt_tokens\": 94,\n \"completion_tokens\": 12,\n
+ \ \"total_tokens\": 106,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_a8eb499e0d\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c39988afa9abfa-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '848'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:01:36 GMT
+ openai-processing-ms:
+ - '604'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999967'
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_110f9383dbbf42b4a35da0f4320d5379
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"session_id":"ca380d3f-1290-49d1-8fd1-4234f24d61b4","run_id":"33f187ef-d597-4be6-b623-dc76ce1005b9","data":{"agent_id":"weather-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_learnings":false,"has_culture":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.4.0","type":"agent"}'
+ headers:
+ accept:
+ - '*/*'
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '496'
+ content-type:
+ - application/json
+ host:
+ - os-api.agno.com
+ user-agent:
+ - agno/2.4.0
+ method: POST
+ uri: https://os-api.agno.com/telemetry/runs
+ response:
+ body:
+ string: '{"message":"Run creation acknowledged: 33f187ef-d597-4be6-b623-dc76ce1005b9","status":"success"}'
+ headers:
+ content-length:
+ - '96'
+ content-type:
+ - application/json
+ cross-origin-opener-policy:
+ - same-origin
+ cross-origin-resource-policy:
+ - same-origin
+ date:
+ - Thu, 16 Jul 2026 20:01:37 GMT
+ permissions-policy:
+ - camera=(), microphone=(), geolocation=()
+ referrer-policy:
+ - strict-origin-when-cross-origin
+ server:
+ - uvicorn
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains
+ x-content-type-options:
+ - nosniff
+ x-frame-options:
+ - DENY
+ status:
+ code: 201
+ message: Created
+version: 1
diff --git a/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_image_input_materializes_attachment.yaml
new file mode 100644
index 00000000..739035da
--- /dev/null
+++ b/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_image_input_materializes_attachment.yaml
@@ -0,0 +1,157 @@
+interactions:
+- request:
+ body: '{"messages":[{"role":"developer","content":"Describe the image in one word."},{"role":"user","content":[{"type":"text","text":"What''s
+ in this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}}]}],"model":"gpt-4o-mini"}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '338'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MYEAGGbKfSph0LSwpPHjkGelm1k\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232050,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": \"Red.\",\n \"refusal\": null,\n
+ \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\":
+ \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 8523,\n \"completion_tokens\":
+ 2,\n \"total_tokens\": 8525,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_844ddd95ff\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c3986d3cebab5a-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '814'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:00:51 GMT
+ openai-processing-ms:
+ - '486'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-input-images:
+ - '50000'
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-input-images:
+ - '49999'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999217'
+ x-ratelimit-reset-input-images:
+ - 1ms
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_36660cffe8ba4ddcb3b3b549e5bb4303
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"session_id":"196de1f1-cfd3-42c0-9b9b-f8d63e9c66e9","run_id":"c4a4ab9b-191d-4f14-93dc-ece642e77d7e","data":{"agent_id":"vision-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_learnings":false,"has_culture":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.7.2","type":"agent"}'
+ headers:
+ accept:
+ - '*/*'
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '495'
+ content-type:
+ - application/json
+ host:
+ - os-api.agno.com
+ user-agent:
+ - agno/2.7.2
+ method: POST
+ uri: https://os-api.agno.com/telemetry/runs
+ response:
+ body:
+ string: '{"message":"Run creation acknowledged: c4a4ab9b-191d-4f14-93dc-ece642e77d7e","status":"success"}'
+ headers:
+ content-length:
+ - '96'
+ content-type:
+ - application/json
+ cross-origin-opener-policy:
+ - same-origin
+ cross-origin-resource-policy:
+ - same-origin
+ date:
+ - Thu, 16 Jul 2026 20:00:51 GMT
+ permissions-policy:
+ - camera=(), microphone=(), geolocation=()
+ referrer-policy:
+ - strict-origin-when-cross-origin
+ server:
+ - uvicorn
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains
+ x-content-type-options:
+ - nosniff
+ x-frame-options:
+ - DENY
+ status:
+ code: 201
+ message: Created
+version: 1
diff --git a/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_tools_metadata_placement.yaml b/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_tools_metadata_placement.yaml
new file mode 100644
index 00000000..33216517
--- /dev/null
+++ b/py/src/braintrust/integrations/agno/cassettes/latest/test_agno_agent_tools_metadata_placement.yaml
@@ -0,0 +1,260 @@
+interactions:
+- request:
+ body: '{"messages":[{"role":"developer","content":"Use the get_weather tool to
+ answer questions."},{"role":"user","content":"What''s the weather in Paris?"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return
+ the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '382'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MXTQrN5sd3ptBLIdUmFnHFC6SNv\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232003,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n
+ \ \"id\": \"call_RbqZ3KBymj7NFJLQ5XzZPu5p\",\n \"type\":
+ \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n
+ \ \"arguments\": \"{\\\"city\\\":\\\"Paris\\\"}\"\n }\n
+ \ }\n ],\n \"refusal\": null,\n \"annotations\":
+ []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n
+ \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 61,\n \"completion_tokens\":
+ 14,\n \"total_tokens\": 75,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_a8eb499e0d\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c39742cd5bab88-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '1084'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:00:04 GMT
+ openai-processing-ms:
+ - '575'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999977'
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_bc94e6d110b74df8a2bc2d1d53a7f413
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"messages":[{"role":"developer","content":"Use the get_weather tool to
+ answer questions."},{"role":"user","content":"What''s the weather in Paris?"},{"role":"assistant","tool_calls":[{"id":"call_RbqZ3KBymj7NFJLQ5XzZPu5p","function":{"arguments":"{\"city\":\"Paris\"}","name":"get_weather"},"type":"function"}],"content":""},{"role":"tool","content":"The
+ weather in Paris is 72F and sunny.","tool_call_id":"call_RbqZ3KBymj7NFJLQ5XzZPu5p"}],"model":"gpt-4o-mini","tools":[{"type":"function","function":{"name":"get_weather","description":"Return
+ the current weather for *city*.","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]}'
+ headers:
+ accept:
+ - application/json
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '671'
+ content-type:
+ - application/json
+ host:
+ - api.openai.com
+ user-agent:
+ - OpenAI/Python 2.31.0
+ x-stainless-arch:
+ - arm64
+ x-stainless-async:
+ - 'false'
+ x-stainless-lang:
+ - python
+ x-stainless-os:
+ - MacOS
+ x-stainless-package-version:
+ - 2.31.0
+ x-stainless-read-timeout:
+ - '600'
+ x-stainless-retry-count:
+ - '0'
+ x-stainless-runtime:
+ - CPython
+ x-stainless-runtime-version:
+ - 3.14.6
+ method: POST
+ uri: https://api.openai.com/v1/chat/completions
+ response:
+ body:
+ string: "{\n \"id\": \"chatcmpl-E2MXU1FvuDn3iIDaUdTWPX4FReNil\",\n \"object\":
+ \"chat.completion\",\n \"created\": 1784232004,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
+ \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
+ \"assistant\",\n \"content\": \"The weather in Paris is 72\xB0F and
+ sunny.\",\n \"refusal\": null,\n \"annotations\": []\n },\n
+ \ \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n
+ \ \"usage\": {\n \"prompt_tokens\": 94,\n \"completion_tokens\": 12,\n
+ \ \"total_tokens\": 106,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
+ 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
+ {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
+ 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\":
+ \"default\",\n \"system_fingerprint\": \"fp_a8eb499e0d\"\n}\n"
+ headers:
+ access-control-expose-headers:
+ - X-Request-ID
+ - CF-Ray
+ - CF-Ray
+ alt-svc:
+ - h3=":443"; ma=86400
+ cf-cache-status:
+ - DYNAMIC
+ cf-ray:
+ - a1c3974aac4fab88-YYZ
+ connection:
+ - keep-alive
+ content-length:
+ - '848'
+ content-type:
+ - application/json
+ date:
+ - Thu, 16 Jul 2026 20:00:04 GMT
+ openai-processing-ms:
+ - '575'
+ openai-version:
+ - '2020-10-01'
+ server:
+ - cloudflare
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains; preload
+ transfer-encoding:
+ - chunked
+ x-content-type-options:
+ - nosniff
+ x-openai-proxy-wasm:
+ - v0.1
+ x-ratelimit-limit-requests:
+ - '30000'
+ x-ratelimit-limit-tokens:
+ - '150000000'
+ x-ratelimit-remaining-requests:
+ - '29999'
+ x-ratelimit-remaining-tokens:
+ - '149999967'
+ x-ratelimit-reset-requests:
+ - 2ms
+ x-ratelimit-reset-tokens:
+ - 0s
+ x-request-id:
+ - req_30cc951acc2b4e088ea653348b4cf3c8
+ status:
+ code: 200
+ message: OK
+- request:
+ body: '{"session_id":"579034d6-9325-41eb-805d-831364c2680f","run_id":"5525f7ad-894b-4b39-b816-ef3809903d29","data":{"agent_id":"weather-agent","db_type":null,"model_provider":"OpenAI","model_name":"OpenAIChat","model_id":"gpt-4o-mini","parser_model":null,"output_model":null,"has_tools":true,"has_memory":false,"has_learnings":false,"has_culture":false,"has_reasoning":false,"has_knowledge":false,"has_input_schema":false,"has_output_schema":false,"has_team":false},"sdk_version":"2.7.2","type":"agent"}'
+ headers:
+ accept:
+ - '*/*'
+ accept-encoding:
+ - gzip, deflate
+ connection:
+ - keep-alive
+ content-length:
+ - '496'
+ content-type:
+ - application/json
+ host:
+ - os-api.agno.com
+ user-agent:
+ - agno/2.7.2
+ method: POST
+ uri: https://os-api.agno.com/telemetry/runs
+ response:
+ body:
+ string: '{"message":"Run creation acknowledged: 5525f7ad-894b-4b39-b816-ef3809903d29","status":"success"}'
+ headers:
+ content-length:
+ - '96'
+ content-type:
+ - application/json
+ cross-origin-opener-policy:
+ - same-origin
+ cross-origin-resource-policy:
+ - same-origin
+ date:
+ - Thu, 16 Jul 2026 20:00:05 GMT
+ permissions-policy:
+ - camera=(), microphone=(), geolocation=()
+ referrer-policy:
+ - strict-origin-when-cross-origin
+ server:
+ - uvicorn
+ strict-transport-security:
+ - max-age=31536000; includeSubDomains
+ x-content-type-options:
+ - nosniff
+ x-frame-options:
+ - DENY
+ status:
+ code: 201
+ message: Created
+version: 1
diff --git a/py/src/braintrust/integrations/agno/test_agno.py b/py/src/braintrust/integrations/agno/test_agno.py
index 0d17262d..dd22859d 100644
--- a/py/src/braintrust/integrations/agno/test_agno.py
+++ b/py/src/braintrust/integrations/agno/test_agno.py
@@ -10,7 +10,7 @@
from braintrust.integrations.agno import tracing as agno_tracing_module
from braintrust.integrations.agno.patchers import wrap_agent, wrap_team
from braintrust.integrations.test_utils import verify_autoinstrument_script
-from braintrust.logger import start_span
+from braintrust.logger import Attachment, start_span
from braintrust.test_helpers import init_test_logger
from ._test_agno_helpers import (
@@ -37,6 +37,15 @@ def setup_wrapper():
yield
+_TOOL_METADATA_ONLY_KEYS = ("tools", "tool_choice", "functions", "tool_call_limit")
+
+
+def _assert_tool_fields_not_in_input(llm_span) -> None:
+ """Guardrail against regressing the SKILL rule that puts tool definitions in metadata."""
+ for forbidden in _TOOL_METADATA_ONLY_KEYS:
+ assert forbidden not in llm_span["input"], f"{forbidden!r} must live under metadata, not input"
+
+
@pytest.mark.vcr
def test_agno_simple_agent_execution(memory_logger):
agent_module = pytest.importorskip("agno.agent")
@@ -97,6 +106,9 @@ def test_agno_simple_agent_execution(memory_logger):
assert "librarian" in messages[0]["content"]
assert messages[1]["role"] == "user"
assert messages[1]["content"] == "Charlotte's Web"
+ # Tool-related fields must not leak into `input` even when they are absent
+ # from this particular call — see integrations SKILL "Span Design / Fields".
+ _assert_tool_fields_not_in_input(llm_span)
assert llm_span["output"]["content"] == "E.B. White"
assert llm_span["metrics"]["prompt_tokens"] > 0
assert llm_span["metrics"]["completion_tokens"] > 0
@@ -106,6 +118,108 @@ def test_agno_simple_agent_execution(memory_logger):
)
+@pytest.mark.vcr
+def test_agno_agent_tools_metadata_placement(memory_logger):
+ """Tool definitions must live in `metadata.tools`, not in the span input.
+
+ Cassette: recorded against the real OpenAI API with a small ``get_weather``
+ tool. Re-record with ``nox -s "test_agno(latest)" -- --vcr-record=all -k
+ "test_agno_agent_tools_metadata_placement"``.
+ """
+ agent_module = pytest.importorskip("agno.agent")
+ openai_module = pytest.importorskip("agno.models.openai")
+ Agent = agent_module.Agent
+ OpenAIChat = openai_module.OpenAIChat
+
+ def get_weather(city: str) -> str:
+ """Return the current weather for *city*."""
+ return f"The weather in {city} is 72F and sunny."
+
+ assert not memory_logger.pop()
+
+ agent = Agent(
+ name="Weather Agent",
+ model=OpenAIChat(id="gpt-4o-mini"),
+ tools=[get_weather],
+ instructions="Use the get_weather tool to answer questions.",
+ )
+
+ response = agent.run("What's the weather in Paris?")
+ assert response and response.content
+
+ spans = memory_logger.pop()
+ llm_spans = [s for s in spans if s["span_attributes"]["type"].value == "llm"]
+ assert llm_spans, "expected at least one llm span"
+
+ for llm_span in llm_spans:
+ _assert_tool_fields_not_in_input(llm_span)
+ tools_meta = llm_span["metadata"].get("tools")
+ assert tools_meta, "expected metadata.tools to be populated on llm spans"
+ # Agno passes its own tool schema (name / description / parameters) to
+ # Model.response — spec placement rule is satisfied as long as the tool
+ # definitions live in metadata, not input. Normalization to the fully
+ # OpenAI-wrapped `{"type": "function", "function": {...}}` shape is
+ # tracked separately.
+ names = {t.get("name") or t.get("function", {}).get("name") for t in tools_meta}
+ assert "get_weather" in names
+
+
+@pytest.mark.vcr
+def test_agno_agent_image_input_materializes_attachment(memory_logger):
+ """Inline image bytes must be replaced by a Braintrust ``Attachment``.
+
+ Cassette: recorded against the real OpenAI vision API with a 1x1 PNG.
+ Re-record with ``nox -s "test_agno(latest)" -- --vcr-record=all -k
+ "test_agno_agent_image_input_materializes_attachment"``.
+ """
+ agent_module = pytest.importorskip("agno.agent")
+ openai_module = pytest.importorskip("agno.models.openai")
+ media_module = pytest.importorskip("agno.media")
+ Agent = agent_module.Agent
+ OpenAIChat = openai_module.OpenAIChat
+ Image = media_module.Image
+
+ # 1x1 transparent PNG
+ png_bytes = (
+ b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"
+ b"\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4"
+ b"\x89\x00\x00\x00\rIDATx\xdac\xfc\xcf\xc0\xf0\x1f\x00\x05\x05\x02"
+ b"\x00_\xc8\xf1\xd2\x00\x00\x00\x00IEND\xaeB`\x82"
+ )
+
+ assert not memory_logger.pop()
+
+ agent = Agent(
+ name="Vision Agent",
+ model=OpenAIChat(id="gpt-4o-mini"),
+ instructions="Describe the image in one word.",
+ )
+
+ response = agent.run("What's in this image?", images=[Image(content=png_bytes, format="png")])
+ assert response and response.content
+
+ spans = memory_logger.pop()
+ llm_spans = [s for s in spans if s["span_attributes"]["type"].value == "llm"]
+ assert llm_spans
+
+ def _find_attachments(obj):
+ found = []
+ if isinstance(obj, Attachment):
+ found.append(obj)
+ elif isinstance(obj, dict):
+ for v in obj.values():
+ found.extend(_find_attachments(v))
+ elif isinstance(obj, list):
+ for v in obj:
+ found.extend(_find_attachments(v))
+ return found
+
+ attachments = _find_attachments(llm_spans[0]["input"])
+ assert attachments, "expected image bytes to be materialized as an Attachment"
+ att = attachments[0]
+ assert att.reference["content_type"].startswith("image/")
+
+
def test_get_model_name_prefers_stable_provider_attribute():
class FakeModel:
provider = "OpenAI"
diff --git a/py/src/braintrust/integrations/agno/tracing.py b/py/src/braintrust/integrations/agno/tracing.py
index 5c8983d4..6263cf2f 100644
--- a/py/src/braintrust/integrations/agno/tracing.py
+++ b/py/src/braintrust/integrations/agno/tracing.py
@@ -2,7 +2,7 @@
from inspect import isawaitable
from typing import Any
-from braintrust.integrations.utils import _try_to_dict
+from braintrust.integrations.utils import _materialize_attachment, _try_to_dict
from braintrust.logger import start_span as _bt_start_span
@@ -33,8 +33,94 @@ def clean(obj: dict[str, Any]) -> dict[str, Any]:
return {k: v for k, v in obj.items() if v is not None}
-def get_args_kwargs(args: list[str], kwargs: dict[str, Any], keys: list[str]):
- return {k: args[i] if args else kwargs.get(k) for i, k in enumerate(keys)}, omit(kwargs, keys)
+# Keys the SDK-integrations spec routes into metadata rather than the span input.
+_MODEL_METADATA_KEYS = ("tools", "tool_choice", "functions", "tool_call_limit")
+
+
+def _split_model_call(
+ args: tuple, kwargs: dict[str, Any], positional_order: list[str]
+) -> tuple[dict[str, Any], dict[str, Any]]:
+ """Bind positional args to their names and split into (input, metadata_extras).
+
+ Tools, tool_choice, function schemas and tool-call limits go to metadata;
+ everything else that names a request field stays in input.
+ """
+ combined: dict[str, Any] = dict(kwargs)
+ for i, key in enumerate(positional_order):
+ if i < len(args):
+ combined[key] = args[i]
+ metadata_extras: dict[str, Any] = {}
+ for k in _MODEL_METADATA_KEYS:
+ if k in combined:
+ metadata_extras[k] = combined.pop(k)
+ return combined, metadata_extras
+
+
+def _prepare_model_input(input_data: dict[str, Any]) -> dict[str, Any]:
+ """Materialize inline media in an Agno request-input dict before logging.
+
+ Fast path: leave `messages` as raw Agno objects when no message carries
+ inline binary media — Braintrust's log-time serializer handles the rest.
+ """
+ messages = input_data.get("messages")
+ if not isinstance(messages, list) or not any(_message_has_inline_media(m) for m in messages):
+ return input_data
+ return {**input_data, "messages": _materialize_agno_messages(messages)}
+
+
+def _prepare_model_output(result: Any) -> Any:
+ """Materialize inline media in an Agno model response before logging.
+
+ Fast path: pass the raw SDK object through when it has no inline binary
+ media — Braintrust's log-time serializer already converts dataclasses and
+ Pydantic models. Only when we actually need to swap in an Attachment do we
+ pay the dict conversion.
+ """
+ if not _result_has_inline_media(result):
+ return result
+ try:
+ as_dict = _try_to_dict(result)
+ except Exception:
+ return result
+ if not isinstance(as_dict, dict):
+ return result
+ return _materialize_agno_output_media(dict(as_dict))
+
+
+def _agno_media_has_inline_bytes(media: Any) -> bool:
+ """True when *media* carries a `content` byte payload or a `filepath`."""
+ if media is None:
+ return False
+ content = getattr(media, "content", None) if not isinstance(media, dict) else media.get("content")
+ if isinstance(content, (bytes, bytearray)) and content:
+ return True
+ if isinstance(content, str) and content:
+ return True
+ filepath = getattr(media, "filepath", None) if not isinstance(media, dict) else media.get("filepath")
+ return isinstance(filepath, str) and bool(filepath)
+
+
+def _iter_media_field(container: Any, field: str) -> Any:
+ val = getattr(container, field, None) if not isinstance(container, dict) else container.get(field)
+ if val is None:
+ return ()
+ return val if isinstance(val, list) else (val,)
+
+
+def _message_has_inline_media(msg: Any) -> bool:
+ for field, _ in _AGNO_MESSAGE_MEDIA_FIELDS:
+ for item in _iter_media_field(msg, field):
+ if _agno_media_has_inline_bytes(item):
+ return True
+ return False
+
+
+def _result_has_inline_media(result: Any) -> bool:
+ for field in ("images", "audio", "audios", "videos", "files"):
+ for item in _iter_media_field(result, field):
+ if _agno_media_has_inline_bytes(item):
+ return True
+ return False
def is_sync_iterator(result: Any) -> bool:
@@ -45,6 +131,126 @@ def is_async_iterator(result: Any) -> bool:
return hasattr(result, "__aiter__") and hasattr(result, "__anext__")
+# ---------------------------------------------------------------------------
+# Multimodal materialization
+# ---------------------------------------------------------------------------
+#
+# Agno's Image / Audio / Video / File objects carry inline bytes on ``content``
+# (plus optional ``url`` / ``filepath`` / ``mime_type`` / ``format`` / ``filename``).
+# The integrations spec requires inline binary media to be converted to
+# Braintrust ``Attachment`` objects at the leaf position; remote URLs stay as
+# strings and unrecognized shapes pass through unchanged.
+
+
+def _agno_media_mime_type(as_dict: dict[str, Any], media_kind: str) -> str | None:
+ mime = as_dict.get("mime_type")
+ if isinstance(mime, str) and mime:
+ return mime
+ fmt = as_dict.get("format")
+ if isinstance(fmt, str) and fmt:
+ return f"{media_kind}/{fmt}"
+ return None
+
+
+def _materialize_agno_media(value: Any, media_kind: str) -> Any:
+ """Materialize a single Agno media object to a serializable dict.
+
+ Returns the input unchanged when it is not a recognizable Agno media object
+ or when materialization is not possible.
+ """
+ if value is None:
+ return value
+
+ as_dict = value if isinstance(value, dict) else _try_to_dict(value)
+ if not isinstance(as_dict, dict):
+ return value
+
+ content = as_dict.get("content")
+ filepath = as_dict.get("filepath")
+ url = as_dict.get("url")
+ filename = as_dict.get("filename")
+ mime_type = _agno_media_mime_type(as_dict, media_kind)
+
+ raw: Any = content if isinstance(content, (bytes, bytearray, str)) and content else None
+ if raw is None and isinstance(filepath, str) and filepath:
+ raw = filepath
+
+ resolved = None
+ if raw is not None:
+ try:
+ resolved = _materialize_attachment(
+ raw,
+ mime_type=mime_type,
+ filename=filename if isinstance(filename, str) else None,
+ label=media_kind,
+ prefix=media_kind,
+ )
+ except Exception:
+ resolved = None
+
+ result: dict[str, Any] = {k: v for k, v in as_dict.items() if k not in ("content", "filepath")}
+ if resolved is not None:
+ result.update(resolved.multimodal_part_payload)
+ elif isinstance(url, str) and url and "url" not in result:
+ # Preserve remote URL references verbatim per the spec.
+ result["url"] = url
+ return result
+
+
+def _materialize_agno_media_list(value: Any, media_kind: str) -> Any:
+ if not isinstance(value, list):
+ return value
+ return [_materialize_agno_media(v, media_kind) for v in value]
+
+
+_AGNO_MESSAGE_MEDIA_FIELDS = (
+ ("images", "image"),
+ ("image_output", "image"),
+ ("audio", "audio"),
+ ("audio_output", "audio"),
+ ("videos", "video"),
+ ("video_output", "video"),
+ ("files", "file"),
+ ("file_output", "file"),
+)
+
+
+def _materialize_agno_message(msg: Any) -> Any:
+ """Return a dict form of an Agno Message with inline media replaced by attachments."""
+ as_dict = msg if isinstance(msg, dict) else _try_to_dict(msg)
+ if not isinstance(as_dict, dict):
+ return msg
+ result = dict(as_dict)
+ for field, kind in _AGNO_MESSAGE_MEDIA_FIELDS:
+ if field in result and result[field]:
+ if isinstance(result[field], list):
+ result[field] = [_materialize_agno_media(v, kind) for v in result[field]]
+ else:
+ result[field] = _materialize_agno_media(result[field], kind)
+ return result
+
+
+def _materialize_agno_messages(messages: Any) -> Any:
+ if not isinstance(messages, list):
+ return messages
+ return [_materialize_agno_message(m) for m in messages]
+
+
+def _materialize_agno_output_media(aggregated: dict[str, Any]) -> dict[str, Any]:
+ """Materialize model-response-style media fields in an aggregated dict in place."""
+ for field, kind in (("images", "image"), ("videos", "video"), ("files", "file")):
+ val = aggregated.get(field)
+ if isinstance(val, list) and val:
+ aggregated[field] = [_materialize_agno_media(v, kind) for v in val]
+ audio_val = aggregated.get("audio")
+ if audio_val is not None:
+ if isinstance(audio_val, list):
+ aggregated["audio"] = [_materialize_agno_media(v, "audio") for v in audio_val]
+ else:
+ aggregated["audio"] = _materialize_agno_media(audio_val, "audio")
+ return aggregated
+
+
# ---------------------------------------------------------------------------
# Metrics mapping & extraction
# ---------------------------------------------------------------------------
@@ -207,7 +413,7 @@ def _aggregate_model_chunks(chunks: list[Any]) -> dict[str, Any]:
else:
aggregated["metrics"] = None
- return aggregated
+ return _materialize_agno_output_media(aggregated)
def _aggregate_response_stream_chunks(chunks: list[Any]) -> dict[str, Any]:
@@ -267,7 +473,7 @@ def _aggregate_response_stream_chunks(chunks: list[Any]) -> dict[str, Any]:
else:
aggregated["metrics"] = None
- return aggregated
+ return _materialize_agno_output_media(aggregated)
def _aggregate_agent_chunks(chunks: list[Any]) -> dict[str, Any]:
@@ -393,7 +599,7 @@ def _inner():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -421,7 +627,7 @@ async def _inner():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -506,7 +712,7 @@ def _trace_stream():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -547,7 +753,7 @@ async def _trace_stream():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -622,7 +828,7 @@ def _trace_stream():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -663,7 +869,7 @@ async def _trace_stream():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -710,7 +916,7 @@ def _run_public_dispatch_wrapper(
span.end()
return result
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
span.unset_current()
span.end()
raise
@@ -754,7 +960,7 @@ async def _trace_awaitable():
span.log(output=awaited, metrics=extract_metrics(awaited))
return awaited
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_end_span:
@@ -771,7 +977,7 @@ async def _trace_awaitable():
span.end()
return result
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
span.unset_current()
span.end()
raise
@@ -817,41 +1023,38 @@ def _get_model_name(instance: Any) -> str:
def _model_invoke_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
model_name = _get_model_name(instance)
- input, clean_kwargs = get_args_kwargs(
- args, kwargs, ["assistant_message", "messages", "response_format", "tools", "tool_choice"]
- )
+ input, metadata_extras = _split_model_call(args, kwargs, ["assistant_message", "messages"])
+ input = _prepare_model_input(input)
with start_span(
name=f"{model_name}.invoke",
type=SpanTypeAttribute.LLM,
input=input,
- metadata={**clean_kwargs, **extract_metadata(instance, "model")},
+ metadata={**metadata_extras, **extract_metadata(instance, "model")},
) as span:
result = wrapped(*args, **kwargs)
- span.log(output=result, metrics=extract_metrics(result, kwargs.get("messages", [])))
+ span.log(output=_prepare_model_output(result), metrics=extract_metrics(result, kwargs.get("messages", [])))
return result
async def _model_ainvoke_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
model_name = _get_model_name(instance)
- input, clean_kwargs = get_args_kwargs(
- args, kwargs, ["messages", "assistant_message", "response_format", "tools", "tool_choice"]
- )
+ input, metadata_extras = _split_model_call(args, kwargs, ["messages", "assistant_message"])
+ input = _prepare_model_input(input)
with start_span(
name=f"{model_name}.ainvoke",
type=SpanTypeAttribute.LLM,
input=input,
- metadata={**clean_kwargs, **extract_metadata(instance, "model")},
+ metadata={**metadata_extras, **extract_metadata(instance, "model")},
) as span:
result = await wrapped(*args, **kwargs)
- span.log(output=result, metrics=extract_metrics(result, kwargs.get("messages", [])))
+ span.log(output=_prepare_model_output(result), metrics=extract_metrics(result, kwargs.get("messages", [])))
return result
def _model_invoke_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
model_name = _get_model_name(instance)
- input, clean_kwargs = get_args_kwargs(
- args, kwargs, ["messages", "assistant_messages", "response_format", "tools", "tool_choice"]
- )
+ input, metadata_extras = _split_model_call(args, kwargs, ["messages", "assistant_messages"])
+ input = _prepare_model_input(input)
def _trace_stream():
start = time.time()
@@ -859,7 +1062,7 @@ def _trace_stream():
name=f"{model_name}.invoke_stream",
type=SpanTypeAttribute.LLM,
input=input,
- metadata={**clean_kwargs, **extract_metadata(instance, "model")},
+ metadata={**metadata_extras, **extract_metadata(instance, "model")},
) as span:
first = True
collected_chunks = []
@@ -877,9 +1080,8 @@ def _trace_stream():
def _model_ainvoke_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
model_name = _get_model_name(instance)
- input, clean_kwargs = get_args_kwargs(
- args, kwargs, ["messages", "assistant_messages", "response_format", "tools", "tool_choice"]
- )
+ input, metadata_extras = _split_model_call(args, kwargs, ["messages", "assistant_messages"])
+ input = _prepare_model_input(input)
async def _trace_astream():
start = time.time()
@@ -887,7 +1089,7 @@ async def _trace_astream():
name=f"{model_name}.ainvoke_stream",
type=SpanTypeAttribute.LLM,
input=input,
- metadata={**clean_kwargs, **extract_metadata(instance, "model")},
+ metadata={**metadata_extras, **extract_metadata(instance, "model")},
) as span:
first = True
collected_chunks = []
@@ -905,41 +1107,38 @@ async def _trace_astream():
def _model_response_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
model_name = _get_model_name(instance)
- input, clean_kwargs = get_args_kwargs(
- args, kwargs, ["messages", "response_format", "tools", "functions", "tool_chocie", "tool_call_limit"]
- )
+ input, metadata_extras = _split_model_call(args, kwargs, ["messages"])
+ input = _prepare_model_input(input)
with start_span(
name=f"{model_name}.response",
type=SpanTypeAttribute.LLM,
input=input,
- metadata={**clean_kwargs, **extract_metadata(instance, "model")},
+ metadata={**metadata_extras, **extract_metadata(instance, "model")},
) as span:
result = wrapped(*args, **kwargs)
- span.log(output=result, metrics=extract_metrics(result, kwargs.get("messages", [])))
+ span.log(output=_prepare_model_output(result), metrics=extract_metrics(result, kwargs.get("messages", [])))
return result
async def _model_aresponse_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
model_name = _get_model_name(instance)
- input, clean_kwargs = get_args_kwargs(
- args, kwargs, ["messages", "response_format", "tools", "functions", "tool_chocie", "tool_call_limit"]
- )
+ input, metadata_extras = _split_model_call(args, kwargs, ["messages"])
+ input = _prepare_model_input(input)
with start_span(
name=f"{model_name}.aresponse",
type=SpanTypeAttribute.LLM,
input=input,
- metadata={**clean_kwargs, **extract_metadata(instance, "model")},
+ metadata={**metadata_extras, **extract_metadata(instance, "model")},
) as span:
result = await wrapped(*args, **kwargs)
- span.log(output=result, metrics=extract_metrics(result, kwargs.get("messages", [])))
+ span.log(output=_prepare_model_output(result), metrics=extract_metrics(result, kwargs.get("messages", [])))
return result
def _model_response_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
model_name = _get_model_name(instance)
- input, clean_kwargs = get_args_kwargs(
- args, kwargs, ["messages", "response_format", "tools", "functions", "tool_chocie", "tool_call_limit"]
- )
+ input, metadata_extras = _split_model_call(args, kwargs, ["messages"])
+ input = _prepare_model_input(input)
def _trace_stream():
start = time.time()
@@ -947,7 +1146,7 @@ def _trace_stream():
name=f"{model_name}.response_stream",
type=SpanTypeAttribute.LLM,
input=input,
- metadata={**clean_kwargs, **extract_metadata(instance, "model")},
+ metadata={**metadata_extras, **extract_metadata(instance, "model")},
) as span:
first = True
collected_chunks = []
@@ -965,9 +1164,8 @@ def _trace_stream():
def _model_aresponse_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
model_name = _get_model_name(instance)
- input, clean_kwargs = get_args_kwargs(
- args, kwargs, ["messages", "response_format", "tools", "functions", "tool_chocie", "tool_call_limit"]
- )
+ input, metadata_extras = _split_model_call(args, kwargs, ["messages"])
+ input = _prepare_model_input(input)
async def _trace_astream():
start = time.time()
@@ -975,7 +1173,7 @@ async def _trace_astream():
name=f"{model_name}.aresponse_stream",
type=SpanTypeAttribute.LLM,
input=input,
- metadata={**clean_kwargs, **extract_metadata(instance, "model")},
+ metadata={**metadata_extras, **extract_metadata(instance, "model")},
) as span:
first = True
collected_chunks = []
@@ -1002,18 +1200,34 @@ def _get_function_name(instance) -> str:
return "Unknown"
+def _function_call_metadata(instance: Any) -> dict[str, Any]:
+ """Best-effort metadata extraction for a FunctionCall. Contains instrumentation errors."""
+ metadata: dict[str, Any] = {}
+ try:
+ metadata["name"] = instance.function.name
+ except Exception:
+ pass
+ try:
+ metadata["entrypoint"] = instance.function.entrypoint.__name__
+ except Exception:
+ pass
+ try:
+ entrypoint_args = instance._build_entrypoint_args()
+ if entrypoint_args:
+ metadata.update(entrypoint_args)
+ except Exception:
+ pass
+ return metadata
+
+
def _function_call_execute_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
function_name = _get_function_name(instance)
- entrypoint_args = instance._build_entrypoint_args()
+ metadata = _function_call_metadata(instance)
with start_span(
name=f"{function_name}.execute",
type=SpanTypeAttribute.TOOL,
- input=(instance.arguments or {}),
- metadata={
- "name": instance.function.name,
- "entrypoint": instance.function.entrypoint.__name__,
- **(entrypoint_args or {}),
- },
+ input=(getattr(instance, "arguments", None) or {}),
+ metadata=metadata,
) as span:
result = wrapped(*args, **kwargs)
span.log(output=result)
@@ -1022,16 +1236,12 @@ def _function_call_execute_wrapper(wrapped: Any, instance: Any, args: Any, kwarg
async def _function_call_aexecute_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
function_name = _get_function_name(instance)
- entrypoint_args = instance._build_entrypoint_args()
+ metadata = _function_call_metadata(instance)
with start_span(
name=f"{function_name}.aexecute",
type=SpanTypeAttribute.TOOL,
- input=(instance.arguments or {}),
- metadata={
- "name": instance.function.name,
- "entrypoint": instance.function.entrypoint.__name__,
- **(entrypoint_args or {}),
- },
+ input=(getattr(instance, "arguments", None) or {}),
+ metadata=metadata,
) as span:
result = await wrapped(*args, **kwargs)
span.log(output=result)
@@ -1125,7 +1335,7 @@ def _trace_stream():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -1183,7 +1393,7 @@ async def _trace_stream():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -1230,7 +1440,7 @@ def _trace_stream():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -1244,7 +1454,7 @@ def _trace_stream():
span.end()
return result
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
span.unset_current()
span.end()
raise
@@ -1287,7 +1497,7 @@ async def _trace_stream():
should_unset = False
raise
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
raise
finally:
if should_unset:
@@ -1301,7 +1511,7 @@ async def _trace_stream():
span.end()
return result
except Exception as e:
- span.log(error=str(e))
+ span.log(error=e)
span.unset_current()
span.end()
raise
diff --git a/py/src/braintrust/test_vcr_config.py b/py/src/braintrust/test_vcr_config.py
new file mode 100644
index 00000000..e70ee712
--- /dev/null
+++ b/py/src/braintrust/test_vcr_config.py
@@ -0,0 +1,19 @@
+from braintrust.conftest import get_vcr_config
+
+
+def test_vcr_config_scrubs_sensitive_provider_headers():
+ config = get_vcr_config()
+
+ assert {"cookie", "openai-organization", "openai-project"} <= set(config["filter_headers"])
+
+ response = {
+ "headers": {
+ "Content-Type": ["application/json"],
+ "OpenAI-Organization": ["org-sensitive"],
+ "OpenAI-Project": ["proj-sensitive"],
+ "Set-Cookie": ["session=sensitive"],
+ }
+ }
+ scrubbed = config["before_record_response"](response)
+
+ assert scrubbed["headers"] == {"Content-Type": ["application/json"]}