From f2d59b48f68b6d96a2c33f96b04878f91027622f Mon Sep 17 00:00:00 2001 From: Starfolk Date: Thu, 16 Jul 2026 22:37:07 +0000 Subject: [PATCH 1/2] =?UTF-8?q?fix(bedrock=5Fruntime):=20align=20spans=20w?= =?UTF-8?q?ith=20SKILL.md=20=E2=80=94=20tools,=20attachments,=20latency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `bedrock_latency_ms` moves from `metrics` to `metadata` since it's not on the spec's metric allowlist. - `toolConfig` is normalized into OpenAI-shaped `metadata.tools` and `metadata.tool_choice` (was previously dumped as a raw `metadata.tool_config` blob). - Image and document content parts now emit the spec shapes (`{"type": "image_url", "image_url": {"url": Attachment}}` and `{"type": "file", "file": {"file_data": Attachment, "filename": ...}}`) and drop the redundant `source` field once the payload has been materialized into an `Attachment`. Provider-native shapes (e.g. `s3Location` refs) still pass through unchanged. - Adds VCR-backed tests for tool use, image input, and document input (cassettes still to be recorded). Co-Authored-By: Claude Opus 4.7 --- .../bedrock_runtime/test_bedrock_runtime.py | 169 +++++++++++++++++- .../integrations/bedrock_runtime/tracing.py | 137 ++++++++++---- 2 files changed, 270 insertions(+), 36 deletions(-) diff --git a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py index 583b9f86..7f04fa10 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py +++ b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py @@ -1,12 +1,13 @@ """Tests for the boto3 Bedrock Runtime integration.""" +import base64 import inspect import json import os import time import pytest -from braintrust import logger +from braintrust import Attachment, logger from braintrust.integrations.bedrock_runtime import BedrockRuntimeIntegration, setup_bedrock, wrap_bedrock from braintrust.integrations.bedrock_runtime.patchers import ( BedrockClientCreatorPatcher, @@ -266,6 +267,172 @@ def test_wrap_bedrock_invoke_model_preserves_response_body_and_logs_json(memory_ assert_metrics_are_valid(span["metrics"], start, end) +# A 1x1 transparent PNG for image tests. +_TINY_PNG_BYTES = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" +) + +# A minimal valid PDF for document tests. +_TINY_PDF_BYTES = ( + b"%PDF-1.1\n1 0 obj<>endobj\n" + b"2 0 obj<>endobj\n" + b"3 0 obj<>endobj\n" + b"xref\n0 4\n0000000000 65535 f\n" + b"0000000010 00000 n\n0000000053 00000 n\n0000000096 00000 n\n" + b"trailer<>\nstartxref\n144\n%%EOF" +) + + +_WEATHER_TOOL = { + "toolSpec": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "inputSchema": { + "json": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + }, + "required": ["city"], + } + }, + } +} + + +def _converse_tool_kwargs(model_id=CONVERSE_MODEL): + return { + "modelId": model_id, + "messages": [ + {"role": "user", "content": [{"text": "What's the weather in Paris? Use the tool."}]} + ], + "inferenceConfig": {"maxTokens": 200, "temperature": 0}, + "toolConfig": { + "tools": [_WEATHER_TOOL], + "toolChoice": {"tool": {"name": "get_weather"}}, + }, + } + + +@pytest.mark.vcr +def test_wrap_bedrock_converse_normalizes_tool_config_and_output(memory_logger): + assert not memory_logger.pop() + client = wrap_bedrock(_bedrock_client()) + + response = client.converse(**_converse_tool_kwargs()) + + assert response["output"]["message"]["role"] == "assistant" + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + + # Tools are normalized to OpenAI shape and live in metadata. + assert span["metadata"]["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string", "description": "City name"}}, + "required": ["city"], + }, + }, + } + ] + assert span["metadata"]["tool_choice"] == {"type": "function", "function": {"name": "get_weather"}} + # No raw Bedrock-shaped copy left behind. + assert "tool_config" not in span["metadata"] + + # Output tool_use block is normalized. + content = span["output"][0]["content"] + tool_uses = [block for block in content if block.get("type") == "tool_use"] + assert tool_uses, f"expected a tool_use block in {content!r}" + tool_use = tool_uses[0] + assert tool_use.get("name") == "get_weather" + assert "id" in tool_use + assert isinstance(tool_use.get("input"), dict) + + +@pytest.mark.vcr +def test_wrap_bedrock_converse_with_image_input_materializes_attachment(memory_logger): + assert not memory_logger.pop() + client = wrap_bedrock(_bedrock_client()) + + kwargs = { + "modelId": CONVERSE_MODEL, + "messages": [ + { + "role": "user", + "content": [ + {"text": "Describe this image in one word."}, + {"image": {"format": "png", "source": {"bytes": _TINY_PNG_BYTES}}}, + ], + } + ], + "inferenceConfig": {"maxTokens": 20, "temperature": 0}, + } + response = client.converse(**kwargs) + assert response["output"]["message"]["role"] == "assistant" + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + + parts = span["input"][0]["content"] + image_part = next(p for p in parts if p.get("type") == "image_url") + assert image_part["format"] == "png" + assert isinstance(image_part["image_url"]["url"], Attachment) + assert image_part["image_url"]["url"].reference["content_type"] == "image/png" + # No raw bytes hanging around next to the attachment. + assert "source" not in image_part + # Original request kwargs are not mutated. + assert kwargs["messages"][0]["content"][1]["image"]["source"]["bytes"] == _TINY_PNG_BYTES + + +@pytest.mark.vcr +def test_wrap_bedrock_converse_with_document_input_materializes_attachment(memory_logger): + assert not memory_logger.pop() + client = wrap_bedrock(_bedrock_client()) + + kwargs = { + "modelId": CONVERSE_MODEL, + "messages": [ + { + "role": "user", + "content": [ + {"text": "Summarize this document."}, + { + "document": { + "format": "pdf", + "name": "tiny.pdf", + "source": {"bytes": _TINY_PDF_BYTES}, + } + }, + ], + } + ], + "inferenceConfig": {"maxTokens": 40, "temperature": 0}, + } + response = client.converse(**kwargs) + assert response["output"]["message"]["role"] == "assistant" + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + + parts = span["input"][0]["content"] + file_part = next(p for p in parts if p.get("type") == "file") + assert file_part["format"] == "pdf" + assert file_part["name"] == "tiny.pdf" + assert file_part["file"]["filename"] == "tiny.pdf" + assert isinstance(file_part["file"]["file_data"], Attachment) + assert file_part["file"]["file_data"].reference["content_type"] == "application/pdf" + assert "source" not in file_part + + @pytest.mark.vcr def test_auto_instrument_bedrock_runtime_subprocess(): verify_autoinstrument_script("test_auto_bedrock_runtime.py", timeout=60) diff --git a/py/src/braintrust/integrations/bedrock_runtime/tracing.py b/py/src/braintrust/integrations/bedrock_runtime/tracing.py index fee3f1f1..c778e359 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/tracing.py +++ b/py/src/braintrust/integrations/bedrock_runtime/tracing.py @@ -32,7 +32,6 @@ def start_span(*args, **kwargs): _CONVERSE_METADATA_KEYS = ( "guardrailConfig", - "toolConfig", "additionalModelRequestFields", "additionalModelResponseFieldPaths", "performanceConfig", @@ -63,7 +62,6 @@ def _converse_wrapper(wrapped, instance, args, kwargs): # noqa: ARG001 metrics = { **_timing_metrics(start_time, time.time()), **_converse_usage_metrics(result.get("usage") if isinstance(result, dict) else None), - **_bedrock_latency_metrics(result.get("metrics") if isinstance(result, dict) else None), } _log_and_end_span(span, output=_converse_output(result), metrics=metrics, metadata=metadata or None) return result @@ -160,6 +158,9 @@ def _converse_metadata(kwargs: dict[str, Any], *, endpoint: str, stream: bool = value = inference_config.get(source_key) if value is not None: metadata[dest_key] = value + tool_config = kwargs.get("toolConfig") + if isinstance(tool_config, dict): + metadata.update(_tool_config_to_metadata(tool_config)) for key in _CONVERSE_METADATA_KEYS: value = kwargs.get(key) if value is not None: @@ -169,6 +170,55 @@ def _converse_metadata(kwargs: dict[str, Any], *, endpoint: str, stream: bool = return metadata +def _tool_config_to_metadata(tool_config: dict[str, Any]) -> dict[str, Any]: + """Normalize a Bedrock Converse ``toolConfig`` into OpenAI-shaped metadata.""" + result: dict[str, Any] = {} + tools = tool_config.get("tools") + if isinstance(tools, list): + openai_tools = [_tool_spec_to_openai(t) for t in tools] + if openai_tools: + result["tools"] = openai_tools + tool_choice = _tool_choice_to_openai(tool_config.get("toolChoice")) + if tool_choice is not None: + result["tool_choice"] = tool_choice + return result + + +def _tool_spec_to_openai(tool: Any) -> Any: + if not isinstance(tool, dict): + return tool + spec = tool.get("toolSpec") + if not isinstance(spec, dict): + return tool + function: dict[str, Any] = {} + name = spec.get("name") + if name is not None: + function["name"] = name + description = spec.get("description") + if description is not None: + function["description"] = description + input_schema = spec.get("inputSchema") + if isinstance(input_schema, dict) and isinstance(input_schema.get("json"), dict): + function["parameters"] = input_schema["json"] + elif input_schema is not None: + function["parameters"] = input_schema + return {"type": "function", "function": function} + + +def _tool_choice_to_openai(tool_choice: Any) -> Any: + if not isinstance(tool_choice, dict): + return None + if "auto" in tool_choice: + return "auto" + if "any" in tool_choice: + return "required" + if isinstance(tool_choice.get("tool"), dict): + name = tool_choice["tool"].get("name") + if isinstance(name, str): + return {"type": "function", "function": {"name": name}} + return tool_choice + + def _invoke_model_metadata(model_id: Any, *, endpoint: str, stream: bool = False) -> dict[str, Any]: metadata: dict[str, Any] = {"provider": _PROVIDER, "endpoint": endpoint} if model_id is not None: @@ -181,8 +231,8 @@ def _invoke_model_metadata(model_id: Any, *, endpoint: str, stream: bool = False def _camel_metadata_key(key: str) -> str: return { "guardrailConfig": "guardrail_config", - "toolConfig": "tool_config", "additionalModelRequestFields": "additional_model_request_fields", + "additionalModelResponseFields": "additional_model_response_fields", "additionalModelResponseFieldPaths": "additional_model_response_field_paths", "performanceConfig": "performance_config", "requestMetadata": "request_metadata", @@ -286,44 +336,54 @@ def _tool_result_content_to_json(block: Any) -> Any: def _image_to_json(value: Any) -> dict[str, Any]: if not isinstance(value, dict): return {"type": "image", "value": value} - result: dict[str, Any] = {"type": "image"} image_format = value.get("format") + source = value.get("source") + if isinstance(source, dict) and isinstance(source.get("bytes"), (bytes, bytearray)): + mime_type = f"image/{image_format or 'png'}" + resolved = _materialize_attachment(source["bytes"], mime_type=mime_type, prefix="image") + if resolved is not None: + result: dict[str, Any] = {"type": "image_url", **resolved.multimodal_part_payload} + if image_format is not None: + result["format"] = image_format + return result + # Preserve provider-native shape (e.g. s3Location references) when we can't materialize. + result = {"type": "image"} if image_format is not None: result["format"] = image_format - source = value.get("source") - if isinstance(source, dict): - source_result = dict(source) - image_bytes = source_result.pop("bytes", None) - if image_bytes is not None: - mime_type = f"image/{image_format or 'png'}" - resolved = _materialize_attachment(image_bytes, mime_type=mime_type, prefix="image") - if resolved is not None: - result["image_url"] = {"url": resolved.attachment} - else: - source_result["bytes"] = image_bytes - result["source"] = source_result + if source is not None: + result["source"] = source return result def _document_to_json(value: Any) -> dict[str, Any]: if not isinstance(value, dict): return {"type": "document", "value": value} - result: dict[str, Any] = {"type": "document"} + document_format = value.get("format") + name = value.get("name") if isinstance(value.get("name"), str) else None + source = value.get("source") + if isinstance(source, dict) and isinstance(source.get("bytes"), (bytes, bytearray)): + mime_type = _document_mime_type(document_format) + resolved = _materialize_attachment( + source["bytes"], + mime_type=mime_type, + filename=name, + prefix="document", + ) + if resolved is not None: + result: dict[str, Any] = {"type": "file", **resolved.multimodal_part_payload} + if document_format is not None: + result["format"] = document_format + if value.get("name") is not None: + result["name"] = value["name"] + if value.get("citations") is not None: + result["citations"] = value["citations"] + return result + result = {"type": "document"} for key in ("format", "name", "citations"): if value.get(key) is not None: result[key] = value[key] - source = value.get("source") - if isinstance(source, dict): - source_result = dict(source) - document_bytes = source_result.pop("bytes", None) - if document_bytes is not None: - mime_type = _document_mime_type(value.get("format")) - resolved = _materialize_attachment(document_bytes, mime_type=mime_type, prefix="document") - if resolved is not None: - result["file"] = {"file_data": resolved.attachment, "filename": resolved.filename} - else: - source_result["bytes"] = document_bytes - result["source"] = source_result + if source is not None: + result["source"] = source return result @@ -373,6 +433,9 @@ def _converse_response_metadata(result: Any) -> dict[str, Any]: for key in ("additionalModelResponseFields", "trace"): if result.get(key) is not None: metadata[_camel_metadata_key(key)] = result[key] + latency = _bedrock_latency_ms(result.get("metrics")) + if latency is not None: + metadata["bedrock_latency_ms"] = latency return metadata @@ -398,11 +461,11 @@ def _converse_usage_metrics(usage: Any) -> dict[str, Any]: return metrics -def _bedrock_latency_metrics(metrics_payload: Any) -> dict[str, Any]: +def _bedrock_latency_ms(metrics_payload: Any) -> Any: if not isinstance(metrics_payload, dict): - return {} + return None latency = metrics_payload.get("latencyMs") - return {"bedrock_latency_ms": latency} if is_numeric(latency) else {} + return latency if is_numeric(latency) else None def _numeric_or_zero(value: Any) -> Any: @@ -598,13 +661,17 @@ def _finish(self, error: BaseException | None) -> None: block["text"] = text content = [self._content_blocks[idx] for idx in sorted(self._content_blocks)] output = [{"role": self._message_role, "content": content}] if content else None - metadata = {"stop_reason": self._stop_reason} if self._stop_reason else None + metadata: dict[str, Any] = {} + if self._stop_reason: + metadata["stop_reason"] = self._stop_reason + latency = _bedrock_latency_ms(self._metrics_payload) + if latency is not None: + metadata["bedrock_latency_ms"] = latency metrics = { **_timing_metrics(self._start_time, time.time(), self._first_token_time), **_converse_usage_metrics(self._usage), - **_bedrock_latency_metrics(self._metrics_payload), } - _log_and_end_span(self._span, output=output, metrics=metrics, metadata=metadata) + _log_and_end_span(self._span, output=output, metrics=metrics, metadata=metadata or None) class _TracedInvokeModelStream: From acc7d90dfec7d2123aaa98c874f4f9ed2836da04 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 16 Jul 2026 20:04:16 -0400 Subject: [PATCH 2/2] record cassettes --- ...rse_normalizes_tool_config_and_output.yaml | 39 +++++++++++++++++++ ...h_image_input_materializes_attachment.yaml | 36 +++++++++++++++++ ...rse_normalizes_tool_config_and_output.yaml | 39 +++++++++++++++++++ ...ocument_input_materializes_attachment.yaml | 38 ++++++++++++++++++ ...h_image_input_materializes_attachment.yaml | 36 +++++++++++++++++ .../bedrock_runtime/test_bedrock_runtime.py | 12 +++--- 6 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_normalizes_tool_config_and_output.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_normalizes_tool_config_and_output.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_with_document_input_materializes_attachment.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_normalizes_tool_config_and_output.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_normalizes_tool_config_and_output.yaml new file mode 100644 index 00000000..de5a0ef6 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_normalizes_tool_config_and_output.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "What''s the weather + in Paris? Use the tool."}]}], "inferenceConfig": {"maxTokens": 200, "temperature": + 0}, "toolConfig": {"tools": [{"toolSpec": {"name": "get_weather", "description": + "Get the current weather for a city.", "inputSchema": {"json": {"type": "object", + "properties": {"city": {"type": "string", "description": "City name"}}, "required": + ["city"]}}}}], "toolChoice": {"tool": {"name": "get_weather"}}}}' + headers: + Content-Length: + - '466' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS4zNC4xMTYgbWQvQm90b2NvcmUjMS4zNC4xMTYgdWEvMi4wIG9zL21hY29zIzI1LjUu + MCBtZC9hcmNoI2FybTY0IGxhbmcvcHl0aG9uIzMuMTQuNiBtZC9weWltcGwjQ1B5dGhvbiBjZmcv + cmV0cnktbW9kZSNsZWdhY3kgQm90b2NvcmUvMS4zNC4xMTY= + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":718},"output":{"message":{"content":[{"toolUse":{"input":{"city":"Paris"},"name":"get_weather","toolUseId":"tooluse_0sfJW3PuOzXeEWkRdpK5Rl"}}],"role":"assistant"}},"stopReason":"tool_use","usage":{"inputTokens":411,"outputTokens":16,"serverToolUsage":{},"totalTokens":427}}' + headers: + Connection: + - keep-alive + Content-Length: + - '297' + Content-Type: + - application/json + Date: + - Thu, 16 Jul 2026 23:59:02 GMT + x-amzn-RequestId: + - aadac6be-089d-4be7-b2bc-108426aeab7f + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml new file mode 100644 index 00000000..4cf8b7da --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml @@ -0,0 +1,36 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Describe this image + in one word."}, {"image": {"format": "png", "source": {"bytes": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="}}}]}], + "inferenceConfig": {"maxTokens": 20, "temperature": 0}}' + headers: + Content-Length: + - '294' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS4zNC4xMTYgbWQvQm90b2NvcmUjMS4zNC4xMTYgdWEvMi4wIG9zL21hY29zIzI1LjUu + MCBtZC9hcmNoI2FybTY0IGxhbmcvcHl0aG9uIzMuMTQuNiBtZC9weWltcGwjQ1B5dGhvbiBjZmcv + cmV0cnktbW9kZSNsZWdhY3kgQm90b2NvcmUvMS4zNC4xMTY= + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":561},"output":{"message":{"content":[{"text":"Serene"}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":537,"outputTokens":3,"serverToolUsage":{},"totalTokens":540}}' + headers: + Connection: + - keep-alive + Content-Length: + - '209' + Content-Type: + - application/json + Date: + - Thu, 16 Jul 2026 23:59:03 GMT + x-amzn-RequestId: + - af123bb1-9f24-4913-bd80-e3966474b7cf + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_normalizes_tool_config_and_output.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_normalizes_tool_config_and_output.yaml new file mode 100644 index 00000000..1ba490ee --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_normalizes_tool_config_and_output.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "What''s the weather + in Paris? Use the tool."}]}], "inferenceConfig": {"maxTokens": 200, "temperature": + 0}, "toolConfig": {"tools": [{"toolSpec": {"name": "get_weather", "description": + "Get the current weather for a city.", "inputSchema": {"json": {"type": "object", + "properties": {"city": {"type": "string", "description": "City name"}}, "required": + ["city"]}}}}], "toolChoice": {"tool": {"name": "get_weather"}}}}' + headers: + Content-Length: + - '466' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My40NiBtZC9Cb3RvY29yZSMxLjQzLjQ2IHVhLzIuMSBvcy9tYWNvcyMyNS41LjAg + bWQvYXJjaCNhcm02NCBsYW5nL3B5dGhvbiMzLjE0LjYgbWQvcHlpbXBsI0NQeXRob24gbS8zLFos + YixEIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3RvY29yZS8xLjQzLjQ2 + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":592},"output":{"message":{"content":[{"toolUse":{"input":{"city":"Paris"},"name":"get_weather","toolUseId":"tooluse_Yq5W3hF35lip9DBOtn4cFO"}}],"role":"assistant"}},"stopReason":"tool_use","usage":{"inputTokens":411,"outputTokens":16,"serverToolUsage":{},"totalTokens":427}}' + headers: + Connection: + - keep-alive + Content-Length: + - '297' + Content-Type: + - application/json + Date: + - Thu, 16 Jul 2026 23:55:11 GMT + x-amzn-RequestId: + - 17e70e3c-ae3e-4a8d-8ba7-1a773b61e39f + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_with_document_input_materializes_attachment.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_with_document_input_materializes_attachment.yaml new file mode 100644 index 00000000..bd86a278 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_with_document_input_materializes_attachment.yaml @@ -0,0 +1,38 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Summarize this document."}, + {"document": {"format": "pdf", "name": "tiny-pdf", "source": {"bytes": "JVBERi0xLjEKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBvYmo8PC9UeXBlL1BhZ2VzL0tpZHNbMyAwIFJdL0NvdW50IDE+PmVuZG9iagozIDAgb2JqPDwvVHlwZS9QYWdlL1BhcmVudCAyIDAgUi9NZWRpYUJveFswIDAgOCA4XT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDEwIDAwMDAwIG4KMDAwMDAwMDA1MyAwMDAwMCBuCjAwMDAwMDAwOTYgMDAwMDAgbgp0cmFpbGVyPDwvU2l6ZSA0L1Jvb3QgMSAwIFI+PgpzdGFydHhyZWYKMTQ0CiUlRU9G"}}}]}], + "inferenceConfig": {"maxTokens": 40, "temperature": 0}}' + headers: + Content-Length: + - '609' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My40NiBtZC9Cb3RvY29yZSMxLjQzLjQ2IHVhLzIuMSBvcy9tYWNvcyMyNS41LjAg + bWQvYXJjaCNhcm02NCBsYW5nL3B5dGhvbiMzLjE0LjYgbWQvcHlpbXBsI0NQeXRob24gbS9iLDMs + RCxaIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3RvY29yZS8xLjQzLjQ2 + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":1430},"output":{"message":{"content":[{"text":"Sorry, + I cannot process this request because the document is not provided. Can you + provide more information about the document?"}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":536,"outputTokens":24,"serverToolUsage":{},"totalTokens":560}}' + headers: + Connection: + - keep-alive + Content-Length: + - '332' + Content-Type: + - application/json + Date: + - Thu, 16 Jul 2026 23:57:28 GMT + x-amzn-RequestId: + - d21e398c-670a-46c9-a1a3-e1c8bb8c9e13 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml new file mode 100644 index 00000000..621a6e80 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml @@ -0,0 +1,36 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Describe this image + in one word."}, {"image": {"format": "png", "source": {"bytes": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="}}}]}], + "inferenceConfig": {"maxTokens": 20, "temperature": 0}}' + headers: + Content-Length: + - '294' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My40NiBtZC9Cb3RvY29yZSMxLjQzLjQ2IHVhLzIuMSBvcy9tYWNvcyMyNS41LjAg + bWQvYXJjaCNhcm02NCBsYW5nL3B5dGhvbiMzLjE0LjYgbWQvcHlpbXBsI0NQeXRob24gbS8zLFos + YixEIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3RvY29yZS8xLjQzLjQ2 + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":636},"output":{"message":{"content":[{"text":"Serene"}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":537,"outputTokens":3,"serverToolUsage":{},"totalTokens":540}}' + headers: + Connection: + - keep-alive + Content-Length: + - '209' + Content-Type: + - application/json + Date: + - Thu, 16 Jul 2026 23:55:11 GMT + x-amzn-RequestId: + - 41351ba5-3dcd-4035-a0b2-e5e94a836225 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py index 7f04fa10..ee873f6b 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py +++ b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py @@ -303,9 +303,7 @@ def test_wrap_bedrock_invoke_model_preserves_response_body_and_logs_json(memory_ def _converse_tool_kwargs(model_id=CONVERSE_MODEL): return { "modelId": model_id, - "messages": [ - {"role": "user", "content": [{"text": "What's the weather in Paris? Use the tool."}]} - ], + "messages": [{"role": "user", "content": [{"text": "What's the weather in Paris? Use the tool."}]}], "inferenceConfig": {"maxTokens": 200, "temperature": 0}, "toolConfig": { "tools": [_WEATHER_TOOL], @@ -396,6 +394,8 @@ def test_wrap_bedrock_converse_with_image_input_materializes_attachment(memory_l def test_wrap_bedrock_converse_with_document_input_materializes_attachment(memory_logger): assert not memory_logger.pop() client = wrap_bedrock(_bedrock_client()) + if "document" not in client.meta.service_model.shape_for("ContentBlock").members: + pytest.skip("installed botocore does not support Converse document content blocks") kwargs = { "modelId": CONVERSE_MODEL, @@ -407,7 +407,7 @@ def test_wrap_bedrock_converse_with_document_input_materializes_attachment(memor { "document": { "format": "pdf", - "name": "tiny.pdf", + "name": "tiny-pdf", "source": {"bytes": _TINY_PDF_BYTES}, } }, @@ -426,8 +426,8 @@ def test_wrap_bedrock_converse_with_document_input_materializes_attachment(memor parts = span["input"][0]["content"] file_part = next(p for p in parts if p.get("type") == "file") assert file_part["format"] == "pdf" - assert file_part["name"] == "tiny.pdf" - assert file_part["file"]["filename"] == "tiny.pdf" + assert file_part["name"] == "tiny-pdf" + assert file_part["file"]["filename"] == "tiny-pdf" assert isinstance(file_part["file"]["file_data"], Attachment) assert file_part["file"]["file_data"].reference["content_type"] == "application/pdf" assert "source" not in file_part