diff --git a/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java b/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java index 1cb5a4ee..3618fd70 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java @@ -461,11 +461,6 @@ private void handleStreamingEval( taskSpan.end(); evalSpan.setStatus( StatusCode.ERROR, e.getMessage()); - evalSpan.setAttribute( - "braintrust.output_json", - toJson( - Collections.singletonMap( - "output", null))); log.debug( "Task threw exception for input: " + datasetCase.input(), @@ -681,7 +676,7 @@ private void setEvalSpanAttributes( } evalSpan.setAttribute(PARENT, braintrustParent.toParentValue()) .setAttribute("braintrust.span_attributes", toJson(spanAttrs)) - .setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input()))) + .setAttribute("braintrust.input_json", toJson(datasetCase.input())) .setAttribute("braintrust.expected_json", toJson(datasetCase.expected())); if (datasetCase.origin().isPresent()) { @@ -694,8 +689,7 @@ private void setEvalSpanAttributes( if (!datasetCase.metadata().isEmpty()) { evalSpan.setAttribute("braintrust.metadata", toJson(datasetCase.metadata())); } - evalSpan.setAttribute( - "braintrust.output_json", toJson(Map.of("output", taskResult.result()))); + evalSpan.setAttribute("braintrust.output_json", toJson(taskResult.result())); } /** @@ -715,7 +709,7 @@ private void setEvalSpanAttributesForError( } evalSpan.setAttribute(PARENT, braintrustParent.toParentValue()) .setAttribute("braintrust.span_attributes", toJson(spanAttrs)) - .setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input()))) + .setAttribute("braintrust.input_json", toJson(datasetCase.input())) .setAttribute("braintrust.expected_json", toJson(datasetCase.expected())); if (datasetCase.origin().isPresent()) { @@ -745,9 +739,9 @@ private void setTaskSpanAttributes( taskSpan.setAttribute(PARENT, braintrustParent.toParentValue()) .setAttribute("braintrust.span_attributes", toJson(taskSpanAttrs)) - .setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input()))) - .setAttribute( - "braintrust.output_json", toJson(Map.of("output", taskResult.result()))); + .setAttribute("braintrust.input_json", toJson(datasetCase.input())) + .setAttribute("braintrust.expected_json", toJson(datasetCase.expected())) + .setAttribute("braintrust.output_json", toJson(taskResult.result())); } private void setScoreSpanAttributes( diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java index fc9e12c0..ed3613a2 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java @@ -120,10 +120,8 @@ private void evalOne(String experimentId, DatasetCase datasetCase .setSpanKind(SpanKind.CLIENT) .setAttribute(PARENT, "experiment_id:" + experimentId) .setAttribute("braintrust.span_attributes", toJson(Map.of("type", "eval"))) - .setAttribute( - "braintrust.input_json", - toJson(Map.of("input", datasetCase.input()))) - .setAttribute("braintrust.expected", toJson(datasetCase.expected())) + .setAttribute("braintrust.input_json", toJson(datasetCase.input())) + .setAttribute("braintrust.expected_json", toJson(datasetCase.expected())) .startSpan(); if (datasetCase.origin().isPresent()) { rootSpan.setAttribute("braintrust.origin", toJson(datasetCase.origin().get())); @@ -146,22 +144,21 @@ private void evalOne(String experimentId, DatasetCase datasetCase .setAttribute( "braintrust.span_attributes", toJson(Map.of("type", "task"))) + .setAttribute("braintrust.input_json", toJson(datasetCase.input())) + .setAttribute( + "braintrust.expected_json", toJson(datasetCase.expected())) .startSpan(); taskSpanId = taskSpan.getSpanContext().getSpanId(); try (var unused = BraintrustContext.ofExperiment(experimentId, taskSpan).makeCurrent()) { taskResult = task.apply(datasetCase, parameters); - rootSpan.setAttribute( - "braintrust.output_json", - toJson(Map.of("output", taskResult.result()))); + taskSpan.setAttribute("braintrust.output_json", toJson(taskResult.result())); + rootSpan.setAttribute("braintrust.output_json", toJson(taskResult.result())); } catch (Exception e) { taskSpan.setStatus(StatusCode.ERROR, e.getMessage()); taskSpan.recordException(e); taskSpan.end(); rootSpan.setStatus(StatusCode.ERROR, e.getMessage()); - rootSpan.setAttribute( - "braintrust.output_json", - toJson(Collections.singletonMap("output", null))); log.debug("Task threw exception for input: " + datasetCase.input(), e); // run scoreForTaskException on each scorer for (var scorer : scorers) { diff --git a/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java index 173fdb69..f3ad627c 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java @@ -534,6 +534,10 @@ void testStreamingEval() throws Exception { io.opentelemetry.api.common.AttributeKey.stringKey( "braintrust.input_json")); assertNotNull(inputJson, "Eval span should have input_json"); + JsonNode input = JSON_MAPPER.readTree(inputJson); + assertTrue( + input.isTextual(), + "Eval span input_json should be the raw input value, not a wrapper object"); String expectedJson = evalSpan.getAttributes() @@ -541,6 +545,10 @@ void testStreamingEval() throws Exception { io.opentelemetry.api.common.AttributeKey.stringKey( "braintrust.expected_json")); assertNotNull(expectedJson, "Eval span should have expected_json"); + JsonNode expected = JSON_MAPPER.readTree(expectedJson); + assertTrue( + expected.isTextual(), + "Eval span expected_json should be the raw expected value"); String outputJson = evalSpan.getAttributes() @@ -549,7 +557,7 @@ void testStreamingEval() throws Exception { "braintrust.output_json")); assertNotNull(outputJson, "Eval span should have output_json"); JsonNode output = JSON_MAPPER.readTree(outputJson); - assertEquals("java-fruit", output.get("output").asText()); + assertEquals("java-fruit", output.asText()); } for (SpanData taskSpan : taskSpans) { @@ -578,6 +586,21 @@ void testStreamingEval() throws Exception { io.opentelemetry.api.common.AttributeKey.stringKey( "braintrust.input_json")); assertNotNull(inputJson, "Task span should have input_json"); + JsonNode input = JSON_MAPPER.readTree(inputJson); + assertTrue( + input.isTextual(), + "Task span input_json should be the raw input value, not a wrapper object"); + + String expectedJson = + taskSpan.getAttributes() + .get( + io.opentelemetry.api.common.AttributeKey.stringKey( + "braintrust.expected_json")); + assertNotNull(expectedJson, "Task span should have expected_json"); + JsonNode expected = JSON_MAPPER.readTree(expectedJson); + assertTrue( + expected.isTextual(), + "Task span expected_json should be the raw expected value"); String outputJson = taskSpan.getAttributes() @@ -586,7 +609,7 @@ void testStreamingEval() throws Exception { "braintrust.output_json")); assertNotNull(outputJson, "Task span should have output_json"); JsonNode output = JSON_MAPPER.readTree(outputJson); - assertEquals("java-fruit", output.get("output").asText()); + assertEquals("java-fruit", output.asText()); } for (SpanData scoreSpan : scoreSpans) { @@ -762,8 +785,8 @@ void testExperimentEval() throws Exception { assertFalse( spanAttrs.has("generation"), "experiment spans should not carry generation"); assertNotNull( - evalSpan.getAttributes().get(AttributeKey.stringKey("braintrust.expected")), - "standard Eval decorator should set braintrust.expected"); + evalSpan.getAttributes() + .get(AttributeKey.stringKey("braintrust.expected_json"))); } } @@ -1039,15 +1062,11 @@ void testTaskErrorHandling() throws Exception { erroredEvalSpan.getStatus().getDescription().contains("task failed on bad-input"), "eval span error should contain the exception message"); - // The errored eval span should have output: null - @SuppressWarnings("unchecked") - Map erroredOutputJson = - fromJson( - erroredEvalSpan - .getAttributes() - .get(AttributeKey.stringKey("braintrust.output_json")), - Map.class); - assertNull(erroredOutputJson.get("output"), "errored case output should be null"); + assertNull( + erroredEvalSpan + .getAttributes() + .get(AttributeKey.stringKey("braintrust.output_json")), + "errored eval span should not have output_json"); // Find the task span for the errored case (should have ERROR status) var erroredTaskSpan = @@ -1105,15 +1124,12 @@ void testTaskErrorHandling() throws Exception { .filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR) .findFirst() .orElseThrow(() -> new AssertionError("expected a successful eval span")); - @SuppressWarnings("unchecked") - Map successOutputJson = - fromJson( + JsonNode successOutputJson = + JSON_MAPPER.readTree( successEvalSpan .getAttributes() - .get(AttributeKey.stringKey("braintrust.output_json")), - Map.class); - assertEquals( - "result", successOutputJson.get("output"), "successful case should have output"); + .get(AttributeKey.stringKey("braintrust.output_json"))); + assertEquals("result", successOutputJson.asText(), "successful case should have output"); } @Test diff --git a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java index c4cd1f00..9c63549c 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java @@ -86,28 +86,45 @@ public void evalOtelTraceWithProperAttributes() { "all eval spans must set the parent to the experiment id"); if (span.getParentSpanId().equals(SpanId.getInvalid())) { numRootSpans.incrementAndGet(); - var inputJson = + var input = fromJson( span.getAttributes() .get(AttributeKey.stringKey("braintrust.input_json")), - Map.class); - assertNotNull(inputJson.get("input"), "invlaid input: " + inputJson); + String.class); + assertTrue( + input.equals("strawberry") || input.equals("asparagus"), + "invalid input: " + input); var expected = fromJson( span.getAttributes() - .get(AttributeKey.stringKey("braintrust.expected")), + .get(AttributeKey.stringKey("braintrust.expected_json")), String.class); assertTrue(isFruitOrVegetable(expected), "invalid expected: " + expected); - var outputJson = + var output = fromJson( span.getAttributes() .get(AttributeKey.stringKey("braintrust.output_json")), - Map.class); - var output = outputJson.get("output"); - assertNotNull(output, "invlaid output: " + outputJson); - assertTrue(isFruitOrVegetable(String.valueOf(output)), "invalid output: " + output); + String.class); + assertNotNull(output, "invalid output: " + output); + assertTrue(isFruitOrVegetable(output), "invalid output: " + output); + } else if ("task".equals(span.getName())) { + // task span carries its own bare input/output + var input = + fromJson( + span.getAttributes() + .get(AttributeKey.stringKey("braintrust.input_json")), + String.class); + assertTrue( + input.equals("strawberry") || input.equals("asparagus"), + "invalid task input: " + input); + var output = + fromJson( + span.getAttributes() + .get(AttributeKey.stringKey("braintrust.output_json")), + String.class); + assertTrue(isFruitOrVegetable(output), "invalid task output: " + output); } } assertEquals(2, numRootSpans.get(), "each case should make a root span"); @@ -315,8 +332,7 @@ public void evalRootSpanPassesOriginIfPresent() { var inputJson = span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json")); assertNotNull(inputJson); - fromJson(inputJson, Map.class); - var input = (String) (fromJson(inputJson, Map.class)).get("input"); + var input = fromJson(inputJson, String.class); assertNotNull(input); var origin = span.getAttributes().get(AttributeKey.stringKey("braintrust.origin")); switch (input) { @@ -481,14 +497,11 @@ void evalContinuesWhenTaskThrows() { erroredRootSpan.getStatus().getDescription().contains("task failed on bad-input"), "root span error should contain the exception message"); - // The errored root span should have output: null - var erroredOutputJson = - fromJson( - erroredRootSpan - .getAttributes() - .get(AttributeKey.stringKey("braintrust.output_json")), - Map.class); - assertNull(erroredOutputJson.get("output"), "errored case output should be null"); + assertNull( + erroredRootSpan + .getAttributes() + .get(AttributeKey.stringKey("braintrust.output_json")), + "errors should not set output json"); // Find the task span for the errored case (child of errored root, type=task, status=ERROR) var erroredTaskSpan = @@ -519,6 +532,11 @@ void evalContinuesWhenTaskThrows() { assertTrue( erroredTaskSpan.getEvents().stream().anyMatch(e -> e.getName().equals("exception")), "task span should have an exception event"); + assertNull( + erroredTaskSpan + .getAttributes() + .get(AttributeKey.stringKey("braintrust.output_json")), + "errors should not set output json"); // The errored case should still have score spans (from scoreForTaskException default = 0.0) var erroredScoreSpans = @@ -560,14 +578,13 @@ void evalContinuesWhenTaskThrows() { .filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR) .findFirst() .orElseThrow(() -> new AssertionError("expected a successful root span")); - var successOutputJson = + var successOutput = fromJson( successRootSpan .getAttributes() .get(AttributeKey.stringKey("braintrust.output_json")), - Map.class); - assertEquals( - "result", successOutputJson.get("output"), "successful case should have output"); + String.class); + assertEquals("result", successOutput, "successful case should have output"); } @Test