Skip to content

update eval span attributes to render input/output like we do for datasets #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Andrew Kent (realark) wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

update eval span attributes to render input/output like we do for datasets #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 6 additions & 12 deletions braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,11 +461,6 @@ private <I, O> void handleStreamingEval(
taskSpan.end();
evalSpan.setStatus(
StatusCode.ERROR, e.getMessage());
evalSpan.setAttribute(
"braintrust.output_json",
toJson(
Collections.singletonMap(
"output", null)));
log.debug(
"Task threw exception for input: "
+ datasetCase.input(),
Expand Down Expand Up @@ -681,7 +676,7 @@ private void setEvalSpanAttributes(
}
evalSpan.setAttribute(PARENT, braintrustParent.toParentValue())
.setAttribute("braintrust.span_attributes", toJson(spanAttrs))
.setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input())))
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()));

if (datasetCase.origin().isPresent()) {
Expand All @@ -694,8 +689,7 @@ private void setEvalSpanAttributes(
if (!datasetCase.metadata().isEmpty()) {
evalSpan.setAttribute("braintrust.metadata", toJson(datasetCase.metadata()));
}
evalSpan.setAttribute(
"braintrust.output_json", toJson(Map.of("output", taskResult.result())));
evalSpan.setAttribute("braintrust.output_json", toJson(taskResult.result()));
}

/**
Expand All @@ -715,7 +709,7 @@ private void setEvalSpanAttributesForError(
}
evalSpan.setAttribute(PARENT, braintrustParent.toParentValue())
.setAttribute("braintrust.span_attributes", toJson(spanAttrs))
.setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input())))
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()));

if (datasetCase.origin().isPresent()) {
Expand Down Expand Up @@ -745,9 +739,9 @@ private void setTaskSpanAttributes(

taskSpan.setAttribute(PARENT, braintrustParent.toParentValue())
.setAttribute("braintrust.span_attributes", toJson(taskSpanAttrs))
.setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input())))
.setAttribute(
"braintrust.output_json", toJson(Map.of("output", taskResult.result())));
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()))
.setAttribute("braintrust.output_json", toJson(taskResult.result()));
}

private void setScoreSpanAttributes(
Expand Down
17 changes: 7 additions & 10 deletions braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,8 @@ private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase
.setSpanKind(SpanKind.CLIENT)
.setAttribute(PARENT, "experiment_id:" + experimentId)
.setAttribute("braintrust.span_attributes", toJson(Map.of("type", "eval")))
.setAttribute(
"braintrust.input_json",
toJson(Map.of("input", datasetCase.input())))
.setAttribute("braintrust.expected", toJson(datasetCase.expected()))
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()))
.startSpan();
if (datasetCase.origin().isPresent()) {
rootSpan.setAttribute("braintrust.origin", toJson(datasetCase.origin().get()));
Expand All @@ -146,22 +144,21 @@ private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase
.setAttribute(
"braintrust.span_attributes",
toJson(Map.of("type", "task")))
.setAttribute("braintrust.input_json", toJson(datasetCase.input()))
.setAttribute(
"braintrust.expected_json", toJson(datasetCase.expected()))
.startSpan();
taskSpanId = taskSpan.getSpanContext().getSpanId();
try (var unused =
BraintrustContext.ofExperiment(experimentId, taskSpan).makeCurrent()) {
taskResult = task.apply(datasetCase, parameters);
rootSpan.setAttribute(
"braintrust.output_json",
toJson(Map.of("output", taskResult.result())));
taskSpan.setAttribute("braintrust.output_json", toJson(taskResult.result()));
rootSpan.setAttribute("braintrust.output_json", toJson(taskResult.result()));
} catch (Exception e) {
taskSpan.setStatus(StatusCode.ERROR, e.getMessage());
taskSpan.recordException(e);
taskSpan.end();
rootSpan.setStatus(StatusCode.ERROR, e.getMessage());
rootSpan.setAttribute(
"braintrust.output_json",
toJson(Collections.singletonMap("output", null)));
log.debug("Task threw exception for input: " + datasetCase.input(), e);
// run scoreForTaskException on each scorer
for (var scorer : scorers) {
Expand Down
56 changes: 36 additions & 20 deletions braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,13 +534,21 @@ void testStreamingEval() throws Exception {
io.opentelemetry.api.common.AttributeKey.stringKey(
"braintrust.input_json"));
assertNotNull(inputJson, "Eval span should have input_json");
JsonNode input = JSON_MAPPER.readTree(inputJson);
assertTrue(
input.isTextual(),
"Eval span input_json should be the raw input value, not a wrapper object");

String expectedJson =
evalSpan.getAttributes()
.get(
io.opentelemetry.api.common.AttributeKey.stringKey(
"braintrust.expected_json"));
assertNotNull(expectedJson, "Eval span should have expected_json");
JsonNode expected = JSON_MAPPER.readTree(expectedJson);
assertTrue(
expected.isTextual(),
"Eval span expected_json should be the raw expected value");

String outputJson =
evalSpan.getAttributes()
Expand All @@ -549,7 +557,7 @@ void testStreamingEval() throws Exception {
"braintrust.output_json"));
assertNotNull(outputJson, "Eval span should have output_json");
JsonNode output = JSON_MAPPER.readTree(outputJson);
assertEquals("java-fruit", output.get("output").asText());
assertEquals("java-fruit", output.asText());
}

for (SpanData taskSpan : taskSpans) {
Expand Down Expand Up @@ -578,6 +586,21 @@ void testStreamingEval() throws Exception {
io.opentelemetry.api.common.AttributeKey.stringKey(
"braintrust.input_json"));
assertNotNull(inputJson, "Task span should have input_json");
JsonNode input = JSON_MAPPER.readTree(inputJson);
assertTrue(
input.isTextual(),
"Task span input_json should be the raw input value, not a wrapper object");

String expectedJson =
taskSpan.getAttributes()
.get(
io.opentelemetry.api.common.AttributeKey.stringKey(
"braintrust.expected_json"));
assertNotNull(expectedJson, "Task span should have expected_json");
JsonNode expected = JSON_MAPPER.readTree(expectedJson);
assertTrue(
expected.isTextual(),
"Task span expected_json should be the raw expected value");

String outputJson =
taskSpan.getAttributes()
Expand All @@ -586,7 +609,7 @@ void testStreamingEval() throws Exception {
"braintrust.output_json"));
assertNotNull(outputJson, "Task span should have output_json");
JsonNode output = JSON_MAPPER.readTree(outputJson);
assertEquals("java-fruit", output.get("output").asText());
assertEquals("java-fruit", output.asText());
}

for (SpanData scoreSpan : scoreSpans) {
Expand Down Expand Up @@ -762,8 +785,8 @@ void testExperimentEval() throws Exception {
assertFalse(
spanAttrs.has("generation"), "experiment spans should not carry generation");
assertNotNull(
evalSpan.getAttributes().get(AttributeKey.stringKey("braintrust.expected")),
"standard Eval decorator should set braintrust.expected");
evalSpan.getAttributes()
.get(AttributeKey.stringKey("braintrust.expected_json")));
}
}

Expand Down Expand Up @@ -1039,15 +1062,11 @@ void testTaskErrorHandling() throws Exception {
erroredEvalSpan.getStatus().getDescription().contains("task failed on bad-input"),
"eval span error should contain the exception message");

// The errored eval span should have output: null
@SuppressWarnings("unchecked")
Map<String, Object> erroredOutputJson =
fromJson(
erroredEvalSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
Map.class);
assertNull(erroredOutputJson.get("output"), "errored case output should be null");
assertNull(
erroredEvalSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
"errored eval span should not have output_json");

// Find the task span for the errored case (should have ERROR status)
var erroredTaskSpan =
Expand Down Expand Up @@ -1105,15 +1124,12 @@ void testTaskErrorHandling() throws Exception {
.filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR)
.findFirst()
.orElseThrow(() -> new AssertionError("expected a successful eval span"));
@SuppressWarnings("unchecked")
Map<String, Object> successOutputJson =
fromJson(
JsonNode successOutputJson =
JSON_MAPPER.readTree(
successEvalSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
Map.class);
assertEquals(
"result", successOutputJson.get("output"), "successful case should have output");
.get(AttributeKey.stringKey("braintrust.output_json")));
assertEquals("result", successOutputJson.asText(), "successful case should have output");
}

@Test
Expand Down
63 changes: 40 additions & 23 deletions braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,28 +86,45 @@ public void evalOtelTraceWithProperAttributes() {
"all eval spans must set the parent to the experiment id");
if (span.getParentSpanId().equals(SpanId.getInvalid())) {
numRootSpans.incrementAndGet();
var inputJson =
var input =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.input_json")),
Map.class);
assertNotNull(inputJson.get("input"), "invlaid input: " + inputJson);
String.class);
assertTrue(
input.equals("strawberry") || input.equals("asparagus"),
"invalid input: " + input);

var expected =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.expected")),
.get(AttributeKey.stringKey("braintrust.expected_json")),
String.class);
assertTrue(isFruitOrVegetable(expected), "invalid expected: " + expected);

var outputJson =
var output =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
Map.class);
var output = outputJson.get("output");
assertNotNull(output, "invlaid output: " + outputJson);
assertTrue(isFruitOrVegetable(String.valueOf(output)), "invalid output: " + output);
String.class);
assertNotNull(output, "invalid output: " + output);
assertTrue(isFruitOrVegetable(output), "invalid output: " + output);
} else if ("task".equals(span.getName())) {
// task span carries its own bare input/output
var input =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.input_json")),
String.class);
assertTrue(
input.equals("strawberry") || input.equals("asparagus"),
"invalid task input: " + input);
var output =
fromJson(
span.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
String.class);
assertTrue(isFruitOrVegetable(output), "invalid task output: " + output);
}
}
assertEquals(2, numRootSpans.get(), "each case should make a root span");
Expand Down Expand Up @@ -315,8 +332,7 @@ public void evalRootSpanPassesOriginIfPresent() {
var inputJson =
span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json"));
assertNotNull(inputJson);
fromJson(inputJson, Map.class);
var input = (String) (fromJson(inputJson, Map.class)).get("input");
var input = fromJson(inputJson, String.class);
assertNotNull(input);
var origin = span.getAttributes().get(AttributeKey.stringKey("braintrust.origin"));
switch (input) {
Expand Down Expand Up @@ -481,14 +497,11 @@ void evalContinuesWhenTaskThrows() {
erroredRootSpan.getStatus().getDescription().contains("task failed on bad-input"),
"root span error should contain the exception message");

// The errored root span should have output: null
var erroredOutputJson =
fromJson(
erroredRootSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
Map.class);
assertNull(erroredOutputJson.get("output"), "errored case output should be null");
assertNull(
erroredRootSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
"errors should not set output json");

// Find the task span for the errored case (child of errored root, type=task, status=ERROR)
var erroredTaskSpan =
Expand Down Expand Up @@ -519,6 +532,11 @@ void evalContinuesWhenTaskThrows() {
assertTrue(
erroredTaskSpan.getEvents().stream().anyMatch(e -> e.getName().equals("exception")),
"task span should have an exception event");
assertNull(
erroredTaskSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
"errors should not set output json");

// The errored case should still have score spans (from scoreForTaskException default = 0.0)
var erroredScoreSpans =
Expand Down Expand Up @@ -560,14 +578,13 @@ void evalContinuesWhenTaskThrows() {
.filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR)
.findFirst()
.orElseThrow(() -> new AssertionError("expected a successful root span"));
var successOutputJson =
var successOutput =
fromJson(
successRootSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.output_json")),
Map.class);
assertEquals(
"result", successOutputJson.get("output"), "successful case should have output");
String.class);
assertEquals("result", successOutput, "successful case should have output");
}

@Test
Expand Down
Loading