Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions crates/libsy-llm-client/src/observability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,14 @@ fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {
record_gen_ai_usage(span, &response.usage);
}

/// Records semantic token totals and cache-usage breakdowns on a client-call span.
fn record_gen_ai_usage(span: &Span, usage: &Usage) {
let cache_read = usage.cached_input_tokens();
let cache_creation = usage.cache_creation_input_tokens();
let cache_creation_1h = usage
.cache
.as_ref()
.and_then(|cache| cache.cache_creation_1h_input_tokens);
if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {
let input_tokens = usage
.input_tokens
Expand All @@ -127,6 +132,10 @@ fn record_gen_ai_usage(span: &Span, usage: &Usage) {
("gen_ai.usage.output_tokens", usage.output_tokens),
("gen_ai.usage.cache_read.input_tokens", cache_read),
("gen_ai.usage.cache_creation.input_tokens", cache_creation),
(
"gen_ai.usage.cache_creation_1h.input_tokens",
cache_creation_1h,
),
(
"gen_ai.usage.reasoning.output_tokens",
usage.reasoning_tokens,
Expand Down
1 change: 1 addition & 0 deletions crates/libsy-llm-client/src/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,7 @@ impl CallPhase {
gen_ai.usage.output_tokens = tracing::field::Empty,
gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty,
gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
outcome = tracing::field::Empty,
otel.status_code = tracing::field::Empty,
Expand Down
25 changes: 15 additions & 10 deletions crates/libsy-llm-client/tests/observability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -989,16 +989,20 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya
let total_errors_before =
u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();

let client = Arc::new(UsageClient {
usage: Usage {
input_tokens: Some(11),
output_tokens: Some(7),
total_tokens: Some(25),
reasoning_tokens: Some(2),
cache: Usage::cache_details(Some(3), Some(4)),
provider_cost: None,
},
}) as Arc<dyn RoutedLlmClient>;
let mut usage = Usage {
input_tokens: Some(11),
output_tokens: Some(7),
total_tokens: Some(25),
reasoning_tokens: Some(2),
cache: Usage::cache_details(Some(3), Some(4)),
provider_cost: None,
};
usage
.cache
.as_mut()
.expect("cache details")
.cache_creation_1h_input_tokens = Some(2);
let client = Arc::new(UsageClient { usage }) as Arc<dyn RoutedLlmClient>;
let mut request = request_with_metadata("obs-session-1", "obs-corr-1");
request.llm_request.sampling.temperature = Some(0.25);
request.llm_request.sampling.top_p = Some(0.9);
Expand Down Expand Up @@ -1206,6 +1210,7 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya
("gen_ai.usage.output_tokens", "7"),
("gen_ai.usage.cache_read.input_tokens", "3"),
("gen_ai.usage.cache_creation.input_tokens", "4"),
("gen_ai.usage.cache_creation_1h.input_tokens", "2"),
("gen_ai.usage.reasoning.output_tokens", "2"),
] {
assert_eq!(
Expand Down
29 changes: 29 additions & 0 deletions crates/switchyard-nemo-relay-plugin/src/span_capture.rs
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ impl CapturedSpan {
}
}

/// Reconstructs the bounded response metadata forwarded to NeMo Relay.
fn response_data(fields: &Json) -> Option<Json> {
let mut output = Map::new();
for (source, target) in [
Expand All @@ -184,6 +185,12 @@ fn response_data(fields: &Json) -> Option<Json> {
usage.insert(target.into(), value.clone());
}
}
if let Some(value) = fields.get("gen_ai.usage.cache_creation_1h.input_tokens") {
usage.insert(
"cache_creation".into(),
serde_json::json!({"ephemeral_1h_input_tokens": value}),
);
}
if let Some(input) = fields
.get("gen_ai.usage.input_tokens")
.and_then(Json::as_u64)
Expand Down Expand Up @@ -318,6 +325,7 @@ fn relay_scope_type(name: &str) -> ScopeType {
}
}

/// Returns whether a captured span field is safe to forward across the plugin boundary.
fn is_safe_field(name: &str) -> bool {
matches!(
name,
Expand All @@ -341,6 +349,7 @@ fn is_safe_field(name: &str) -> bool {
| "gen_ai.usage.output_tokens"
| "gen_ai.usage.cache_read.input_tokens"
| "gen_ai.usage.cache_creation.input_tokens"
| "gen_ai.usage.cache_creation_1h.input_tokens"
| "gen_ai.usage.reasoning.output_tokens"
| "model"
| "wire_format"
Expand Down Expand Up @@ -446,6 +455,26 @@ mod tests {
assert_eq!(call.children[0].fields["model"], "[REDACTED]");
}

#[test]
fn response_preserves_one_hour_cache_creation_subset() {
let fields = serde_json::json!({
"gen_ai.response.model": "target/model",
"gen_ai.usage.input_tokens": 180,
"gen_ai.usage.output_tokens": 7,
"gen_ai.usage.cache_read.input_tokens": 30,
"gen_ai.usage.cache_creation.input_tokens": 50,
"gen_ai.usage.cache_creation_1h.input_tokens": 20,
});

let response = response_data(&fields).expect("response data");
assert_eq!(response["usage"]["input_tokens"], 100);
assert_eq!(response["usage"]["cache_creation_input_tokens"], 50);
assert_eq!(
response["usage"]["cache_creation"]["ephemeral_1h_input_tokens"],
20
);
}

#[tokio::test]
async fn captures_spans_that_close_after_the_scoped_future_returns() {
let (capture, captured) = recorder();
Expand Down
Loading