diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs index fd1c13a10e..9e45aef00e 100644 --- a/crates/libsy-llm-client/src/observability.rs +++ b/crates/libsy-llm-client/src/observability.rs @@ -112,9 +112,14 @@ fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) { record_gen_ai_usage(span, &response.usage); } +/// Records semantic token totals and cache-usage breakdowns on a client-call span. fn record_gen_ai_usage(span: &Span, usage: &Usage) { let cache_read = usage.cached_input_tokens(); let cache_creation = usage.cache_creation_input_tokens(); + let cache_creation_1h = usage + .cache + .as_ref() + .and_then(|cache| cache.cache_creation_1h_input_tokens); if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() { let input_tokens = usage .input_tokens @@ -127,6 +132,10 @@ fn record_gen_ai_usage(span: &Span, usage: &Usage) { ("gen_ai.usage.output_tokens", usage.output_tokens), ("gen_ai.usage.cache_read.input_tokens", cache_read), ("gen_ai.usage.cache_creation.input_tokens", cache_creation), + ( + "gen_ai.usage.cache_creation_1h.input_tokens", + cache_creation_1h, + ), ( "gen_ai.usage.reasoning.output_tokens", usage.reasoning_tokens, diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 8e8053e6a1..d451205e76 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -419,6 +419,7 @@ impl CallPhase { gen_ai.usage.output_tokens = tracing::field::Empty, gen_ai.usage.cache_read.input_tokens = tracing::field::Empty, gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty, + gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty, gen_ai.usage.reasoning.output_tokens = tracing::field::Empty, outcome = tracing::field::Empty, otel.status_code = tracing::field::Empty, diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index ec16a2c0fe..9ca3687699 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -989,16 +989,20 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya let total_errors_before = u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default(); - let client = Arc::new(UsageClient { - usage: Usage { - input_tokens: Some(11), - output_tokens: Some(7), - total_tokens: Some(25), - reasoning_tokens: Some(2), - cache: Usage::cache_details(Some(3), Some(4)), - provider_cost: None, - }, - }) as Arc; + let mut usage = Usage { + input_tokens: Some(11), + output_tokens: Some(7), + total_tokens: Some(25), + reasoning_tokens: Some(2), + cache: Usage::cache_details(Some(3), Some(4)), + provider_cost: None, + }; + usage + .cache + .as_mut() + .expect("cache details") + .cache_creation_1h_input_tokens = Some(2); + let client = Arc::new(UsageClient { usage }) as Arc; let mut request = request_with_metadata("obs-session-1", "obs-corr-1"); request.llm_request.sampling.temperature = Some(0.25); request.llm_request.sampling.top_p = Some(0.9); @@ -1206,6 +1210,7 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya ("gen_ai.usage.output_tokens", "7"), ("gen_ai.usage.cache_read.input_tokens", "3"), ("gen_ai.usage.cache_creation.input_tokens", "4"), + ("gen_ai.usage.cache_creation_1h.input_tokens", "2"), ("gen_ai.usage.reasoning.output_tokens", "2"), ] { assert_eq!( diff --git a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs index 94037b0240..31bf477b87 100644 --- a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs +++ b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs @@ -158,6 +158,7 @@ impl CapturedSpan { } } +/// Reconstructs the bounded response metadata forwarded to NeMo Relay. fn response_data(fields: &Json) -> Option { let mut output = Map::new(); for (source, target) in [ @@ -184,6 +185,12 @@ fn response_data(fields: &Json) -> Option { usage.insert(target.into(), value.clone()); } } + if let Some(value) = fields.get("gen_ai.usage.cache_creation_1h.input_tokens") { + usage.insert( + "cache_creation".into(), + serde_json::json!({"ephemeral_1h_input_tokens": value}), + ); + } if let Some(input) = fields .get("gen_ai.usage.input_tokens") .and_then(Json::as_u64) @@ -318,6 +325,7 @@ fn relay_scope_type(name: &str) -> ScopeType { } } +/// Returns whether a captured span field is safe to forward across the plugin boundary. fn is_safe_field(name: &str) -> bool { matches!( name, @@ -341,6 +349,7 @@ fn is_safe_field(name: &str) -> bool { | "gen_ai.usage.output_tokens" | "gen_ai.usage.cache_read.input_tokens" | "gen_ai.usage.cache_creation.input_tokens" + | "gen_ai.usage.cache_creation_1h.input_tokens" | "gen_ai.usage.reasoning.output_tokens" | "model" | "wire_format" @@ -446,6 +455,26 @@ mod tests { assert_eq!(call.children[0].fields["model"], "[REDACTED]"); } + #[test] + fn response_preserves_one_hour_cache_creation_subset() { + let fields = serde_json::json!({ + "gen_ai.response.model": "target/model", + "gen_ai.usage.input_tokens": 180, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.cache_read.input_tokens": 30, + "gen_ai.usage.cache_creation.input_tokens": 50, + "gen_ai.usage.cache_creation_1h.input_tokens": 20, + }); + + let response = response_data(&fields).expect("response data"); + assert_eq!(response["usage"]["input_tokens"], 100); + assert_eq!(response["usage"]["cache_creation_input_tokens"], 50); + assert_eq!( + response["usage"]["cache_creation"]["ephemeral_1h_input_tokens"], + 20 + ); + } + #[tokio::test] async fn captures_spans_that_close_after_the_scoped_future_returns() { let (capture, captured) = recorder();