From eb351c2582c253ae9aabfcb1f1908282a89af1ee Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Mon, 19 Oct 2026 15:58:46 -0700 Subject: [PATCH 1/8] feat(server): record each call's cost in the routing log Signed-off-by: Elyas Mehtabuddin --- Cargo.lock | 1 + .../libsy-llm-client/tests/observability.rs | 1 + crates/protocol/src/llm.rs | 9 +- .../src/runtime.rs | 1 + crates/switchyard-runner/Cargo.toml | 1 + crates/switchyard-runner/src/lib.rs | 1 + crates/switchyard-runner/src/pricing.rs | 336 ++++++++++++++++++ crates/switchyard-server/README.md | 5 + crates/switchyard-server/src/cli.rs | 7 + crates/switchyard-server/src/lib.rs | 65 +++- crates/switchyard-server/src/routing_log.rs | 79 +++- crates/switchyard-server/src/usage_metrics.rs | 70 +++- crates/switchyard-server/tests/cli.rs | 58 +++ .../tests/litellm_gateway.rs | 148 ++++++++ crates/switchyard-server/tests/server.rs | 142 +++++++- .../src/codecs/anthropic/buffered.rs | 9 +- .../src/codecs/anthropic/stream.rs | 16 +- .../src/codecs/common.rs | 45 +++ .../src/codecs/openai_chat/buffered.rs | 11 +- .../src/codecs/openai_chat/stream.rs | 11 +- .../src/codecs/responses/buffered.rs | 36 +- .../src/codecs/responses/stream.rs | 32 +- .../tests/response_translation.rs | 64 +++- .../tests/stream_translation.rs | 44 +++ docs/cli_reference.md | 1 + docs/integrations/costs.md | 166 +++++++++ docs/integrations/oh_my_pi.md | 6 +- docs/integrations/pi.md | 44 +-- examples/costs/claude_statusline.py | 45 +++ examples/costs/display.ts | 41 +++ examples/costs/omp.ts | 13 + examples/costs/pi.ts | 17 + 32 files changed, 1422 insertions(+), 103 deletions(-) create mode 100644 crates/switchyard-runner/src/pricing.rs create mode 100644 docs/integrations/costs.md create mode 100644 examples/costs/claude_statusline.py create mode 100644 examples/costs/display.ts create mode 100644 examples/costs/omp.ts create mode 100644 examples/costs/pi.ts diff --git a/Cargo.lock b/Cargo.lock index 705b86d08..2597f3f97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3830,6 +3830,7 @@ dependencies = [ "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", + "tempfile", "thiserror 2.0.18", "tokio", "toml 1.1.3+spec-1.1.0", diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 27a5f3862..ec16a2c0f 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -996,6 +996,7 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya total_tokens: Some(25), reasoning_tokens: Some(2), cache: Usage::cache_details(Some(3), Some(4)), + provider_cost: None, }, }) as Arc; let mut request = request_with_metadata("obs-session-1", "obs-corr-1"); diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index 9429354e5..cb3731aef 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -559,13 +559,16 @@ pub struct LlmRequest { /// Normalized token usage counts. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Usage { + /// The upstream provider reports this monetary charge in USD. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_cost: Option, /// Non-cached input tokens. Provider codecs normalize aggregate OpenAI /// input counts by subtracting the cache detail fields. pub input_tokens: Option, /// Cache-read and cache-creation token detail, when reported. #[serde(flatten)] pub cache: Option>, - /// Generated output tokens, excluding reasoning detail when the provider reports it separately. + /// The output count includes reasoning tokens when the provider counts them. pub output_tokens: Option, /// Provider-reported or codec-computed total token count. /// @@ -579,6 +582,9 @@ pub struct Usage { /// Optional cache-token detail kept out of the common, cache-free usage allocation. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct InputCacheUsage { + /// The provider reports these cache-write tokens at the one-hour rate. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_1h_input_tokens: Option, /// Input tokens read from a provider cache. pub cached_input_tokens: Option, /// Input tokens written into a provider cache. @@ -595,6 +601,7 @@ impl Usage { return None; } Some(Box::new(InputCacheUsage { + cache_creation_1h_input_tokens: None, cached_input_tokens, cache_creation_input_tokens, })) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 087034832..e561ff8c9 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -1415,6 +1415,7 @@ mod tests { output_tokens: Some(7), total_tokens: Some(23), reasoning_tokens: Some(5), + provider_cost: None, }), }; diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.toml index 3a80b61b3..443491ea9 100644 --- a/crates/switchyard-runner/Cargo.toml +++ b/crates/switchyard-runner/Cargo.toml @@ -32,4 +32,5 @@ tracing.workspace = true [dev-dependencies] async-trait.workspace = true futures-util.workspace = true +tempfile = "3" tokio.workspace = true diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rs index 36ff10841..76ebf69ae 100644 --- a/crates/switchyard-runner/src/lib.rs +++ b/crates/switchyard-runner/src/lib.rs @@ -6,6 +6,7 @@ mod algorithm; mod config; mod failure; +pub mod pricing; mod provider_key_redactor; mod route; mod runner; diff --git a/crates/switchyard-runner/src/pricing.rs b/crates/switchyard-runner/src/pricing.rs new file mode 100644 index 000000000..cee8379eb --- /dev/null +++ b/crates/switchyard-runner/src/pricing.rs @@ -0,0 +1,336 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! This module prices observed calls and keeps missing usage or rates unknown. + +use std::collections::HashMap; +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use switchyard_protocol::Usage; + +use crate::RunnerError; + +/// Pricing maps served model IDs to token rates in USD per million tokens. The server prices +/// routing records with it, and the desktop app prices its baseline comparison with it. +#[derive(Debug, Default)] +pub struct Pricing(HashMap); + +impl<'de> Deserialize<'de> for Pricing { + fn deserialize>(deserializer: D) -> Result { + // This visitor rejects duplicate IDs before a map can overwrite a rate table. + struct UniqueModels; + impl<'de> serde::de::Visitor<'de> for UniqueModels { + type Value = Pricing; + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a map of unique model IDs to token rates") + } + fn visit_map>( + self, + mut map: M, + ) -> Result { + let mut rates = HashMap::new(); + while let Some((model, rate)) = map.next_entry::()? { + if rates.insert(model.clone(), rate).is_some() { + return Err(serde::de::Error::custom(format!( + "duplicate pricing model {model:?}" + ))); + } + } + Ok(Pricing(rates)) + } + } + deserializer.deserialize_map(UniqueModels) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Rates { + input: f64, + output: f64, + cache_read: f64, + cache_write: f64, + cache_write_1h: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CostSource { + ProviderReported, + Estimated, +} + +#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)] +pub struct RecordedCost { + pub usd: f64, + pub source: CostSource, +} + +impl Pricing { + pub fn load(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|error| { + RunnerError::configuration(format!( + "failed to read pricing file {}: {error}", + path.display() + )) + })?; + Self::parse(&bytes).map_err(|error| { + RunnerError::configuration(format!("invalid pricing file {}: {error}", path.display())) + }) + } + + /// This function reads a pricing file's JSON and checks every model ID and rate. + pub fn parse(json: &[u8]) -> Result { + let pricing: Self = serde_json::from_slice(json).map_err(|error| error.to_string())?; + for (model, rate) in &pricing.0 { + if model.trim().is_empty() + || model.trim() != model + || [ + Some(rate.input), + Some(rate.output), + Some(rate.cache_read), + Some(rate.cache_write), + rate.cache_write_1h, + ] + .into_iter() + .flatten() + .any(|value| !value.is_finite() || !(0.0..=10000.0).contains(&value)) + { + return Err(format!( + "invalid pricing for model {model:?}: use a nonempty model ID and rates from 0 to 10000 USD per million tokens" + )); + } + } + Ok(pricing) + } + + /// This function reports whether the file has rates for `model`. + pub fn contains(&self, model: &str) -> bool { + self.0.contains_key(model) + } + + /// Provider charges take precedence because token rates may omit provider surcharges. + pub fn cost(&self, model: &str, usage: &Usage) -> Option { + if let Some(usd) = usage + .provider_cost + .filter(|value| value.is_finite() && *value >= 0.0) + { + return Some(RecordedCost { + usd, + source: CostSource::ProviderReported, + }); + } + let rate = self.0.get(model)?; + let input = usage.input_tokens?; + let output = usage.output_tokens?; + let read = usage.cached_input_tokens().unwrap_or(0); + let write = usage.cache_creation_input_tokens().unwrap_or(0); + let long_write = usage + .cache + .as_ref() + .and_then(|cache| cache.cache_creation_1h_input_tokens) + .unwrap_or(0); + let short_write = write.checked_sub(long_write)?; + let long_rate = if long_write > 0 { + rate.cache_write_1h? + } else { + 0.0 + }; + let usd = (input as f64 * rate.input + + output as f64 * rate.output + + read as f64 * rate.cache_read + + short_write as f64 * rate.cache_write + + long_write as f64 * long_rate) + / 1_000_000.0; + Some(RecordedCost { + usd, + source: CostSource::Estimated, + }) + } +} + +/// The known subtotal remains visible when any call has unknown cost. +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub struct CostTotals { + pub known_usd: f64, + pub unknown_calls: u64, + pub estimated_calls: u64, + pub provider_reported_calls: u64, + pub total_usd: Option, +} + +impl Default for CostTotals { + fn default() -> Self { + Self { + known_usd: 0.0, + unknown_calls: 0, + estimated_calls: 0, + provider_reported_calls: 0, + total_usd: Some(0.0), + } + } +} + +impl CostTotals { + pub fn add(&mut self, cost: Option) { + match cost.filter(|cost| { + cost.usd.is_finite() && cost.usd >= 0.0 && (self.known_usd + cost.usd).is_finite() + }) { + Some(cost) => { + self.known_usd += cost.usd; + match cost.source { + CostSource::Estimated => self.estimated_calls += 1, + CostSource::ProviderReported => self.provider_reported_calls += 1, + } + } + None => self.unknown_calls += 1, + } + self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); + } + + /// This function adds another set of totals, such as one day's totals to a week. + pub fn merge(&mut self, other: &Self) { + let known = self.known_usd + other.known_usd; + if known.is_finite() { + self.known_usd = known; + self.estimated_calls += other.estimated_calls; + self.provider_reported_calls += other.provider_reported_calls; + } else { + self.unknown_calls += other.estimated_calls + other.provider_reported_calls; + } + self.unknown_calls += other.unknown_calls; + self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_invalid_rate_tables() -> Result<(), Box> { + let valid = r#"{"input":1,"output":5,"cache_read":0.1,"cache_write":2}"#; + for rates in [ + r#"{"input":-1,"output":5,"cache_read":0.1,"cache_write":2}"#.to_string(), + r#"{"input":1,"output":5,"cache_read":0.1}"#.to_string(), + r#"{"input":1,"output":5,"cache_read":0.1,"cache_write":2,"cache_wirte":2}"# + .to_string(), + r#"{"input":1e999,"output":5,"cache_read":0.1,"cache_write":2}"#.to_string(), + ] { + let file = tempfile::NamedTempFile::new()?; + std::fs::write(file.path(), format!("{{\"model\":{rates}}}"))?; + assert!(Pricing::load(file.path()).is_err(), "{rates}"); + } + let file = tempfile::NamedTempFile::new()?; + std::fs::write( + file.path(), + format!("{{\"model\":{valid},\"model\":{valid}}}"), + )?; + assert!(Pricing::load(file.path()).is_err()); + Ok(()) + } + + #[test] + fn estimates_disjoint_buckets_and_prefers_reported_charges() + -> Result<(), Box> { + let file = tempfile::NamedTempFile::new()?; + std::fs::write( + file.path(), + r#"{"model":{"input":1,"output":5,"cache_read":0.1,"cache_write":2,"cache_write_1h":3},"free":{"input":0,"output":0,"cache_read":0,"cache_write":0}}"#, + )?; + let prices = Pricing::load(file.path())?; + let mut usage = Usage { + input_tokens: Some(50), + output_tokens: Some(10), + reasoning_tokens: Some(4), + cache: Usage::cache_details(Some(20), Some(30)), + ..Usage::default() + }; + usage + .cache + .as_mut() + .ok_or("missing cache detail")? + .cache_creation_1h_input_tokens = Some(10); + assert!( + (prices.cost("model", &usage).ok_or("missing estimate")?.usd - 0.000172).abs() < 1e-12 + ); + assert_eq!( + prices + .cost( + "free", + &Usage { + cache: None, + ..usage.clone() + } + ) + .ok_or("missing free price")? + .usd, + 0.0 + ); + assert!(prices.cost("unknown", &usage).is_none()); + assert!(prices.cost("model", &Usage::default()).is_none()); + assert!(prices.cost("free", &usage).is_none()); + let mut invalid_duration = usage.clone(); + invalid_duration + .cache + .as_mut() + .ok_or("missing cache detail")? + .cache_creation_1h_input_tokens = Some(31); + assert!(prices.cost("model", &invalid_duration).is_none()); + usage.provider_cost = Some(0.0); + assert_eq!( + prices + .cost("unknown", &usage) + .ok_or("missing reported cost")? + .usd, + 0.0 + ); + usage.provider_cost = Some(0.25); + assert_eq!( + prices + .cost("model", &usage) + .ok_or("missing reported cost")? + .usd, + 0.25 + ); + Ok(()) + } + + #[test] + fn partial_and_overflowing_totals_remain_unknown() { + let mut totals = CostTotals::default(); + totals.add(Some(RecordedCost { + usd: 0.25, + source: CostSource::Estimated, + })); + totals.add(None); + assert_eq!(totals.known_usd, 0.25); + assert_eq!(totals.unknown_calls, 1); + assert_eq!(totals.total_usd, None); + let mut week = CostTotals::default(); + week.add(Some(RecordedCost { + usd: 0.5, + source: CostSource::ProviderReported, + })); + assert_eq!(week.total_usd, Some(0.5)); + week.merge(&totals); + assert_eq!((week.known_usd, week.unknown_calls), (0.75, 1)); + assert_eq!(week.total_usd, None); + totals.add(Some(RecordedCost { + usd: f64::MAX, + source: CostSource::ProviderReported, + })); + totals.add(Some(RecordedCost { + usd: f64::MAX, + source: CostSource::ProviderReported, + })); + assert!(totals.known_usd.is_finite()); + assert_eq!(totals.unknown_calls, 2); + // Merging totals whose sum would overflow counts the merged priced calls as unknown. + let mut doubled = totals.clone(); + doubled.merge(&totals); + assert!(doubled.known_usd.is_finite()); + assert_eq!(doubled.unknown_calls, 6); + } +} diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 08335197f..05929f950 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -124,6 +124,11 @@ served model. The legacy `proxy_x_session_id` remains a fallback when no normali present. The endpoint returns `404` when the session has no records and is not registered when routing logging is disabled. +Add `--pricing-file prices.json` to record each call's `cost` in USD. The server records the +charge that the upstream reports and otherwise estimates it from the file's rates for the served +model. The session stats then include cost totals. [Display Switchyard session +costs](../../docs/integrations/costs.md) describes the file format. + Clients can send `x-switchyard-origin: codex-cli` (or another client label) to include an `origin` field in each routing record. Missing, empty, or non-text header values produce `"origin": null`. The value is supplied by the caller; it is not inferred from `User-Agent`. diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index 231d4e2bc..969445792 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -52,6 +52,10 @@ pub(crate) struct ServerArgs { #[arg(long, value_name = "PATH")] routing_log_file: Option, + /// This JSON file maps served model IDs to rates in USD per million tokens. + #[arg(long, value_name = "PATH", requires = "routing_log_file")] + pricing_file: Option, + /// TLS certificate path in PEM format. #[arg(long, requires = "tls_key")] tls_cert: Option, @@ -72,6 +76,9 @@ impl ServerArgs { if let Some(path) = self.routing_log_file { state = state.with_routing_log(path)?; } + if let Some(path) = self.pricing_file { + state = state.with_pricing_file(path)?; + } let tls = match (self.tls_cert, self.tls_key) { (Some(cert), Some(key)) => { if !cert.exists() || !key.exists() { diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 9e3ac630a..de3e89366 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -237,6 +237,17 @@ impl ServerState { Ok(self) } + /// This method loads rates in USD per million tokens for saved routing records. + pub fn with_pricing_file(self, path: impl AsRef) -> ServerResult { + let prices = switchyard_runner::pricing::Pricing::load(path.as_ref())?; + let log = self + .routing_log + .as_ref() + .ok_or_else(|| ServerError::new("pricing requires a routing log"))?; + log.writer.lock().pricing = prices; + Ok(self) + } + /// Returns the route model IDs served by the configured algorithms. pub fn models(&self) -> impl Iterator { self.runner.models().map(|model| model.id.as_str()) @@ -429,13 +440,11 @@ struct RequestStart(Instant); /// Routing-log marker distinguishing classifier and judge calls from answer calls. const CLASSIFIER_TIER: &str = "classifier"; -/// Maps answer-call observations to backend stats, routing calls to classifier/judge -/// stats, and records routing overhead once the algorithm run completes. +/// This observer updates backend stats and records classifier and judge calls. /// -/// Successful classifier/judge calls are also appended to `classifier_log` when one is -/// configured, so per-session routing snapshots account for judge token overhead. Routed -/// calls stay off this path: the served call is logged with its terminal usage in -/// [`usage_metrics::observe`], which would make a second append here a double count. +/// The routing log also records failed answer calls with unknown cost. Successful +/// answer calls are recorded by [`usage_metrics::observe`] with their terminal usage; +/// recording them here would count each answer twice. fn stats_observer( stats: StatsAccumulator, classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>, @@ -447,22 +456,28 @@ fn stats_observer( if call.is_success { stats.record_success(&call.selected_model, latency_ms); } else { + if let Some((log, context)) = classifier_log.as_ref() { + log.append( + context.clone(), + &call.selected_model, + None, + &Usage::default(), + ); + } stats.record_error(&call.selected_model); } } RunObservation::LlmCall(call) | RunObservation::DecisionCall(call) => { let latency_ms = call.duration.as_secs_f64() * 1_000.0; + if let Some((log, context)) = classifier_log.as_ref() { + log.append( + context.clone(), + &call.selected_model, + Some(CLASSIFIER_TIER), + call.usage.as_ref().unwrap_or(&Usage::default()), + ); + } if call.is_success { - if let (Some((log, context)), Some(usage)) = - (classifier_log.as_ref(), call.usage.as_ref()) - { - log.append( - context.clone(), - &call.selected_model, - Some(CLASSIFIER_TIER), - usage, - ); - } stats.record_classifier_success( call.selected_model, call.usage.as_ref().map(usage_metrics::token_usage), @@ -1810,8 +1825,9 @@ targets = ["only"] } /// The observer logs judge calls with their model IDs, classifier tier, and - /// recorded session and turn IDs. The served response logs routed calls with - /// their final usage, so logging them here would count each routed call twice. + /// recorded session and turn IDs, even when a call reports no usage. The served + /// response logs routed calls with their final usage, so logging them here would + /// count each routed call twice. #[test] fn stats_observer_logs_judge_calls_to_the_routing_log() -> Result<(), Box> { @@ -1843,6 +1859,12 @@ targets = ["only"] }; observer(call("judge-model", false)); observer(call("routed-model", true)); + observer(RunObservation::LlmCall(ModelCallObservation { + selected_model: ModelId::from("streamed-judge"), + is_success: true, + duration: Duration::from_millis(3), + usage: None, + })); let snapshot = log .snapshot_session("session-1") @@ -1853,7 +1875,12 @@ targets = ["only"] assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100); assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7); assert!(snapshot["models"].get("routed-model").is_none()); - let record: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&log.path)?)?; + assert_eq!(snapshot["models"]["streamed-judge"]["calls"], 1); + assert_eq!(snapshot["routing_cost"]["unknown_calls"], 2); + assert!(snapshot["routing_cost"]["total_usd"].is_null()); + let text = std::fs::read_to_string(&log.path)?; + let record: serde_json::Value = + serde_json::from_str(text.lines().next().ok_or("empty log")?)?; assert_eq!(record["session_id"], "session-1"); assert_eq!(record["turn_id"], "turn-1"); Ok(()) diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs index 674db1b6c..250046546 100644 --- a/crates/switchyard-server/src/routing_log.rs +++ b/crates/switchyard-server/src/routing_log.rs @@ -16,6 +16,7 @@ use switchyard_protocol::{Metadata, ModelId, Usage}; use crate::usage_metrics::token_usage; use crate::{ServerError, ServerResult}; +use switchyard_runner::pricing::{CostTotals, Pricing, RecordedCost}; const LEGACY_SESSION_ID_HEADER: &str = "proxy_x_session_id"; const ORIGIN_HEADER: &str = "x-switchyard-origin"; @@ -23,7 +24,10 @@ const TASK_HEADER: &str = "x-switchyard-intake-task"; const TRIAL_ID_HEADER: &str = "x-switchyard-trial-id"; /// Append-only writer for one routing JSONL file. -pub(crate) struct RoutingLog(fs::File); +pub(crate) struct RoutingLog { + file: fs::File, + pub(crate) pricing: Pricing, +} impl RoutingLog { pub(crate) fn new(path: impl Into) -> ServerResult { @@ -39,7 +43,10 @@ impl RoutingLog { .append(true) .open(&path) .map_err(|error| routing_log_error(&path, error))?; - Ok(Self(file)) + Ok(Self { + file, + pricing: Pricing::default(), + }) } pub(crate) fn append( @@ -49,8 +56,15 @@ impl RoutingLog { tier: Option<&str>, usage: &Usage, ) -> std::io::Result<()> { + let cost = self.pricing.cost(model, usage); + let cache_creation_1h_tokens = usage + .cache + .as_ref() + .and_then(|cache| cache.cache_creation_1h_input_tokens) + .unwrap_or(0); let usage = token_usage(usage); let record = RoutingRecord { + cost, ts: format_rfc3339_millis(SystemTime::now()).to_string().into(), route_id: context.route_id.into(), algorithm: context.algorithm.into(), @@ -64,6 +78,7 @@ impl RoutingLog { prompt_tokens: usage.prompt_tokens, cached_tokens: usage.cached_tokens, cache_creation_tokens: usage.cache_creation_tokens, + cache_creation_1h_tokens, completion_tokens: usage.completion_tokens, reasoning_tokens: usage.reasoning_tokens, total_tokens: usage.prompt_tokens.saturating_add(usage.completion_tokens), @@ -71,7 +86,7 @@ impl RoutingLog { let mut line = serde_json::to_vec(&record).map_err(std::io::Error::other)?; line.push(b'\n'); - self.0.write_all(&line) + self.file.write_all(&line) } } @@ -151,6 +166,8 @@ impl RoutingLogContext { #[derive(Default, Deserialize, Serialize)] #[serde(default)] struct RoutingRecord<'a> { + #[serde(default)] + cost: Option, ts: Cow<'a, str>, route_id: Cow<'a, str>, algorithm: Cow<'a, str>, @@ -169,15 +186,25 @@ struct RoutingRecord<'a> { prompt_tokens: u64, cached_tokens: u64, cache_creation_tokens: u64, + /// The part of `cache_creation_tokens` written at the one-hour rate, when reported. + #[serde(skip_serializing_if = "is_zero")] + cache_creation_1h_tokens: u64, completion_tokens: u64, reasoning_tokens: u64, total_tokens: u64, } +fn is_zero(value: &u64) -> bool { + *value == 0 +} + /// Session totals returned by the routing stats endpoint. #[derive(Serialize)] pub(crate) struct SessionStatsSnapshot { session_id: String, + cost: CostTotals, + answer_cost: CostTotals, + routing_cost: CostTotals, total_calls: u64, total_prompt_tokens: u64, total_cached_tokens: u64, @@ -188,6 +215,7 @@ pub(crate) struct SessionStatsSnapshot { #[derive(Default, Serialize)] struct SessionModelStats { + cost: CostTotals, calls: u64, prompt_tokens: u64, cached_tokens: u64, @@ -199,6 +227,9 @@ impl SessionStatsSnapshot { fn new(session_id: &str) -> Self { Self { session_id: session_id.to_string(), + cost: CostTotals::default(), + answer_cost: CostTotals::default(), + routing_cost: CostTotals::default(), total_calls: 0, total_prompt_tokens: 0, total_cached_tokens: 0, @@ -217,6 +248,13 @@ impl SessionStatsSnapshot { model => model, }; let stats = self.models.entry(ModelId::from(model)).or_default(); + stats.cost.add(record.cost); + self.cost.add(record.cost); + if record.tier == "classifier" { + self.routing_cost.add(record.cost); + } else { + self.answer_cost.add(record.cost); + } stats.calls = stats.calls.saturating_add(1); stats.prompt_tokens = stats.prompt_tokens.saturating_add(record.prompt_tokens); stats.cached_tokens = stats.cached_tokens.saturating_add(record.cached_tokens); @@ -301,7 +339,7 @@ mod tests { fs::write( &path, concat!( - r#"{"session_id":"a","model":"m1","fallback_reason":"unavailable","prompt_tokens":10,"completion_tokens":2}"#, + r#"{"session_id":"a","model":"m1","fallback_reason":"unavailable","prompt_tokens":10,"completion_tokens":2,"cost":{"usd":0.25,"source":"estimated"}}"#, "\n", r#"{"session_id":"b","model":"m1","prompt_tokens":99,"completion_tokens":99}"#, "\n", @@ -318,6 +356,39 @@ mod tests { assert_eq!(stats.total_completion_tokens, 2); assert_eq!(stats.models["m1"].calls, 1); assert_eq!(stats.models["unknown"].prompt_tokens, 5); + assert_eq!(stats.cost.known_usd, 0.25); + assert_eq!(stats.cost.estimated_calls, 1); + assert_eq!(stats.cost.unknown_calls, 1); + assert_eq!(stats.cost.total_usd, None); assert!(snapshot(&path, "missing").expect("read log").is_none()); } + + /// One-hour cache writes keep their own count, so a reader can price them at their rate. + #[test] + fn records_name_one_hour_cache_writes_only_when_reported() + -> Result<(), Box> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("routing.jsonl"); + let mut log = RoutingLog::new(&path)?; + let mut usage = Usage { + input_tokens: Some(10), + output_tokens: Some(1), + cache: Usage::cache_details(None, Some(30)), + ..Usage::default() + }; + let context = || RoutingLogContext::from_metadata(&Metadata::default()); + log.append(context(), "m", None, &usage)?; + if let Some(cache) = usage.cache.as_mut() { + cache.cache_creation_1h_input_tokens = Some(20); + } + log.append(context(), "m", None, &usage)?; + let records = fs::read_to_string(&path)? + .lines() + .map(serde_json::from_str) + .collect::, _>>()?; + assert_eq!(records[0]["cache_creation_tokens"], 30); + assert!(records[0].get("cache_creation_1h_tokens").is_none()); + assert_eq!(records[1]["cache_creation_1h_tokens"], 20); + Ok(()) + } } diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs index 045f51c86..ad5a6a77f 100644 --- a/crates/switchyard-server/src/usage_metrics.rs +++ b/crates/switchyard-server/src/usage_metrics.rs @@ -3,6 +3,7 @@ //! Response usage and full-turn latency metrics for routed requests. +use std::borrow::Cow; use std::time::{Duration, Instant}; use futures_util::StreamExt; @@ -33,7 +34,8 @@ pub(crate) fn observe( LlmResponse::Agg(agg) => { record_terminal(&stats, &agg.usage, &model, started, cache_eligible); if let Some((log, context)) = routing_log { - log.append(context, &model, None, &agg.usage); + let usage = with_gateway_charge(&agg.usage, &upstream_headers); + log.append(context, &model, None, &usage); } LlmResponse::Agg(agg) } @@ -66,6 +68,10 @@ pub(crate) fn observe( } } } + if failed && !recorded + && let Some((log, context)) = routing_log.as_ref() { + log.append(context.clone(), &model, None, &Usage::default()); + } if failed { record_stream_error(&stats, &model); } @@ -104,6 +110,27 @@ pub(crate) fn observe( } } +/// LiteLLM sends the charge for a buffered response in this header and leaves the cost out of +/// the response body. Its streamed responses omit the header and report `cost` in the final +/// usage instead, which the response codecs read. +const LITELLM_COST_HEADER: &str = "x-litellm-response-cost"; + +/// This function adds the gateway's header charge to `usage` when the body reported none. +fn with_gateway_charge<'a>(usage: &'a Usage, headers: &http::HeaderMap) -> Cow<'a, Usage> { + let charge = headers + .get(LITELLM_COST_HEADER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .filter(|usd| usd.is_finite() && *usd >= 0.0); + match charge { + Some(usd) if usage.provider_cost.is_none() => Cow::Owned(Usage { + provider_cost: Some(usd), + ..usage.clone() + }), + _ => Cow::Borrowed(usage), + } +} + // Records a terminal stream failure after the routed call was already counted. fn record_stream_error(stats: &StatsAccumulator, model: &str) { stats.record_response_error(model); @@ -187,6 +214,47 @@ mod tests { use super::*; + /// A stream error must leave one unpriced record even when no usage was reported. + #[tokio::test] + async fn stream_failure_records_one_unknown_call_and_terminates() + -> Result<(), Box> { + let dir = tempfile::tempdir()?; + let log = SharedRoutingLog::new(dir.path().join("routing.jsonl"))?; + let context = RoutingLogContext::from_metadata(&Metadata { + session_id: Some("failed-stream".to_string()), + ..Metadata::default() + }); + let error = LlmResponseStreamEvent::new(vec![LlmResponseChunk::StreamError { + message: "failed".to_string(), + }]); + let source = stream::iter([Ok(error.clone()), Ok(error)]); + let observed = observe( + Response { + llm_response: LlmResponse::Stream(Box::pin(source)), + metadata: None, + upstream_headers: http::HeaderMap::new(), + }, + "model/worker", + Instant::now(), + StatsAccumulator::default(), + 0.0, + Some((log.clone(), context)), + ); + let LlmResponse::Stream(mut observed) = observed.llm_response else { + return Err("expected stream".into()); + }; + assert!(observed.next().await.is_some()); + assert!(observed.next().await.is_none()); + let snapshot = log + .snapshot_session("failed-stream")? + .ok_or("missing session")?; + let snapshot = serde_json::to_value(snapshot)?; + assert_eq!(snapshot["total_calls"], 1); + assert_eq!(snapshot["cost"]["unknown_calls"], 1); + assert!(snapshot["cost"]["total_usd"].is_null()); + Ok(()) + } + /// An OpenAI Responses client may stop polling immediately after receiving the /// terminal `response.completed` event. Switchyard must record usage and routing /// data before returning that event because the stream wrapper will not resume diff --git a/crates/switchyard-server/tests/cli.rs b/crates/switchyard-server/tests/cli.rs index a0e535544..c60a02ec3 100644 --- a/crates/switchyard-server/tests/cli.rs +++ b/crates/switchyard-server/tests/cli.rs @@ -127,3 +127,61 @@ fn dry_run_rejects_unsendable_configured_headers() -> TestResult { } Ok(()) } + +#[test] +fn pricing_requires_a_log_and_validates_before_serving() -> TestResult { + let directory = tempfile::tempdir()?; + let config = directory.path().join("routes.toml"); + let prices = directory.path().join("prices.json"); + let log = directory.path().join("routing.jsonl"); + fs::write( + &config, + r#" +schema_version = 1 +[llm_clients.upstream] +format = "openai_chat" +base_url = "http://127.0.0.1:1/v1" +[targets.answer] +id = "answer" +llm_client = "upstream" +[routes.direct] +id = "direct" +type = "passthrough" +target = "answer" +"#, + )?; + let output = Command::new(env!("CARGO_BIN_EXE_switchyard-server")) + .arg("--config") + .arg(&config) + .arg("--pricing-file") + .arg(&prices) + .arg("--dry-run") + .output()?; + assert!(!output.status.success()); + assert!(String::from_utf8(output.stderr)?.contains("--routing-log-file")); + assert!(!log.exists()); + for (rates, succeeds) in [ + ( + r#"{"answer":{"input":0,"output":0,"cache_read":0,"cache_write":0}}"#, + true, + ), + (r#"{"answer":{"input":0,"output":0,"cache_read":0}}"#, false), + ] { + fs::write(&prices, rates)?; + let output = Command::new(env!("CARGO_BIN_EXE_switchyard-server")) + .arg("--config") + .arg(&config) + .arg("--pricing-file") + .arg(&prices) + .arg("--routing-log-file") + .arg(&log) + .arg("--dry-run") + .output()?; + assert_eq!(output.status.success(), succeeds); + if !succeeds { + assert!(String::from_utf8(output.stderr)?.contains("invalid pricing file")); + } + assert!(fs::read(&log)?.is_empty()); + } + Ok(()) +} diff --git a/crates/switchyard-server/tests/litellm_gateway.rs b/crates/switchyard-server/tests/litellm_gateway.rs index a83d00998..d760e423d 100644 --- a/crates/switchyard-server/tests/litellm_gateway.rs +++ b/crates/switchyard-server/tests/litellm_gateway.rs @@ -221,3 +221,151 @@ async fn codex_tool_call_streams_reach_response_completed() -> TestResult { } Ok(()) } + +/// The charge that NVIDIA's LiteLLM gateway reported in `x-litellm-response-cost` for a buffered +/// Chat Completions answer. The estimate from `RATES` is 0.000089 instead. +const HEADER_CHARGE: &str = "6.141e-05"; +/// The charge that the gateway reported as `usage.cost` at the end of a stream. The estimate +/// from `RATES` is 0.000059 instead. +const STREAM_CHARGE: f64 = 4.071e-05; +const RATES: &str = r#"{ + "model/claude": {"input": 1, "output": 5, "cache_read": 0.1, "cache_write": 2}, + "model/plain": {"input": 1, "output": 5, "cache_read": 0.1, "cache_write": 2} +}"#; + +/// Answers Chat Completions the way the gateway does. A buffered answer from `model/claude` +/// carries its charge only in the header. A streamed answer has zeroed breakdown headers and the +/// charge in the final usage. `model/plain` reports no charge. +async fn charging_gateway(Json(body): Json) -> Response { + let model = body["model"].as_str().unwrap_or_default().to_string(); + let mut usage = json!({"prompt_tokens": 9, "completion_tokens": 16, "total_tokens": 25}); + if body["stream"] == true { + usage = json!({"prompt_tokens": 9, "completion_tokens": 10, "total_tokens": 19, "cost": STREAM_CHARGE}); + let chunk = |delta: Value, finish: Value, usage: Value| { + json!({"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], "usage": usage}) + }; + let frames = [ + chunk( + json!({"role": "assistant", "content": "hi"}), + Value::Null, + Value::Null, + ), + chunk(json!({}), json!("stop"), usage), + ] + .into_iter() + .map(|event| event.to_string()) + .chain(["[DONE]".to_string()]) + .map(|data| Ok::(Event::default().data(data))); + let mut response = Sse::new(futures_util::stream::iter(frames)).into_response(); + let zero = axum::http::HeaderValue::from_static("0.0"); + response + .headers_mut() + .insert("x-litellm-response-cost-original", zero); + return response; + } + let mut response = Json(json!({ + "id": "c1", "object": "chat.completion", "created": 1, "model": model, "usage": usage, + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}] + })) + .into_response(); + if model == "model/claude" { + let charge = axum::http::HeaderValue::from_static(HEADER_CHARGE); + response + .headers_mut() + .insert("x-litellm-response-cost", charge); + } + response +} + +/// The routing log records the gateway's charge when it reports one and estimates from the +/// pricing file only when it does not. +#[tokio::test] +async fn routing_log_records_the_gateway_charge() -> TestResult { + let app = Router::new().route("/v1/chat/completions", post(charging_gateway)); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let root = format!("http://{}", listener.local_addr()?); + let gateway = Gateway { + root, + task: tokio::spawn(async move { axum::serve(listener, app).await }), + }; + let directory = tempfile::tempdir()?; + let config = directory.path().join("config.toml"); + std::fs::write( + &config, + format!( + r#" +schema_version = 1 + +[llm_clients.gateway] +format = "openai_chat" +base_url = "{root}/v1" +max_retries = 0 + +[targets.claude] +id = "model/claude" +llm_client = "gateway" + +[targets.plain] +id = "model/plain" +llm_client = "gateway" + +[routes.reported] +id = "gateway/reported" +type = "passthrough" +target = "claude" + +[routes.unreported] +id = "gateway/unreported" +type = "passthrough" +target = "plain" +"#, + root = gateway.root + ), + )?; + let prices = directory.path().join("prices.json"); + std::fs::write(&prices, RATES)?; + let state = load_server_state(&config)? + .with_routing_log(directory.path().join("routing.jsonl"))? + .with_pricing_file(&prices)?; + let app = build_switchyard_router(state); + let header_charge: f64 = HEADER_CHARGE.parse()?; + for (route, stream, usd, source) in [ + ( + "gateway/reported", + false, + header_charge, + "provider_reported_calls", + ), + ( + "gateway/reported", + true, + STREAM_CHARGE, + "provider_reported_calls", + ), + ("gateway/unreported", false, 0.000089, "estimated_calls"), + ] { + let session = format!("{route}-{stream}"); + let request = Request::post("/v1/chat/completions") + .header("content-type", "application/json") + .header("x-switchyard-session-id", &session) + .body(Body::from( + json!({"model": route, "stream": stream, "messages": [{"role": "user", "content": "Say hi"}]}) + .to_string(), + ))?; + let response = app.clone().oneshot(request).await?; + assert_eq!(response.status(), StatusCode::OK, "{session}"); + response.into_body().collect().await?; + let request = Request::get(format!("/v1/routing/session-stats?session_id={session}")) + .body(Body::empty())?; + let stats = app.clone().oneshot(request).await?; + let stats: Value = serde_json::from_slice(&stats.into_body().collect().await?.to_bytes())?; + assert_eq!(stats["cost"][source], 1, "{session}: {stats}"); + let recorded = stats["cost"]["total_usd"].as_f64().ok_or("missing cost")?; + assert!( + (recorded - usd).abs() < 1e-12, + "{session}: {recorded} != {usd}" + ); + } + Ok(()) +} diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 315d2470c..a5c2497f2 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -3128,6 +3128,11 @@ recent_turn_window = 1 async fn accepted_escalation_response_is_logged_once_as_the_final_answer() -> TestResult { let upstream = MockUpstream::start().await?; let temp_dir = tempfile::tempdir()?; + let prices = temp_dir.path().join("prices.json"); + std::fs::write( + &prices, + r#"{"model/strong":{"input":10,"output":20,"cache_read":0.2,"cache_write":3},"model/weak":{"input":1,"output":5,"cache_read":0.1,"cache_write":2}}"#, + )?; let state = load_test_config(&format!( r#" schema_version = 1 @@ -3161,7 +3166,8 @@ escalation = {{ confirmations = 1 }} "#, base_url = upstream.base_url ))? - .with_routing_log(temp_dir.path().join("routing.jsonl"))?; + .with_routing_log(temp_dir.path().join("routing.jsonl"))? + .with_pricing_file(prices)?; let app = build_switchyard_router(state); let response = send_with_headers( @@ -3192,6 +3198,16 @@ escalation = {{ confirmations = 1 }} assert_eq!(stats.status, StatusCode::OK); let stats = stats.json()?; assert_eq!(stats["total_calls"], 2); + let answer_cost = stats["answer_cost"]["total_usd"] + .as_f64() + .ok_or("missing answer cost")?; + let routing_cost = stats["routing_cost"]["total_usd"] + .as_f64() + .ok_or("missing routing cost")?; + assert!((answer_cost - 0.0000137).abs() < 1e-12); + assert!((routing_cost - 0.0000714).abs() < 1e-12); + assert_eq!(stats["cost"]["estimated_calls"], 2); + assert_eq!(stats["total_prompt_tokens"], 20); assert_eq!(stats["total_completion_tokens"], 4); assert_eq!(stats["models"]["model/weak"]["calls"], 1); @@ -4699,9 +4715,15 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted let upstream = MockUpstream::start().await?; let temp_dir = tempfile::tempdir()?; let log_path = temp_dir.path().join("routing.jsonl"); + let prices = temp_dir.path().join("prices.json"); + std::fs::write( + &prices, + r#"{"model/strong":{"input":10,"output":20,"cache_read":0.2,"cache_write":3}}"#, + )?; // The fixed seed selects `first` for these requests while keeping `second` enabled. - let state = - weighted_random_state(&upstream.base_url, [1000, 1])?.with_routing_log(&log_path)?; + let state = weighted_random_state(&upstream.base_url, [1000, 1])? + .with_routing_log(&log_path)? + .with_pricing_file(&prices)?; let app = build_switchyard_router(state); let cases = [ ( @@ -4727,7 +4749,14 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted for (path, body) in cases { let previous_call_count = upstream.calls.lock().await.len(); - let response = send(&app, "POST", path, Some(body)).await?; + let response = send_with_headers( + &app, + "POST", + path, + Some(body), + &[("x-switchyard-session-id", "cost-fallback")], + ) + .await?; assert_eq!(response.status, StatusCode::OK); assert_eq!( response @@ -4769,10 +4798,40 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted .lines() .map(serde_json::from_str::) .collect::, _>>()?; - assert_eq!(records.len(), 3); - assert!(records.iter().all(|record| { - record["model"] == "model/strong" && record.get("fallback_reason").is_none() - })); + assert_eq!(records.len(), 6); + assert_eq!( + records + .iter() + .filter(|record| record["model"] == "model/strong") + .count(), + 3 + ); + assert_eq!( + records + .iter() + .filter(|record| record["model"] == "model/weak" && record["cost"].is_null()) + .count(), + 3 + ); + let session = send( + &app, + "GET", + "/v1/routing/session-stats?session_id=cost-fallback", + None, + ) + .await? + .json()?; + assert_eq!(session["cost"]["unknown_calls"], 3); + assert_eq!(session["cost"]["estimated_calls"], 3); + assert!(session["cost"]["total_usd"].is_null()); + assert!( + (session["cost"]["known_usd"] + .as_f64() + .ok_or("missing subtotal")? + - 0.0002142) + .abs() + < 1e-12 + ); let previous_call_count = upstream.calls.lock().await.len(); let response = send( @@ -5340,7 +5399,7 @@ async fn advisor_route_streaming_approval_replays_provider_events() -> TestResul assert_eq!(events[1]["choices"][0]["delta"]["content"], "hello"); assert_eq!(events[2]["choices"][0]["delta"]["content"], "-partial"); assert_eq!(events[3]["choices"][0]["delta"]["content"], "-final"); - // Provider-specific usage detail rides through untouched. + // The response keeps the provider's usage fields unchanged. assert_eq!( events[3]["usage"]["prompt_tokens_details"]["cache_creation_tokens"], 2 @@ -5829,3 +5888,68 @@ seed = 17 } Ok(()) } + +#[tokio::test] +async fn session_cost_uses_served_model_rates_for_every_endpoint() -> TestResult { + let upstream = MockUpstream::start().await?; + let directory = tempfile::tempdir()?; + let prices = directory.path().join("prices.json"); + std::fs::write( + &prices, + r#"{"model/a":{"input":1,"output":5,"cache_read":0.1,"cache_write":2},"model/b":{"input":10,"output":20,"cache_read":0.2,"cache_write":3}}"#, + )?; + let state = random_state( + &upstream.base_url, + &[("route-a", &["model/a"]), ("route-b", &["model/b"])], + )? + .with_routing_log(directory.path().join("routing.jsonl"))? + .with_pricing_file(&prices)?; + let app = build_switchyard_router(state); + for (endpoint, stream, model, expected) in [ + ("/v1/chat/completions", false, "route-a", 0.0000137), + ("/v1/messages", false, "route-a", 0.0000137), + ("/v1/responses", false, "route-a", 0.0000137), + ("/v1/chat/completions", true, "route-a", 0.0000327), + ("/v1/messages", true, "route-a", 0.0000327), + ("/v1/responses", true, "route-b", 0.0001374), + ] { + let session = format!("cost-{model}-{endpoint}-{stream}"); + let mut body = + json!({"model":model,"stream":stream,"messages":[{"role":"user","content":"hello"}]}); + if endpoint == "/v1/responses" { + body = json!({"model":model,"stream":stream,"input":"hello"}); + } + if endpoint == "/v1/messages" { + body["max_tokens"] = json!(100); + } + let response = send_with_headers( + &app, + "POST", + endpoint, + Some(body), + &[("x-switchyard-session-id", &session)], + ) + .await?; + assert_eq!(response.status, StatusCode::OK, "{}", response.text()?); + let stats = send( + &app, + "GET", + &format!("/v1/routing/session-stats?session_id={session}"), + None, + ) + .await? + .json()?; + assert_eq!(stats["total_calls"], 1); + assert_eq!(stats["cost"]["estimated_calls"], 1); + let actual = stats["cost"]["total_usd"] + .as_f64() + .ok_or("missing session cost")?; + assert!( + (actual - expected).abs() < 1e-12, + "{endpoint}: {actual} != {expected}" + ); + assert_eq!(stats["routing_cost"]["total_usd"], 0.0); + } + assert_eq!(upstream.calls.lock().await.len(), 6); + Ok(()) +} diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 88aa8a324..9282c71cc 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -1256,8 +1256,13 @@ fn decode_anthropic_usage(value: Option<&Value>) -> Usage { .and_then(Value::as_u64); let output_tokens = value.get("output_tokens").and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(value), input_tokens, - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + value, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens, total_tokens: input_tokens.zip(output_tokens).map(|(input, output)| { input @@ -1289,7 +1294,7 @@ fn encode_anthropic_usage(usage: &Usage) -> Value { if let Some(thinking_tokens) = usage.reasoning_tokens { value["output_tokens_details"] = json!({"thinking_tokens": thinking_tokens}); } - value + crate::codecs::common::encode_usage_metadata(value, usage) } // Maps Anthropic stop reasons to normalized stop reasons. diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs index 908941a89..f901d875e 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs @@ -628,6 +628,20 @@ fn capture_anthropic_usage(state: &mut StreamTranslationState, usage: &Value) { if let Some(value) = usage.get("input_tokens").and_then(Value::as_u64) { state.usage.input_tokens = Some(value); } + if let Some(cost) = crate::codecs::common::provider_cost(usage) { + state.usage.provider_cost = Some(cost); + } + if let Some(tokens) = usage + .get("cache_creation") + .and_then(|details| details.get("ephemeral_1h_input_tokens")) + .and_then(Value::as_u64) + { + state + .usage + .cache + .get_or_insert_with(Default::default) + .cache_creation_1h_input_tokens = Some(tokens); + } if let Some(value) = usage.get("output_tokens").and_then(Value::as_u64) { state.usage.output_tokens = Some(value); } @@ -676,7 +690,7 @@ fn anthropic_stream_usage(state: &StreamTranslationState) -> Value { json!({"thinking_tokens": value}), ); } - Value::Object(usage) + crate::codecs::common::encode_usage_metadata(Value::Object(usage), &state.usage) } // Converts any upstream message ID into an Anthropic-looking message ID. diff --git a/crates/switchyard-translation/src/codecs/common.rs b/crates/switchyard-translation/src/codecs/common.rs index ddd795f80..303c90321 100644 --- a/crates/switchyard-translation/src/codecs/common.rs +++ b/crates/switchyard-translation/src/codecs/common.rs @@ -231,6 +231,51 @@ pub(crate) fn provider_extensions( extensions } +/// Invalid charges remain unknown so accounting can use configured token rates. +pub(crate) fn provider_cost(value: &Map) -> Option { + value + .get("cost") + .and_then(Value::as_f64) + .filter(|cost| cost.is_finite() && *cost >= 0.0) +} + +/// Cache duration must survive decoding because one-hour writes can have a different rate. +pub(crate) fn cache_usage_details( + value: &Map, + cached: Option, + created: Option, +) -> Option> { + let one_hour = value + .get("cache_creation") + .and_then(|details| details.get("ephemeral_1h_input_tokens")) + .and_then(Value::as_u64); + let mut cache = crate::llm::Usage::cache_details(cached, created); + if let Some(tokens) = one_hour { + cache + .get_or_insert_with(Default::default) + .cache_creation_1h_input_tokens = Some(tokens); + } + cache +} + +/// These fields preserve provider charges and cache duration across response formats. +pub(crate) fn encode_usage_metadata(mut value: Value, usage: &crate::llm::Usage) -> Value { + if let Some(cost) = usage + .provider_cost + .filter(|cost| cost.is_finite() && *cost >= 0.0) + { + value["cost"] = serde_json::json!(cost); + } + if let Some(tokens) = usage + .cache + .as_ref() + .and_then(|cache| cache.cache_creation_1h_input_tokens) + { + value["cache_creation"] = serde_json::json!({"ephemeral_1h_input_tokens": tokens}); + } + value +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 907518fe9..4e00f5d0a 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -1248,6 +1248,7 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage { }) .and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(value), input_tokens: value .get("prompt_tokens") .and_then(Value::as_u64) @@ -1256,7 +1257,11 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage { .saturating_sub(cached_input_tokens.unwrap_or(0)) .saturating_sub(cache_creation_input_tokens.unwrap_or(0)) }), - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + value, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens: value.get("completion_tokens").and_then(Value::as_u64), total_tokens: value.get("total_tokens").and_then(Value::as_u64), reasoning_tokens: value @@ -1287,7 +1292,7 @@ pub(crate) fn encode_openai_usage(usage: &Usage) -> Value { if usage.cached_input_tokens().is_some() || usage.cache_creation_input_tokens().is_some() { value["prompt_tokens_details"] = json!({ "cached_tokens": usage.cached_input_tokens().unwrap_or(0), - "cache_creation_tokens": usage.cache_creation_input_tokens().unwrap_or(0), + "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), }); } if let Some(reasoning_tokens) = usage.reasoning_tokens { @@ -1295,7 +1300,7 @@ pub(crate) fn encode_openai_usage(usage: &Usage) -> Value { "reasoning_tokens": reasoning_tokens, }); } - value + crate::codecs::common::encode_usage_metadata(value, usage) } /// Maps OpenAI finish reasons to normalized stop reasons. diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index c86749728..207ebf33f 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -365,6 +365,7 @@ fn openai_usage(usage: &Map) -> Usage { }) .and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(usage), input_tokens: usage .get("prompt_tokens") .and_then(Value::as_u64) @@ -373,7 +374,11 @@ fn openai_usage(usage: &Map) -> Usage { .saturating_sub(cached_input_tokens.unwrap_or(0)) .saturating_sub(cache_creation_input_tokens.unwrap_or(0)) }), - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + usage, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens: usage.get("completion_tokens").and_then(Value::as_u64), total_tokens: usage.get("total_tokens").and_then(Value::as_u64), reasoning_tokens: usage @@ -471,10 +476,10 @@ fn openai_usage_value(state: &StreamTranslationState) -> Value { { usage["prompt_tokens_details"] = json!({ "cached_tokens": cache_read_tokens, - "cache_creation_tokens": cache_creation_tokens, + "cache_write_tokens": cache_creation_tokens, }); } - usage + crate::codecs::common::encode_usage_metadata(usage, &state.usage) } // Converts any upstream message ID into an OpenAI-looking stream ID. diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 43fab2806..f440dd2e7 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -2100,8 +2100,13 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage { .or_else(|| value.get("completion_tokens")) .and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(value), input_tokens, - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + value, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens, total_tokens: value .get("total_tokens") @@ -2132,19 +2137,22 @@ fn encode_responses_usage(usage: &Usage) -> Value { // upstream that reports no cache or reasoning breakdown means zero, not "unknown"; omitting // them instead yields a payload that fails to deserialize into the OpenAI SDK's // ResponseUsage, which types both as non-optional. - json!({ - "input_tokens": input_tokens, - "output_tokens": usage.output_tokens.unwrap_or(0), - "total_tokens": usage - .total_tokens - .or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0))) - .unwrap_or(0), - "input_tokens_details": { - "cached_tokens": usage.cached_input_tokens().unwrap_or(0), - "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), - }, - "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)}, - }) + crate::codecs::common::encode_usage_metadata( + json!({ + "input_tokens": input_tokens, + "output_tokens": usage.output_tokens.unwrap_or(0), + "total_tokens": usage + .total_tokens + .or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0))) + .unwrap_or(0), + "input_tokens_details": { + "cached_tokens": usage.cached_input_tokens().unwrap_or(0), + "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), + }, + "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)}, + }), + usage, + ) } /// Re-emits captured request extensions that the Responses format accepts. diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index 35f3a1411..06e5576be 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -1163,8 +1163,13 @@ fn responses_usage(usage: &serde_json::Map) -> Usage { }); let output_tokens = usage.get("output_tokens").and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(usage), input_tokens, - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + usage, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens, total_tokens: usage .get("total_tokens") @@ -1189,18 +1194,21 @@ fn responses_usage_value(usage: &Usage) -> Value { + usage.cache_creation_input_tokens().unwrap_or(0); // Both detail objects are always present, for the same reason as the buffered encoder: the // Responses schema types them as required, so a missing breakdown serializes as zero. - json!({ - "input_tokens": input_tokens, - "output_tokens": usage.output_tokens.unwrap_or(0), - "total_tokens": usage.total_tokens.unwrap_or_else(|| { - input_tokens + usage.output_tokens.unwrap_or(0) + crate::codecs::common::encode_usage_metadata( + json!({ + "input_tokens": input_tokens, + "output_tokens": usage.output_tokens.unwrap_or(0), + "total_tokens": usage.total_tokens.unwrap_or_else(|| { + input_tokens + usage.output_tokens.unwrap_or(0) + }), + "input_tokens_details": { + "cached_tokens": usage.cached_input_tokens().unwrap_or(0), + "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), + }, + "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)}, }), - "input_tokens_details": { - "cached_tokens": usage.cached_input_tokens().unwrap_or(0), - "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), - }, - "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)}, - }) + usage, + ) } // Longest response-id discriminator embedded verbatim in a synthesized item id. OpenAI rejects diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs index cef7155dd..3aed128b5 100644 --- a/crates/switchyard-translation/tests/response_translation.rs +++ b/crates/switchyard-translation/tests/response_translation.rs @@ -6,7 +6,7 @@ pub mod common; use pretty_assertions::assert_eq; -use serde_json::json; +use serde_json::{Value, json}; use switchyard_translation::{ ContentBlock, PreservationPolicy, StopReason, TranslationEngine, TranslationPolicy, WireFormat, }; @@ -377,7 +377,7 @@ fn responses_reasoning_usage_translates_to_openai_chat_usage_details() -> TestRe json!({"reasoning_tokens": 3}) ); assert_eq!( - output["usage"]["prompt_tokens_details"]["cache_creation_tokens"], + output["usage"]["prompt_tokens_details"]["cache_write_tokens"], 2 ); let decoded = engine @@ -1335,3 +1335,63 @@ fn responses_custom_tool_call_output_round_trips_with_request_extensions() -> Te assert_eq!(call["function"]["arguments"], "{\"input\":\"ls -la\"}"); Ok(()) } + +#[test] +fn translated_usage_keeps_charges_and_cache_duration() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({"id":"msg_cost", "type":"message", "role":"assistant", "model":"claude", + "content":[{"type":"text","text":"ok"}], "stop_reason":"end_turn", + "usage":{"input_tokens":50,"output_tokens":10,"cache_read_input_tokens":20, + "cache_creation_input_tokens":30,"cache_creation":{"ephemeral_1h_input_tokens":10},"cost":0.25}}); + for target in [WireFormat::OpenAiChat, WireFormat::OpenAiResponses] { + let translated = engine + .translate_response( + WireFormat::AnthropicMessages, + target, + &body, + &TranslationPolicy::default(), + )? + .body; + assert_eq!(translated["usage"]["cost"], 0.25); + assert_eq!( + translated["usage"]["cache_creation"]["ephemeral_1h_input_tokens"], + 10 + ); + let detail = if target == WireFormat::OpenAiChat { + "prompt_tokens_details" + } else { + "input_tokens_details" + }; + assert_eq!(translated["usage"][detail]["cache_write_tokens"], 30); + let restored = engine + .translate_response( + target, + WireFormat::AnthropicMessages, + &translated, + &TranslationPolicy::default(), + )? + .body; + assert_eq!(restored["usage"]["cost"], 0.25); + } + for cost in [json!(0), json!(-1), json!("0.25"), Value::Null] { + let mut altered = body.clone(); + altered["usage"]["cost"] = cost.clone(); + let translated = engine + .translate_response( + WireFormat::AnthropicMessages, + WireFormat::OpenAiChat, + &altered, + &TranslationPolicy::default(), + )? + .body; + assert_eq!( + translated["usage"]["cost"], + if cost == json!(0) { + json!(0.0) + } else { + Value::Null + } + ); + } + Ok(()) +} diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index 2e4f3e53c..635ba19e7 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -3582,3 +3582,47 @@ fn responses_custom_tool_calls_decode_from_the_finished_item() -> TestResult { } Ok(()) } + +/// The final output-only delta must not erase the charge or cache duration from message_start. +#[test] +fn streamed_usage_keeps_charge_and_duration_after_output_only_delta() -> TestResult { + let engine = TranslationEngine::default(); + for target in [WireFormat::OpenAiChat, WireFormat::OpenAiResponses] { + let mut state = StreamTranslationState::new(WireFormat::AnthropicMessages, target); + let mut events = Vec::new(); + for event in [ + json!({"type":"message_start","message":{"id":"msg_cost","type":"message","role":"assistant","model":"claude","content":[],"usage":{"input_tokens":50,"output_tokens":0,"cache_read_input_tokens":20,"cache_creation_input_tokens":30,"cache_creation":{"ephemeral_1h_input_tokens":10},"cost":0.25}}}), + json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":10}}), + json!({"type":"message_stop"}), + ] { + events.extend(engine.translate_event( + &mut state, + WireFormat::AnthropicMessages, + target, + &event, + )?); + } + events.extend(engine.finish_stream(&mut state, target)?); + let usage = events + .iter() + .rev() + .find_map(|event| { + if target == WireFormat::OpenAiResponses { + event.get("response")?.get("usage") + } else { + event.get("usage") + } + }) + .ok_or("missing final usage")?; + assert_eq!(usage["cost"], 0.25); + assert_eq!(usage["cache_creation"]["ephemeral_1h_input_tokens"], 10); + let detail = if target == WireFormat::OpenAiChat { + "prompt_tokens_details" + } else { + "input_tokens_details" + }; + assert_eq!(usage[detail]["cache_write_tokens"], 30); + assert_eq!(state.usage.output_tokens, Some(10)); + } + Ok(()) +} diff --git a/docs/cli_reference.md b/docs/cli_reference.md index e5da32455..30b675818 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -20,6 +20,7 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | +| `--pricing-file PATH` | None | Price each routing record from this JSON file of USD rates per million tokens; requires `--routing-log-file`. See [Display Switchyard session costs](integrations/costs.md). | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | diff --git a/docs/integrations/costs.md b/docs/integrations/costs.md new file mode 100644 index 000000000..34169e800 --- /dev/null +++ b/docs/integrations/costs.md @@ -0,0 +1,166 @@ +# Display Switchyard session costs + +Switchyard can price each observed call using the model that served it. Its session +report includes the answer and the classifier or judge calls used to route it. +The example displays below show this report in Claude Code, pi, and Oh My Pi. + +## Configure rates + +Start the server with a routing log and an optional pricing file: + +```bash +switchyard-server --config routes.toml --routing-log-file routing.jsonl --pricing-file prices.json +``` + +The JSON file maps the served model IDs in the routing log to USD per million +tokens. These are illustrative rates; supply your gateway's rates: + +```json +{ + "upstream/efficient": { + "input": 1, + "output": 5, + "cache_read": 0.1, + "cache_write": 2, + "cache_write_1h": 3 + }, + "upstream/capable": { + "input": 10, + "output": 20, + "cache_read": 0.2, + "cache_write": 3 + } +} +``` + +The four base rates are required. `cache_write_1h` is optional. The server rejects +unknown fields, duplicate model IDs, empty or padded IDs, and rates outside +0 to 10000. Explicit zero rates mean free usage. Missing model rates mean unknown +cost. Model IDs must identify a single rate table across gateways; use distinct +served IDs when the same model has different gateway rates. Rates load at startup. +Restart to load changes; saved records keep the cost calculated when the call +completed. + +A charge that the upstream reports in USD takes precedence over an estimate, +even when it is zero. The server reads it from `usage.cost` in the response, +which OpenRouter and streamed LiteLLM responses include. For a buffered +response, LiteLLM sends the charge in the `x-litellm-response-cost` header +instead, and the server records that value. The server cannot see headers on +classifier and judge calls, so those calls use `usage.cost` or the estimate. Without a charge, Switchyard multiplies the separate input, +cache-read, cache-write, and output token counts by the configured rates. +Reasoning tokens are already part of output usage and are not added again. +When the provider reports one-hour cache-write tokens, their rate is required. +When it does not report cache duration, the estimate uses `cache_write` for all +writes. Supply rates that match the requested tier; this table does not apply +long-context or service-tier surcharges automatically. + +You can omit `--pricing-file` to record provider-reported charges alone. +Translation preserves charges and reported one-hour cache-write counts across +Chat Completions, Anthropic Messages, and Responses. Translated Chat Completions +responses emit cache writes as `prompt_tokens_details.cache_write_tokens`, which pi and Oh My Pi read. +The decoder also accepts the older `cache_creation_tokens` field. Same-format +raw response replay preserves the upstream JSON. + +## Read a session report + +The caller must send a session header that Switchyard recognizes on its model +requests, such as `x-switchyard-session-id: my-session`. Read that session with: + +```bash +curl 'http://localhost:4000/v1/routing/session-stats?session_id=my-session' +``` + +The response adds `cost`, `answer_cost`, and `routing_cost`, plus `cost` for each +model. Each total has `known_usd`, `total_usd`, `unknown_calls`, `estimated_calls`, +and `provider_reported_calls`. `total_usd` is null if any observed call has no +known cost. `known_usd` remains the subtotal of priced calls. Old records without +cost fields contribute unpriced calls. The routing log stores each priced call's +`cost` as an object, such as `{ "usd": 0.01, "source": "estimated" }`. Its `source` +is `"estimated"` or `"provider_reported"`; an unpriced call has `"cost": null`. +When the provider reports one-hour cache writes, the record also names that +part of `cache_creation_tokens` in `cache_creation_1h_tokens`. + +Totals cover recorded calls. Failed answer, classifier, and stream calls that +reach the server's observation code are unpriced. Internal HTTP retries, +cancelled requests, and streams abandoned before observation completes can +leave charges unrecorded. The report cannot establish the provider's bill. +Session totals include all recorded branches and compaction calls carrying the +same session ID; a CLI's active-branch total can cover a different set of calls. +The pricing table stays in memory for the routing log writer's lifetime and does +not grow per call. Its size follows the pricing file. The log grows with recorded +calls; this option adds no rotation or size limit. Separate replica logs produce +separate totals. If a log append fails, the server warns and continues serving; +missed records are not recovered automatically. The session endpoint scans the +log on each read. The displays fetch on agent +completion, rather than polling in the background. + +## Claude Code + +Use the Python example as a status-line command. Set the repository path and the +server address in Claude Code's settings: + +```json +{ + "statusLine": { + "type": "command", + "command": "python3 /path/to/Switchyard/examples/costs/claude_statusline.py --base-url http://localhost:4000" + } +} +``` + +The command reads `session_id` from Claude Code's status-line JSON input and +prints the Switchyard total, including routing cost. If you already have a +status-line command, call the example from that command to retain your display. +You can also run it from a shell with `--session-id my-session`. + +Claude Code's native cost display computes prices locally. Its managed +[`modelPricing`](https://code.claude.com/docs/en/settings-reference#modelpricing) +setting supports model and gateway-alias rate tables in version 2.1.242 or later. +Claude ignores that setting in ordinary user settings and `--settings`. +A fixed rate under a routing alias cannot describe differently priced targets or +include hidden judge calls. The Switchyard status line displays a separate total; +it does not change Claude Code's native cost or budget enforcement. + +## pi + +Load the example extension with your existing Switchyard provider configuration: + +```bash +pi -e /path/to/Switchyard/examples/costs/pi.ts --provider switchyard --model switchyard +``` + +The extension sends `x-switchyard-session-id` and shows a separate footer after +each agent turn. Set `SWITCHYARD_BASE_URL` when the server is elsewhere. Keep the +helper `display.ts` beside `pi.ts` when copying the example. + +pi's OpenAI adapters price the local model entry from its `cost` table. The +response's served model ID or `usage.cost` does not replace that table. A fixed +rate is useful for a fixed target; a router alias needs a separate display for +per-call target pricing. Keep the OpenAI request API described in the +[pi guide](pi.md#which-request-api). + +## Oh My Pi + +Load the example with an `anthropic-messages` Switchyard provider: + +```bash +omp -e /path/to/Switchyard/examples/costs/omp.ts --model switchyard/switchyard +``` + +Oh My Pi sends a session header on that API, so the footer can look up the +session report. The current custom-provider OpenAI adapters send no session +header; this example cannot show their session totals. Use the configuration in +[the Oh My Pi guide](oh_my_pi.md#which-request-api). Keep `display.ts` beside +`omp.ts`, and set `SWITCHYARD_BASE_URL` for another server address. + +Oh My Pi's custom `switchyard` provider uses configured model rates for its +native cost. Its supported OpenRouter integration can instead use `usage.cost`. +Returning the same field to a provider named `switchyard` does not activate that +behavior. The footer leaves native cost and budget enforcement unchanged. + +## Codex + +Codex's token display does not accept a dollar rate from Switchyard. Use the +session report or the Python command with an explicitly supplied session ID. +Per-request pricing still helps compare routed workloads even when the caller +has no native dollar display. diff --git a/docs/integrations/oh_my_pi.md b/docs/integrations/oh_my_pi.md index 2fadf0fa4..e0a225840 100644 --- a/docs/integrations/oh_my_pi.md +++ b/docs/integrations/oh_my_pi.md @@ -93,8 +93,10 @@ replay or compaction. Port 4000 is also the default port for `omp`'s `litellm` provider and for `omp auth-gateway`. If `LITELLM_API_KEY` is set, `omp` probes `http://localhost:4000/v1` as a LiteLLM proxy. In that case, run Switchyard on another port or set -`LITELLM_BASE_URL`. Set `cost` on the model entry if you want `omp` to show a non-zero -cost. +`LITELLM_BASE_URL`. Set `cost` on the model entry for `omp`'s native estimate. +One rate table under a router alias is an approximation when targets have different prices. For a +separate total that includes routing calls, see +[Display Switchyard session costs](costs.md). ## Claude through an LLM gateway diff --git a/docs/integrations/pi.md b/docs/integrations/pi.md index 75a2853b3..07ee70dc1 100644 --- a/docs/integrations/pi.md +++ b/docs/integrations/pi.md @@ -97,7 +97,10 @@ message. When routing picks another target on the next turn, pi treats the chang model switch: it drops thinking signatures and turns off overflow compaction. pi's OpenAI clients keep the local id `switchyard` instead. -Set `cost` on the model entry if you want pi to show a non-zero cost. +Set `cost` on the model entry for pi's native estimate. One rate table under a +router alias is an approximation when targets have different prices. For a +separate total that uses each served model and includes routing calls, see +[Display Switchyard session costs](costs.md). [`benchmark/run-baseline.sh`](../../benchmark/README.md) runs Terminal-Bench tasks with pi through Switchyard when you pass `--agent pi`. @@ -155,19 +158,9 @@ second shows `cached_tokens` close to `prompt_tokens`. If the second record show #### Estimate the cost -The routing log records token counts, not prices. To estimate what a request cost, -multiply each count in its record by the matching price and add the results: - -| Tokens in the record | Price | -|---|---| -| `prompt_tokens - cached_tokens - cache_creation_tokens` | Input | -| `cached_tokens` | Cache read | -| `cache_creation_tokens` | Cache write | -| `completion_tokens` | Output | - -Write the prices to `prices.json` in USD per million tokens, and key each entry by the -record's `model` value. This example uses Anthropic's list prices for Claude Sonnet 5 on -2026-10-02: +Start the server with `--pricing-file prices.json` as well, and each record gets a `cost`. +The file maps each record's `model` value to prices in USD per million tokens. This +example uses Anthropic's list prices for Claude Sonnet 5 on 2026-10-02: ```json { @@ -176,26 +169,17 @@ record's `model` value. This example uses Anthropic's list prices for Claude Son ``` The example's `cache_write` price is for a 5-minute cache. A 1-hour cache costs 2 times -the input price instead of 1.25 times. The routing log does not say which cache the -gateway used. A gateway may also charge its own prices, so the result is an estimate, not -the gateway's bill. This command prints one estimated cost per record, or a warning for a -model that has no entry in `prices.json`: +the input price instead of 1.25 times; set it as `cache_write_1h`. When the gateway +reports its own charge, the server records that charge instead of the estimate. Read +each request's cost with: ```bash -jq -r --slurpfile prices prices.json ' - . as $r - | ($prices[0][$r.model // ""]) as $p - | if $p == null then - "warning: no price for model \($r.model); add it to prices.json" - else - ((($r.prompt_tokens // 0) - ($r.cached_tokens // 0) - ($r.cache_creation_tokens // 0)) * $p.input - + ($r.cached_tokens // 0) * $p.cache_read - + ($r.cache_creation_tokens // 0) * $p.cache_write - + ($r.completion_tokens // 0) * $p.output) / 1000000 - | "\($r.route_id) \($r.model) estimated $\(. * 1000000 | round / 1000000)" - end' PATH +jq -c '{route_id, model, cost}' PATH ``` +[Display Switchyard session costs](costs.md) describes the pricing file and how the +server totals a session. + ### Thinking With `reasoning: true` on the model entry, pi sends a thinking level on every request, diff --git a/examples/costs/claude_statusline.py b/examples/costs/claude_statusline.py new file mode 100644 index 000000000..6adb3ee9a --- /dev/null +++ b/examples/costs/claude_statusline.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""This command prints Switchyard's observed session cost in a status line or a shell.""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default=os.environ.get("SWITCHYARD_BASE_URL", "http://localhost:4000")) + parser.add_argument("--session-id") + args = parser.parse_args() + try: + session_id = args.session_id + if not session_id: + payload = json.load(sys.stdin) + if not isinstance(payload, dict): + raise ValueError("invalid status-line input") + session_id = payload.get("session_id") + if not isinstance(session_id, str) or not session_id: + raise ValueError("missing session ID") + url = args.base_url.rstrip("/").removesuffix("/v1") + "/v1/routing/session-stats?" + urllib.parse.urlencode({"session_id": session_id}) + with urllib.request.urlopen(url, timeout=2) as response: + snapshot = json.load(response) + cost = snapshot["cost"] + if cost["total_usd"] is None: + text = f"${cost['known_usd']:.6f} known + {cost['unknown_calls']} unpriced calls" + else: + label = "estimated" if cost["estimated_calls"] else "reported" + text = f"${cost['total_usd']:.6f} {label}" + print(f"Switchyard {text} (routing ${snapshot['routing_cost']['known_usd']:.6f} known)") + except (OSError, ValueError, KeyError, TypeError, urllib.error.URLError): + print("Switchyard cost unavailable") + + +if __name__ == "__main__": + main() diff --git a/examples/costs/display.ts b/examples/costs/display.ts new file mode 100644 index 000000000..515fb0359 --- /dev/null +++ b/examples/costs/display.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +interface Totals { + known_usd: number; + total_usd: number | null; + unknown_calls: number; + estimated_calls: number; +} + +interface CostContext { + hasUI: boolean; + model?: { provider: string }; + sessionManager: { getSessionId(): string }; + ui: { setStatus(key: string, text: string | undefined): void }; +} + +// Both agents can display a separate footer without rewriting their saved usage. +export async function updateCost(ctx: CostContext): Promise { + if (!ctx.hasUI) return; + if (ctx.model?.provider !== "switchyard") { + ctx.ui.setStatus("switchyard-cost", undefined); + return; + } + const sessionId = ctx.sessionManager.getSessionId(); + const base = (process.env.SWITCHYARD_BASE_URL ?? "http://localhost:4000").replace(/\/$/, "").replace(/\/v1$/, ""); + try { + const response = await fetch(`${base}/v1/routing/session-stats?${new URLSearchParams({ session_id: sessionId })}`, { signal: AbortSignal.timeout(2000) }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const snapshot = await response.json() as { cost: Totals; routing_cost: Totals }; + const cost = snapshot.cost; + const text = cost.total_usd === null + ? `$${cost.known_usd.toFixed(6)} known + ${cost.unknown_calls} unpriced calls` + : `$${cost.total_usd.toFixed(6)} ${cost.estimated_calls ? "estimated" : "reported"}`; + if (ctx.sessionManager.getSessionId() === sessionId && ctx.model?.provider === "switchyard") { + ctx.ui.setStatus("switchyard-cost", `Switchyard ${text} (routing $${snapshot.routing_cost.known_usd.toFixed(6)} known)`); + } + } catch { + if (ctx.sessionManager.getSessionId() === sessionId && ctx.model?.provider === "switchyard") ctx.ui.setStatus("switchyard-cost", "Switchyard cost unavailable"); + } +} diff --git a/examples/costs/omp.ts b/examples/costs/omp.ts new file mode 100644 index 000000000..12455f765 --- /dev/null +++ b/examples/costs/omp.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ExtensionAPI } from "@oh-my-pi/pi-coding-agent"; +import { updateCost } from "./display.ts"; + +// OMP sends its session header on anthropic-messages requests. +export default function(omp: ExtensionAPI): void { + omp.on("session_start", (_event, ctx) => updateCost(ctx)); + omp.on("session_switch", (_event, ctx) => updateCost(ctx)); + omp.on("model_select", (_event, ctx) => updateCost(ctx)); + omp.on("agent_end", (_event, ctx) => updateCost(ctx)); +} diff --git a/examples/costs/pi.ts b/examples/costs/pi.ts new file mode 100644 index 000000000..54ed6b505 --- /dev/null +++ b/examples/costs/pi.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { updateCost } from "./display.ts"; + +export default function(pi: ExtensionAPI): void { + pi.on("before_provider_headers", (event, ctx) => { + if (ctx.model?.provider === "switchyard") { + event.headers["x-switchyard-session-id"] = ctx.sessionManager.getSessionId(); + } + }); + pi.on("session_start", (_event, ctx) => updateCost(ctx)); + pi.on("session_switch", (_event, ctx) => updateCost(ctx)); + pi.on("model_select", (_event, ctx) => updateCost(ctx)); + pi.on("agent_end", (_event, ctx) => updateCost(ctx)); +} From 9378479d4b63e74e4321d5dbcb77168e74473b28 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 20 Oct 2026 11:51:15 -0700 Subject: [PATCH 2/8] chore(costs): simplify pricing and consolidate coverage Signed-off-by: Elyas Mehtabuddin --- Cargo.lock | 1 - crates/switchyard-runner/Cargo.toml | 1 - crates/switchyard-runner/src/pricing.rs | 71 ++----- crates/switchyard-server/README.md | 2 +- .../tests/litellm_gateway.rs | 148 -------------- crates/switchyard-server/tests/server.rs | 77 +++++-- .../src/codecs/openai_chat/stream.rs | 92 ++------- .../src/codecs/responses/buffered.rs | 2 +- .../src/codecs/responses/stream.rs | 28 +-- docs/cli_reference.md | 2 +- docs/integrations/costs.md | 190 +++++------------- docs/integrations/oh_my_pi.md | 2 +- docs/integrations/pi.md | 4 +- examples/costs/claude_statusline.py | 45 ----- examples/costs/display.ts | 41 ---- examples/costs/omp.ts | 13 -- examples/costs/pi.ts | 17 -- 17 files changed, 146 insertions(+), 590 deletions(-) delete mode 100644 examples/costs/claude_statusline.py delete mode 100644 examples/costs/display.ts delete mode 100644 examples/costs/omp.ts delete mode 100644 examples/costs/pi.ts diff --git a/Cargo.lock b/Cargo.lock index 2597f3f97..705b86d08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3830,7 +3830,6 @@ dependencies = [ "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", - "tempfile", "thiserror 2.0.18", "tokio", "toml 1.1.3+spec-1.1.0", diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.toml index 443491ea9..3a80b61b3 100644 --- a/crates/switchyard-runner/Cargo.toml +++ b/crates/switchyard-runner/Cargo.toml @@ -32,5 +32,4 @@ tracing.workspace = true [dev-dependencies] async-trait.workspace = true futures-util.workspace = true -tempfile = "3" tokio.workspace = true diff --git a/crates/switchyard-runner/src/pricing.rs b/crates/switchyard-runner/src/pricing.rs index cee8379eb..152760192 100644 --- a/crates/switchyard-runner/src/pricing.rs +++ b/crates/switchyard-runner/src/pricing.rs @@ -11,8 +11,7 @@ use switchyard_protocol::Usage; use crate::RunnerError; -/// Pricing maps served model IDs to token rates in USD per million tokens. The server prices -/// routing records with it, and the desktop app prices its baseline comparison with it. +/// Pricing maps served model IDs to token rates in USD per million tokens. #[derive(Debug, Default)] pub struct Pricing(HashMap); @@ -105,11 +104,6 @@ impl Pricing { Ok(pricing) } - /// This function reports whether the file has rates for `model`. - pub fn contains(&self, model: &str) -> bool { - self.0.contains_key(model) - } - /// Provider charges take precedence because token rates may omit provider surcharges. pub fn cost(&self, model: &str, usage: &Usage) -> Option { if let Some(usd) = usage @@ -188,20 +182,6 @@ impl CostTotals { } self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); } - - /// This function adds another set of totals, such as one day's totals to a week. - pub fn merge(&mut self, other: &Self) { - let known = self.known_usd + other.known_usd; - if known.is_finite() { - self.known_usd = known; - self.estimated_calls += other.estimated_calls; - self.provider_reported_calls += other.provider_reported_calls; - } else { - self.unknown_calls += other.estimated_calls + other.provider_reported_calls; - } - self.unknown_calls += other.unknown_calls; - self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); - } } #[cfg(test)] @@ -209,37 +189,30 @@ mod tests { use super::*; #[test] - fn rejects_invalid_rate_tables() -> Result<(), Box> { + fn rejects_invalid_rate_tables() { let valid = r#"{"input":1,"output":5,"cache_read":0.1,"cache_write":2}"#; for rates in [ - r#"{"input":-1,"output":5,"cache_read":0.1,"cache_write":2}"#.to_string(), - r#"{"input":1,"output":5,"cache_read":0.1}"#.to_string(), - r#"{"input":1,"output":5,"cache_read":0.1,"cache_write":2,"cache_wirte":2}"# - .to_string(), - r#"{"input":1e999,"output":5,"cache_read":0.1,"cache_write":2}"#.to_string(), + r#"{"input":-1,"output":5,"cache_read":0.1,"cache_write":2}"#, + r#"{"input":1,"output":5,"cache_read":0.1}"#, + r#"{"input":1,"output":5,"cache_read":0.1,"cache_write":2,"cache_wirte":2}"#, + r#"{"input":1e999,"output":5,"cache_read":0.1,"cache_write":2}"#, ] { - let file = tempfile::NamedTempFile::new()?; - std::fs::write(file.path(), format!("{{\"model\":{rates}}}"))?; - assert!(Pricing::load(file.path()).is_err(), "{rates}"); + assert!( + Pricing::parse(format!("{{\"model\":{rates}}}").as_bytes()).is_err(), + "{rates}" + ); } - let file = tempfile::NamedTempFile::new()?; - std::fs::write( - file.path(), - format!("{{\"model\":{valid},\"model\":{valid}}}"), - )?; - assert!(Pricing::load(file.path()).is_err()); - Ok(()) + assert!( + Pricing::parse(format!("{{\"model\":{valid},\"model\":{valid}}}").as_bytes()).is_err() + ); } #[test] fn estimates_disjoint_buckets_and_prefers_reported_charges() -> Result<(), Box> { - let file = tempfile::NamedTempFile::new()?; - std::fs::write( - file.path(), - r#"{"model":{"input":1,"output":5,"cache_read":0.1,"cache_write":2,"cache_write_1h":3},"free":{"input":0,"output":0,"cache_read":0,"cache_write":0}}"#, + let prices = Pricing::parse( + br#"{"model":{"input":1,"output":5,"cache_read":0.1,"cache_write":2,"cache_write_1h":3},"free":{"input":0,"output":0,"cache_read":0,"cache_write":0}}"#, )?; - let prices = Pricing::load(file.path())?; let mut usage = Usage { input_tokens: Some(50), output_tokens: Some(10), @@ -308,15 +281,6 @@ mod tests { assert_eq!(totals.known_usd, 0.25); assert_eq!(totals.unknown_calls, 1); assert_eq!(totals.total_usd, None); - let mut week = CostTotals::default(); - week.add(Some(RecordedCost { - usd: 0.5, - source: CostSource::ProviderReported, - })); - assert_eq!(week.total_usd, Some(0.5)); - week.merge(&totals); - assert_eq!((week.known_usd, week.unknown_calls), (0.75, 1)); - assert_eq!(week.total_usd, None); totals.add(Some(RecordedCost { usd: f64::MAX, source: CostSource::ProviderReported, @@ -327,10 +291,5 @@ mod tests { })); assert!(totals.known_usd.is_finite()); assert_eq!(totals.unknown_calls, 2); - // Merging totals whose sum would overflow counts the merged priced calls as unknown. - let mut doubled = totals.clone(); - doubled.merge(&totals); - assert!(doubled.known_usd.is_finite()); - assert_eq!(doubled.unknown_calls, 6); } } diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 05929f950..d08b224b3 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -126,7 +126,7 @@ routing logging is disabled. Add `--pricing-file prices.json` to record each call's `cost` in USD. The server records the charge that the upstream reports and otherwise estimates it from the file's rates for the served -model. The session stats then include cost totals. [Display Switchyard session +model. The session stats then include cost totals. [Record Switchyard session costs](../../docs/integrations/costs.md) describes the file format. Clients can send `x-switchyard-origin: codex-cli` (or another client label) to include an diff --git a/crates/switchyard-server/tests/litellm_gateway.rs b/crates/switchyard-server/tests/litellm_gateway.rs index d760e423d..a83d00998 100644 --- a/crates/switchyard-server/tests/litellm_gateway.rs +++ b/crates/switchyard-server/tests/litellm_gateway.rs @@ -221,151 +221,3 @@ async fn codex_tool_call_streams_reach_response_completed() -> TestResult { } Ok(()) } - -/// The charge that NVIDIA's LiteLLM gateway reported in `x-litellm-response-cost` for a buffered -/// Chat Completions answer. The estimate from `RATES` is 0.000089 instead. -const HEADER_CHARGE: &str = "6.141e-05"; -/// The charge that the gateway reported as `usage.cost` at the end of a stream. The estimate -/// from `RATES` is 0.000059 instead. -const STREAM_CHARGE: f64 = 4.071e-05; -const RATES: &str = r#"{ - "model/claude": {"input": 1, "output": 5, "cache_read": 0.1, "cache_write": 2}, - "model/plain": {"input": 1, "output": 5, "cache_read": 0.1, "cache_write": 2} -}"#; - -/// Answers Chat Completions the way the gateway does. A buffered answer from `model/claude` -/// carries its charge only in the header. A streamed answer has zeroed breakdown headers and the -/// charge in the final usage. `model/plain` reports no charge. -async fn charging_gateway(Json(body): Json) -> Response { - let model = body["model"].as_str().unwrap_or_default().to_string(); - let mut usage = json!({"prompt_tokens": 9, "completion_tokens": 16, "total_tokens": 25}); - if body["stream"] == true { - usage = json!({"prompt_tokens": 9, "completion_tokens": 10, "total_tokens": 19, "cost": STREAM_CHARGE}); - let chunk = |delta: Value, finish: Value, usage: Value| { - json!({"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": model, - "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], "usage": usage}) - }; - let frames = [ - chunk( - json!({"role": "assistant", "content": "hi"}), - Value::Null, - Value::Null, - ), - chunk(json!({}), json!("stop"), usage), - ] - .into_iter() - .map(|event| event.to_string()) - .chain(["[DONE]".to_string()]) - .map(|data| Ok::(Event::default().data(data))); - let mut response = Sse::new(futures_util::stream::iter(frames)).into_response(); - let zero = axum::http::HeaderValue::from_static("0.0"); - response - .headers_mut() - .insert("x-litellm-response-cost-original", zero); - return response; - } - let mut response = Json(json!({ - "id": "c1", "object": "chat.completion", "created": 1, "model": model, "usage": usage, - "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}] - })) - .into_response(); - if model == "model/claude" { - let charge = axum::http::HeaderValue::from_static(HEADER_CHARGE); - response - .headers_mut() - .insert("x-litellm-response-cost", charge); - } - response -} - -/// The routing log records the gateway's charge when it reports one and estimates from the -/// pricing file only when it does not. -#[tokio::test] -async fn routing_log_records_the_gateway_charge() -> TestResult { - let app = Router::new().route("/v1/chat/completions", post(charging_gateway)); - let listener = TcpListener::bind("127.0.0.1:0").await?; - let root = format!("http://{}", listener.local_addr()?); - let gateway = Gateway { - root, - task: tokio::spawn(async move { axum::serve(listener, app).await }), - }; - let directory = tempfile::tempdir()?; - let config = directory.path().join("config.toml"); - std::fs::write( - &config, - format!( - r#" -schema_version = 1 - -[llm_clients.gateway] -format = "openai_chat" -base_url = "{root}/v1" -max_retries = 0 - -[targets.claude] -id = "model/claude" -llm_client = "gateway" - -[targets.plain] -id = "model/plain" -llm_client = "gateway" - -[routes.reported] -id = "gateway/reported" -type = "passthrough" -target = "claude" - -[routes.unreported] -id = "gateway/unreported" -type = "passthrough" -target = "plain" -"#, - root = gateway.root - ), - )?; - let prices = directory.path().join("prices.json"); - std::fs::write(&prices, RATES)?; - let state = load_server_state(&config)? - .with_routing_log(directory.path().join("routing.jsonl"))? - .with_pricing_file(&prices)?; - let app = build_switchyard_router(state); - let header_charge: f64 = HEADER_CHARGE.parse()?; - for (route, stream, usd, source) in [ - ( - "gateway/reported", - false, - header_charge, - "provider_reported_calls", - ), - ( - "gateway/reported", - true, - STREAM_CHARGE, - "provider_reported_calls", - ), - ("gateway/unreported", false, 0.000089, "estimated_calls"), - ] { - let session = format!("{route}-{stream}"); - let request = Request::post("/v1/chat/completions") - .header("content-type", "application/json") - .header("x-switchyard-session-id", &session) - .body(Body::from( - json!({"model": route, "stream": stream, "messages": [{"role": "user", "content": "Say hi"}]}) - .to_string(), - ))?; - let response = app.clone().oneshot(request).await?; - assert_eq!(response.status(), StatusCode::OK, "{session}"); - response.into_body().collect().await?; - let request = Request::get(format!("/v1/routing/session-stats?session_id={session}")) - .body(Body::empty())?; - let stats = app.clone().oneshot(request).await?; - let stats: Value = serde_json::from_slice(&stats.into_body().collect().await?.to_bytes())?; - assert_eq!(stats["cost"][source], 1, "{session}: {stats}"); - let recorded = stats["cost"]["total_usd"].as_f64().ok_or("missing cost")?; - assert!( - (recorded - usd).abs() < 1e-12, - "{session}: {recorded} != {usd}" - ); - } - Ok(()) -} diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index a5c2497f2..dd4179429 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -312,11 +312,17 @@ async fn upstream_chat( ); return Sse::new(stream).into_response(); } + let mut usage = json!({"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17, + "prompt_tokens_details": {"cached_tokens": 7, "cache_creation_tokens": 2}, + "completion_tokens_details": {"reasoning_tokens": 3}}); + if prompt == "reported-cost" { + usage["cost"] = json!(0.00004071); + } let events = [ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(), json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "hello"}}]}).to_string(), json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-partial"}}], "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6, "prompt_tokens_details": {"cached_tokens": 2, "cache_creation_tokens": 1}}}).to_string(), - json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-final"}}], "usage": {"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17, "prompt_tokens_details": {"cached_tokens": 7, "cache_creation_tokens": 2}, "completion_tokens_details": {"reasoning_tokens": 3}}}).to_string(), + json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-final"}}], "usage": usage}).to_string(), json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}).to_string(), "[DONE]".to_string(), ]; @@ -325,7 +331,15 @@ async fn upstream_chat( .into_iter() .map(|data| Ok::(Event::default().data(data))), ); - return Sse::new(stream).into_response(); + let mut response = Sse::new(stream).into_response(); + if prompt == "reported-cost" { + // LiteLLM streams report the charge in usage, with a zeroed breakdown header. + response.headers_mut().insert( + "x-litellm-response-cost-original", + HeaderValue::from_static("0.0"), + ); + } + return response; } if model == "model/advisor" { @@ -456,7 +470,7 @@ async fn upstream_chat( } else { "ok".to_string() }; - Json(json!({ + let mut response = Json(json!({ "id": "chatcmpl-test", "object": "chat.completion", "model": model, @@ -472,7 +486,15 @@ async fn upstream_chat( "prompt_tokens_details": {"cached_tokens": 7} } })) - .into_response() + .into_response(); + if prompt == "reported-cost" { + // Buffered LiteLLM answers report their charge only in this header. + response.headers_mut().insert( + "x-litellm-response-cost", + HeaderValue::from_static("6.141e-05"), + ); + } + response } async fn upstream_messages_requires_forwarded_oauth( @@ -5890,7 +5912,7 @@ seed = 17 } #[tokio::test] -async fn session_cost_uses_served_model_rates_for_every_endpoint() -> TestResult { +async fn session_cost_prefers_reported_charges_and_prices_served_models() -> TestResult { let upstream = MockUpstream::start().await?; let directory = tempfile::tempdir()?; let prices = directory.path().join("prices.json"); @@ -5905,19 +5927,31 @@ async fn session_cost_uses_served_model_rates_for_every_endpoint() -> TestResult .with_routing_log(directory.path().join("routing.jsonl"))? .with_pricing_file(&prices)?; let app = build_switchyard_router(state); - for (endpoint, stream, model, expected) in [ - ("/v1/chat/completions", false, "route-a", 0.0000137), - ("/v1/messages", false, "route-a", 0.0000137), - ("/v1/responses", false, "route-a", 0.0000137), - ("/v1/chat/completions", true, "route-a", 0.0000327), - ("/v1/messages", true, "route-a", 0.0000327), - ("/v1/responses", true, "route-b", 0.0001374), + for (endpoint, stream, model, prompt, expected) in [ + ("/v1/chat/completions", false, "route-a", "hello", 0.0000137), + ( + "/v1/messages", + false, + "route-a", + "reported-cost", + 0.00006141, + ), + ("/v1/responses", false, "route-b", "hello", 0.0000714), + ( + "/v1/chat/completions", + true, + "route-a", + "reported-cost", + 0.00004071, + ), + ("/v1/messages", true, "route-a", "hello", 0.0000327), + ("/v1/responses", true, "route-b", "hello", 0.0001374), ] { let session = format!("cost-{model}-{endpoint}-{stream}"); let mut body = - json!({"model":model,"stream":stream,"messages":[{"role":"user","content":"hello"}]}); + json!({"model":model,"stream":stream,"messages":[{"role":"user","content":prompt}]}); if endpoint == "/v1/responses" { - body = json!({"model":model,"stream":stream,"input":"hello"}); + body = json!({"model":model,"stream":stream,"input":prompt}); } if endpoint == "/v1/messages" { body["max_tokens"] = json!(100); @@ -5940,7 +5974,20 @@ async fn session_cost_uses_served_model_rates_for_every_endpoint() -> TestResult .await? .json()?; assert_eq!(stats["total_calls"], 1); - assert_eq!(stats["cost"]["estimated_calls"], 1); + let source = if prompt == "reported-cost" { + "provider_reported_calls" + } else { + "estimated_calls" + }; + assert_eq!(stats["cost"][source], 1); + assert_eq!( + stats["models"][if model == "route-a" { + "model/a" + } else { + "model/b" + }]["cost"][source], + 1 + ); let actual = stats["cost"]["total_usd"] .as_f64() .ok_or("missing session cost")?; diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index 207ebf33f..02cec4b6e 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -5,6 +5,7 @@ use serde_json::{Map, Value, json}; +use super::buffered::{decode_openai_usage, encode_openai_usage}; use crate::LlmResponseChunk; use crate::codecs::common::{first_nonempty_string, reasoning_text_from_details}; use crate::codecs::stream::{ @@ -12,7 +13,6 @@ use crate::codecs::stream::{ target_model_or_source_model, }; use crate::format::{FormatId, WireFormat}; -use crate::llm::Usage; /// Stream codec for OpenAI Chat Completions chunks. pub struct OpenAiChatStreamCodec; @@ -84,8 +84,8 @@ fn decode_openai_chat_stream( }); } - if let Some(usage) = object.get("usage").and_then(Value::as_object) { - let usage = openai_usage(usage); + if let Some(usage) = object.get("usage").filter(|usage| usage.is_object()) { + let usage = decode_openai_usage(Some(usage)); state.usage = usage.clone(); state.saw_backend_usage = true; out.push(LlmResponseChunk::Usage(usage)); @@ -324,7 +324,9 @@ fn encode_openai_chat_stream( state, json!({}), Some(openai_finish_reason(reason.as_deref())), - state.saw_backend_usage.then(|| openai_usage_value(state)), + state + .saw_backend_usage + .then(|| encode_openai_usage(&state.usage)), )] } LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => { @@ -346,53 +348,12 @@ fn finish_openai_chat_stream(state: &mut StreamTranslationState) -> Vec { state, json!({}), Some(openai_finish_reason(state.stop_reason.as_deref())), - state.saw_backend_usage.then(|| openai_usage_value(state)), + state + .saw_backend_usage + .then(|| encode_openai_usage(&state.usage)), )] } -// Normalizes OpenAI token usage fields. -fn openai_usage(usage: &Map) -> Usage { - let cached_input_tokens = usage - .get("prompt_tokens_details") - .and_then(|details| details.get("cached_tokens")) - .and_then(Value::as_u64); - let cache_creation_input_tokens = usage - .get("prompt_tokens_details") - .and_then(|details| { - details - .get("cache_write_tokens") - .or_else(|| details.get("cache_creation_tokens")) - }) - .and_then(Value::as_u64); - Usage { - provider_cost: crate::codecs::common::provider_cost(usage), - input_tokens: usage - .get("prompt_tokens") - .and_then(Value::as_u64) - .map(|tokens| { - tokens - .saturating_sub(cached_input_tokens.unwrap_or(0)) - .saturating_sub(cache_creation_input_tokens.unwrap_or(0)) - }), - cache: crate::codecs::common::cache_usage_details( - usage, - cached_input_tokens, - cache_creation_input_tokens, - ), - output_tokens: usage.get("completion_tokens").and_then(Value::as_u64), - total_tokens: usage.get("total_tokens").and_then(Value::as_u64), - reasoning_tokens: usage - .get("completion_tokens_details") - .and_then(|details| details.get("reasoning_tokens")) - .or_else(|| { - usage - .get("output_tokens_details") - .and_then(|details| details.get("reasoning_tokens")) - }) - .and_then(Value::as_u64), - } -} - // Builds a single OpenAI Chat stream chunk payload. fn openai_stream_chunk( state: &StreamTranslationState, @@ -419,7 +380,12 @@ fn openai_stream_chunk( // Builds the usage-only chunk OpenAI emits after the terminal choices chunk. fn openai_usage_chunk(state: &StreamTranslationState) -> Value { - let mut payload = openai_stream_chunk(state, json!({}), None, Some(openai_usage_value(state))); + let mut payload = openai_stream_chunk( + state, + json!({}), + None, + Some(encode_openai_usage(&state.usage)), + ); payload["choices"] = json!([]); payload } @@ -454,34 +420,6 @@ fn openai_tool_call_chunk( ) } -// Builds OpenAI usage payloads from normalized and provider-extra state. -fn openai_usage_value(state: &StreamTranslationState) -> Value { - let cache_creation_tokens = state.usage.cache_creation_input_tokens().unwrap_or(0); - let cache_read_tokens = state.usage.cached_input_tokens().unwrap_or(0); - let prompt_tokens = - state.usage.input_tokens.unwrap_or(0) + cache_creation_tokens + cache_read_tokens; - let completion_tokens = state.usage.output_tokens.unwrap_or(0); - let mut usage = json!({ - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": state.usage.total_tokens.unwrap_or(prompt_tokens + completion_tokens), - }); - if let Some(reasoning_tokens) = state.usage.reasoning_tokens { - usage["completion_tokens_details"] = json!({ - "reasoning_tokens": reasoning_tokens, - }); - } - if state.usage.cache_creation_input_tokens().is_some() - || state.usage.cached_input_tokens().is_some() - { - usage["prompt_tokens_details"] = json!({ - "cached_tokens": cache_read_tokens, - "cache_write_tokens": cache_creation_tokens, - }); - } - crate::codecs::common::encode_usage_metadata(usage, &state.usage) -} - // Converts any upstream message ID into an OpenAI-looking stream ID. fn openai_stream_id(state: &StreamTranslationState) -> String { let Some(id) = state.message_id.as_deref() else { diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index f440dd2e7..294e59740 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -2129,7 +2129,7 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage { } // Encodes normalized usage into Responses usage JSON. -fn encode_responses_usage(usage: &Usage) -> Value { +pub(super) fn encode_responses_usage(usage: &Usage) -> Value { let input_tokens = usage.input_tokens.unwrap_or(0) + usage.cached_input_tokens().unwrap_or(0) + usage.cache_creation_input_tokens().unwrap_or(0); diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index 06e5576be..b726bea71 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -17,6 +17,8 @@ use crate::codecs::stream::{ use crate::format::{FormatId, WireFormat}; use crate::llm::Usage; +use super::buffered::encode_responses_usage; + /// Stream codec for OpenAI Responses API events. pub struct OpenAiResponsesStreamCodec; @@ -850,7 +852,7 @@ fn responses_stream_response( tool_choice: "auto", tools: Vec::new(), top_p: None, - usage: responses_usage_value(&state.usage), + usage: encode_responses_usage(&state.usage), } } @@ -1187,30 +1189,6 @@ fn responses_usage(usage: &serde_json::Map) -> Usage { } } -// Builds OpenAI Responses usage payloads from normalized usage. -fn responses_usage_value(usage: &Usage) -> Value { - let input_tokens = usage.input_tokens.unwrap_or(0) - + usage.cached_input_tokens().unwrap_or(0) - + usage.cache_creation_input_tokens().unwrap_or(0); - // Both detail objects are always present, for the same reason as the buffered encoder: the - // Responses schema types them as required, so a missing breakdown serializes as zero. - crate::codecs::common::encode_usage_metadata( - json!({ - "input_tokens": input_tokens, - "output_tokens": usage.output_tokens.unwrap_or(0), - "total_tokens": usage.total_tokens.unwrap_or_else(|| { - input_tokens + usage.output_tokens.unwrap_or(0) - }), - "input_tokens_details": { - "cached_tokens": usage.cached_input_tokens().unwrap_or(0), - "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), - }, - "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)}, - }), - usage, - ) -} - // Longest response-id discriminator embedded verbatim in a synthesized item id. OpenAI rejects // item ids over 64 characters, and some upstreams issue response ids several hundred characters // long, so anything longer is replaced by a fixed-width digest. diff --git a/docs/cli_reference.md b/docs/cli_reference.md index 30b675818..84fddb034 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -20,7 +20,7 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | -| `--pricing-file PATH` | None | Price each routing record from this JSON file of USD rates per million tokens; requires `--routing-log-file`. See [Display Switchyard session costs](integrations/costs.md). | +| `--pricing-file PATH` | None | Price each routing record from this JSON file of USD rates per million tokens; requires `--routing-log-file`. See [Record Switchyard session costs](integrations/costs.md). | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | diff --git a/docs/integrations/costs.md b/docs/integrations/costs.md index 34169e800..3809cd872 100644 --- a/docs/integrations/costs.md +++ b/docs/integrations/costs.md @@ -1,166 +1,66 @@ -# Display Switchyard session costs +# Record Switchyard session costs -Switchyard can price each observed call using the model that served it. Its session -report includes the answer and the classifier or judge calls used to route it. -The example displays below show this report in Claude Code, pi, and Oh My Pi. - -## Configure rates - -Start the server with a routing log and an optional pricing file: +Switchyard records each observed call's cost in the routing log, including calls +made by classifiers and judges. To estimate costs that the upstream does not +report, start the server with a pricing file: ```bash switchyard-server --config routes.toml --routing-log-file routing.jsonl --pricing-file prices.json ``` -The JSON file maps the served model IDs in the routing log to USD per million -tokens. These are illustrative rates; supply your gateway's rates: +The JSON file maps served model IDs to USD per million tokens. These rates are +illustrative; supply your provider's rates: ```json { - "upstream/efficient": { - "input": 1, - "output": 5, - "cache_read": 0.1, - "cache_write": 2, + "upstream/model": { + "input": 1, "output": 5, "cache_read": 0.1, "cache_write": 2, "cache_write_1h": 3 - }, - "upstream/capable": { - "input": 10, - "output": 20, - "cache_read": 0.2, - "cache_write": 3 } } ``` -The four base rates are required. `cache_write_1h` is optional. The server rejects -unknown fields, duplicate model IDs, empty or padded IDs, and rates outside -0 to 10000. Explicit zero rates mean free usage. Missing model rates mean unknown -cost. Model IDs must identify a single rate table across gateways; use distinct -served IDs when the same model has different gateway rates. Rates load at startup. -Restart to load changes; saved records keep the cost calculated when the call -completed. - -A charge that the upstream reports in USD takes precedence over an estimate, -even when it is zero. The server reads it from `usage.cost` in the response, -which OpenRouter and streamed LiteLLM responses include. For a buffered -response, LiteLLM sends the charge in the `x-litellm-response-cost` header -instead, and the server records that value. The server cannot see headers on -classifier and judge calls, so those calls use `usage.cost` or the estimate. Without a charge, Switchyard multiplies the separate input, -cache-read, cache-write, and output token counts by the configured rates. -Reasoning tokens are already part of output usage and are not added again. -When the provider reports one-hour cache-write tokens, their rate is required. -When it does not report cache duration, the estimate uses `cache_write` for all -writes. Supply rates that match the requested tier; this table does not apply -long-context or service-tier surcharges automatically. - -You can omit `--pricing-file` to record provider-reported charges alone. -Translation preserves charges and reported one-hour cache-write counts across -Chat Completions, Anthropic Messages, and Responses. Translated Chat Completions -responses emit cache writes as `prompt_tokens_details.cache_write_tokens`, which pi and Oh My Pi read. -The decoder also accepts the older `cache_creation_tokens` field. Same-format -raw response replay preserves the upstream JSON. +The four base rates are required; `cache_write_1h` is optional. The server rejects +unknown fields, duplicate, empty or padded model IDs, and rates outside 0 to 10000. +Zero means free usage. Use distinct served IDs when gateways charge different +rates for the same model. Rates load once at startup; saved costs do not change +when the pricing file changes. -## Read a session report +A valid charge in `usage.cost` takes precedence, including zero. Buffered answers +can also report a charge in `x-litellm-response-cost`. Otherwise, Switchyard prices +input, output, cache reads and cache writes separately. Reasoning tokens already +count toward output. Reported one-hour cache writes require `cache_write_1h`; +writes without a reported duration use `cache_write`. Missing usage or rates leave +cost unknown. The table does not apply service-tier or long-context surcharges. +Omit `--pricing-file` to record reported charges alone. -The caller must send a session header that Switchyard recognizes on its model -requests, such as `x-switchyard-session-id: my-session`. Read that session with: +Send `x-switchyard-session-id: my-session` with model requests, then read: ```bash curl 'http://localhost:4000/v1/routing/session-stats?session_id=my-session' +jq -c '{model, cost}' routing.jsonl ``` -The response adds `cost`, `answer_cost`, and `routing_cost`, plus `cost` for each -model. Each total has `known_usd`, `total_usd`, `unknown_calls`, `estimated_calls`, -and `provider_reported_calls`. `total_usd` is null if any observed call has no -known cost. `known_usd` remains the subtotal of priced calls. Old records without -cost fields contribute unpriced calls. The routing log stores each priced call's -`cost` as an object, such as `{ "usd": 0.01, "source": "estimated" }`. Its `source` -is `"estimated"` or `"provider_reported"`; an unpriced call has `"cost": null`. -When the provider reports one-hour cache writes, the record also names that -part of `cache_creation_tokens` in `cache_creation_1h_tokens`. - -Totals cover recorded calls. Failed answer, classifier, and stream calls that -reach the server's observation code are unpriced. Internal HTTP retries, -cancelled requests, and streams abandoned before observation completes can -leave charges unrecorded. The report cannot establish the provider's bill. -Session totals include all recorded branches and compaction calls carrying the -same session ID; a CLI's active-branch total can cover a different set of calls. -The pricing table stays in memory for the routing log writer's lifetime and does -not grow per call. Its size follows the pricing file. The log grows with recorded -calls; this option adds no rotation or size limit. Separate replica logs produce -separate totals. If a log append fails, the server warns and continues serving; -missed records are not recovered automatically. The session endpoint scans the -log on each read. The displays fetch on agent -completion, rather than polling in the background. - -## Claude Code - -Use the Python example as a status-line command. Set the repository path and the -server address in Claude Code's settings: - -```json -{ - "statusLine": { - "type": "command", - "command": "python3 /path/to/Switchyard/examples/costs/claude_statusline.py --base-url http://localhost:4000" - } -} -``` - -The command reads `session_id` from Claude Code's status-line JSON input and -prints the Switchyard total, including routing cost. If you already have a -status-line command, call the example from that command to retain your display. -You can also run it from a shell with `--session-id my-session`. - -Claude Code's native cost display computes prices locally. Its managed -[`modelPricing`](https://code.claude.com/docs/en/settings-reference#modelpricing) -setting supports model and gateway-alias rate tables in version 2.1.242 or later. -Claude ignores that setting in ordinary user settings and `--settings`. -A fixed rate under a routing alias cannot describe differently priced targets or -include hidden judge calls. The Switchyard status line displays a separate total; -it does not change Claude Code's native cost or budget enforcement. - -## pi - -Load the example extension with your existing Switchyard provider configuration: - -```bash -pi -e /path/to/Switchyard/examples/costs/pi.ts --provider switchyard --model switchyard -``` - -The extension sends `x-switchyard-session-id` and shows a separate footer after -each agent turn. Set `SWITCHYARD_BASE_URL` when the server is elsewhere. Keep the -helper `display.ts` beside `pi.ts` when copying the example. - -pi's OpenAI adapters price the local model entry from its `cost` table. The -response's served model ID or `usage.cost` does not replace that table. A fixed -rate is useful for a fixed target; a router alias needs a separate display for -per-call target pricing. Keep the OpenAI request API described in the -[pi guide](pi.md#which-request-api). - -## Oh My Pi - -Load the example with an `anthropic-messages` Switchyard provider: - -```bash -omp -e /path/to/Switchyard/examples/costs/omp.ts --model switchyard/switchyard -``` - -Oh My Pi sends a session header on that API, so the footer can look up the -session report. The current custom-provider OpenAI adapters send no session -header; this example cannot show their session totals. Use the configuration in -[the Oh My Pi guide](oh_my_pi.md#which-request-api). Keep `display.ts` beside -`omp.ts`, and set `SWITCHYARD_BASE_URL` for another server address. - -Oh My Pi's custom `switchyard` provider uses configured model rates for its -native cost. Its supported OpenRouter integration can instead use `usage.cost`. -Returning the same field to a provider named `switchyard` does not activate that -behavior. The footer leaves native cost and budget enforcement unchanged. - -## Codex - -Codex's token display does not accept a dollar rate from Switchyard. Use the -session report or the Python command with an explicitly supplied session ID. -Per-request pricing still helps compare routed workloads even when the caller -has no native dollar display. +A priced log record contains `"cost": {"usd": 0.01, "source": "estimated"}` or +`"source": "provider_reported"`; an unpriced record contains `"cost": null`. +The report includes session and per-model `cost`, plus `answer_cost` and +`routing_cost`. Each total has `known_usd`, `total_usd`, `unknown_calls`, +`estimated_calls`, and `provider_reported_calls`. If any recorded call has unknown +cost, `total_usd` is null and `known_usd` remains the priced subtotal. Old records +without a cost count as unknown. + +Totals cover recorded calls, including failed calls observed by the server. +Classifier and judge calls cannot use header charges. Internal retries, cancelled +requests and abandoned streams may leave charges unrecorded. These totals cannot +establish the provider's bill or replace a client's native budget enforcement. + +When Switchyard translates a buffered response to Chat Completions, it writes +cache-write tokens as `prompt_tokens_details.cache_write_tokens`. Earlier +versions wrote `cache_creation_tokens` there. Streamed Chat output already used +`cache_write_tokens`. Both names are accepted on input, and same-format replay +returns the upstream JSON unchanged. + +The log writer holds one rate table in memory. The log grows per call and has no +size limit or rotation from this option. Failed appends produce a warning and are +not recovered; separate replica logs give separate totals. The session endpoint +scans the log on each read. diff --git a/docs/integrations/oh_my_pi.md b/docs/integrations/oh_my_pi.md index e0a225840..d98e6fc16 100644 --- a/docs/integrations/oh_my_pi.md +++ b/docs/integrations/oh_my_pi.md @@ -96,7 +96,7 @@ as a LiteLLM proxy. In that case, run Switchyard on another port or set `LITELLM_BASE_URL`. Set `cost` on the model entry for `omp`'s native estimate. One rate table under a router alias is an approximation when targets have different prices. For a separate total that includes routing calls, see -[Display Switchyard session costs](costs.md). +[Record Switchyard session costs](costs.md). ## Claude through an LLM gateway diff --git a/docs/integrations/pi.md b/docs/integrations/pi.md index 07ee70dc1..11ee5f8ae 100644 --- a/docs/integrations/pi.md +++ b/docs/integrations/pi.md @@ -100,7 +100,7 @@ clients keep the local id `switchyard` instead. Set `cost` on the model entry for pi's native estimate. One rate table under a router alias is an approximation when targets have different prices. For a separate total that uses each served model and includes routing calls, see -[Display Switchyard session costs](costs.md). +[Record Switchyard session costs](costs.md). [`benchmark/run-baseline.sh`](../../benchmark/README.md) runs Terminal-Bench tasks with pi through Switchyard when you pass `--agent pi`. @@ -177,7 +177,7 @@ each request's cost with: jq -c '{route_id, model, cost}' PATH ``` -[Display Switchyard session costs](costs.md) describes the pricing file and how the +[Record Switchyard session costs](costs.md) describes the pricing file and how the server totals a session. ### Thinking diff --git a/examples/costs/claude_statusline.py b/examples/costs/claude_statusline.py deleted file mode 100644 index 6adb3ee9a..000000000 --- a/examples/costs/claude_statusline.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""This command prints Switchyard's observed session cost in a status line or a shell.""" - -import argparse -import json -import os -import sys -import urllib.error -import urllib.parse -import urllib.request - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--base-url", default=os.environ.get("SWITCHYARD_BASE_URL", "http://localhost:4000")) - parser.add_argument("--session-id") - args = parser.parse_args() - try: - session_id = args.session_id - if not session_id: - payload = json.load(sys.stdin) - if not isinstance(payload, dict): - raise ValueError("invalid status-line input") - session_id = payload.get("session_id") - if not isinstance(session_id, str) or not session_id: - raise ValueError("missing session ID") - url = args.base_url.rstrip("/").removesuffix("/v1") + "/v1/routing/session-stats?" + urllib.parse.urlencode({"session_id": session_id}) - with urllib.request.urlopen(url, timeout=2) as response: - snapshot = json.load(response) - cost = snapshot["cost"] - if cost["total_usd"] is None: - text = f"${cost['known_usd']:.6f} known + {cost['unknown_calls']} unpriced calls" - else: - label = "estimated" if cost["estimated_calls"] else "reported" - text = f"${cost['total_usd']:.6f} {label}" - print(f"Switchyard {text} (routing ${snapshot['routing_cost']['known_usd']:.6f} known)") - except (OSError, ValueError, KeyError, TypeError, urllib.error.URLError): - print("Switchyard cost unavailable") - - -if __name__ == "__main__": - main() diff --git a/examples/costs/display.ts b/examples/costs/display.ts deleted file mode 100644 index 515fb0359..000000000 --- a/examples/costs/display.ts +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -interface Totals { - known_usd: number; - total_usd: number | null; - unknown_calls: number; - estimated_calls: number; -} - -interface CostContext { - hasUI: boolean; - model?: { provider: string }; - sessionManager: { getSessionId(): string }; - ui: { setStatus(key: string, text: string | undefined): void }; -} - -// Both agents can display a separate footer without rewriting their saved usage. -export async function updateCost(ctx: CostContext): Promise { - if (!ctx.hasUI) return; - if (ctx.model?.provider !== "switchyard") { - ctx.ui.setStatus("switchyard-cost", undefined); - return; - } - const sessionId = ctx.sessionManager.getSessionId(); - const base = (process.env.SWITCHYARD_BASE_URL ?? "http://localhost:4000").replace(/\/$/, "").replace(/\/v1$/, ""); - try { - const response = await fetch(`${base}/v1/routing/session-stats?${new URLSearchParams({ session_id: sessionId })}`, { signal: AbortSignal.timeout(2000) }); - if (!response.ok) throw new Error(`HTTP ${response.status}`); - const snapshot = await response.json() as { cost: Totals; routing_cost: Totals }; - const cost = snapshot.cost; - const text = cost.total_usd === null - ? `$${cost.known_usd.toFixed(6)} known + ${cost.unknown_calls} unpriced calls` - : `$${cost.total_usd.toFixed(6)} ${cost.estimated_calls ? "estimated" : "reported"}`; - if (ctx.sessionManager.getSessionId() === sessionId && ctx.model?.provider === "switchyard") { - ctx.ui.setStatus("switchyard-cost", `Switchyard ${text} (routing $${snapshot.routing_cost.known_usd.toFixed(6)} known)`); - } - } catch { - if (ctx.sessionManager.getSessionId() === sessionId && ctx.model?.provider === "switchyard") ctx.ui.setStatus("switchyard-cost", "Switchyard cost unavailable"); - } -} diff --git a/examples/costs/omp.ts b/examples/costs/omp.ts deleted file mode 100644 index 12455f765..000000000 --- a/examples/costs/omp.ts +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { ExtensionAPI } from "@oh-my-pi/pi-coding-agent"; -import { updateCost } from "./display.ts"; - -// OMP sends its session header on anthropic-messages requests. -export default function(omp: ExtensionAPI): void { - omp.on("session_start", (_event, ctx) => updateCost(ctx)); - omp.on("session_switch", (_event, ctx) => updateCost(ctx)); - omp.on("model_select", (_event, ctx) => updateCost(ctx)); - omp.on("agent_end", (_event, ctx) => updateCost(ctx)); -} diff --git a/examples/costs/pi.ts b/examples/costs/pi.ts deleted file mode 100644 index 54ed6b505..000000000 --- a/examples/costs/pi.ts +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { updateCost } from "./display.ts"; - -export default function(pi: ExtensionAPI): void { - pi.on("before_provider_headers", (event, ctx) => { - if (ctx.model?.provider === "switchyard") { - event.headers["x-switchyard-session-id"] = ctx.sessionManager.getSessionId(); - } - }); - pi.on("session_start", (_event, ctx) => updateCost(ctx)); - pi.on("session_switch", (_event, ctx) => updateCost(ctx)); - pi.on("model_select", (_event, ctx) => updateCost(ctx)); - pi.on("agent_end", (_event, ctx) => updateCost(ctx)); -} From bcd181d8aa4fd892519e567a4cdb3cb4f3bcb292 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 20 Oct 2026 12:47:37 -0700 Subject: [PATCH 3/8] chore(server): keep pricing private and simplify session totals Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-runner/src/lib.rs | 1 - crates/switchyard-server/src/lib.rs | 3 +- .../src/pricing.rs | 36 +++++++++---------- crates/switchyard-server/src/routing_log.rs | 24 ++++--------- 4 files changed, 26 insertions(+), 38 deletions(-) rename crates/{switchyard-runner => switchyard-server}/src/pricing.rs (91%) diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rs index 76ebf69ae..36ff10841 100644 --- a/crates/switchyard-runner/src/lib.rs +++ b/crates/switchyard-runner/src/lib.rs @@ -6,7 +6,6 @@ mod algorithm; mod config; mod failure; -pub mod pricing; mod provider_key_redactor; mod route; mod runner; diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index de3e89366..bdcf46757 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -8,6 +8,7 @@ pub mod config; mod metrics; mod observability; mod observed_translation; +mod pricing; mod redaction; mod response; mod routing_log; @@ -239,7 +240,7 @@ impl ServerState { /// This method loads rates in USD per million tokens for saved routing records. pub fn with_pricing_file(self, path: impl AsRef) -> ServerResult { - let prices = switchyard_runner::pricing::Pricing::load(path.as_ref())?; + let prices = pricing::Pricing::load(path.as_ref())?; let log = self .routing_log .as_ref() diff --git a/crates/switchyard-runner/src/pricing.rs b/crates/switchyard-server/src/pricing.rs similarity index 91% rename from crates/switchyard-runner/src/pricing.rs rename to crates/switchyard-server/src/pricing.rs index 152760192..f2f894e8b 100644 --- a/crates/switchyard-runner/src/pricing.rs +++ b/crates/switchyard-server/src/pricing.rs @@ -9,11 +9,11 @@ use std::path::Path; use serde::{Deserialize, Serialize}; use switchyard_protocol::Usage; -use crate::RunnerError; +use crate::{ServerError, ServerResult}; /// Pricing maps served model IDs to token rates in USD per million tokens. #[derive(Debug, Default)] -pub struct Pricing(HashMap); +pub(crate) struct Pricing(HashMap); impl<'de> Deserialize<'de> for Pricing { fn deserialize>(deserializer: D) -> Result { @@ -55,32 +55,32 @@ struct Rates { #[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub enum CostSource { +pub(crate) enum CostSource { ProviderReported, Estimated, } #[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)] -pub struct RecordedCost { - pub usd: f64, - pub source: CostSource, +pub(crate) struct RecordedCost { + pub(crate) usd: f64, + pub(crate) source: CostSource, } impl Pricing { - pub fn load(path: &Path) -> Result { + pub(crate) fn load(path: &Path) -> ServerResult { let bytes = std::fs::read(path).map_err(|error| { - RunnerError::configuration(format!( + ServerError::new(format!( "failed to read pricing file {}: {error}", path.display() )) })?; Self::parse(&bytes).map_err(|error| { - RunnerError::configuration(format!("invalid pricing file {}: {error}", path.display())) + ServerError::new(format!("invalid pricing file {}: {error}", path.display())) }) } /// This function reads a pricing file's JSON and checks every model ID and rate. - pub fn parse(json: &[u8]) -> Result { + fn parse(json: &[u8]) -> Result { let pricing: Self = serde_json::from_slice(json).map_err(|error| error.to_string())?; for (model, rate) in &pricing.0 { if model.trim().is_empty() @@ -105,7 +105,7 @@ impl Pricing { } /// Provider charges take precedence because token rates may omit provider surcharges. - pub fn cost(&self, model: &str, usage: &Usage) -> Option { + pub(crate) fn cost(&self, model: &str, usage: &Usage) -> Option { if let Some(usd) = usage .provider_cost .filter(|value| value.is_finite() && *value >= 0.0) @@ -146,12 +146,12 @@ impl Pricing { /// The known subtotal remains visible when any call has unknown cost. #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] -pub struct CostTotals { - pub known_usd: f64, - pub unknown_calls: u64, - pub estimated_calls: u64, - pub provider_reported_calls: u64, - pub total_usd: Option, +pub(crate) struct CostTotals { + pub(crate) known_usd: f64, + pub(crate) unknown_calls: u64, + pub(crate) estimated_calls: u64, + pub(crate) provider_reported_calls: u64, + pub(crate) total_usd: Option, } impl Default for CostTotals { @@ -167,7 +167,7 @@ impl Default for CostTotals { } impl CostTotals { - pub fn add(&mut self, cost: Option) { + pub(crate) fn add(&mut self, cost: Option) { match cost.filter(|cost| { cost.usd.is_finite() && cost.usd >= 0.0 && (self.known_usd + cost.usd).is_finite() }) { diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs index 250046546..b0d702a6d 100644 --- a/crates/switchyard-server/src/routing_log.rs +++ b/crates/switchyard-server/src/routing_log.rs @@ -14,9 +14,9 @@ use humantime::format_rfc3339_millis; use serde::{Deserialize, Serialize}; use switchyard_protocol::{Metadata, ModelId, Usage}; +use crate::pricing::{CostTotals, Pricing, RecordedCost}; use crate::usage_metrics::token_usage; use crate::{ServerError, ServerResult}; -use switchyard_runner::pricing::{CostTotals, Pricing, RecordedCost}; const LEGACY_SESSION_ID_HEADER: &str = "proxy_x_session_id"; const ORIGIN_HEADER: &str = "x-switchyard-origin"; @@ -97,7 +97,10 @@ pub(crate) fn snapshot( ) -> std::io::Result> { let mut reader = BufReader::with_capacity(64 * 1024, fs::File::open(path)?); let mut line = Vec::new(); - let mut snapshot = SessionStatsSnapshot::new(session_id); + let mut snapshot = SessionStatsSnapshot { + session_id: session_id.to_string(), + ..Default::default() + }; loop { line.clear(); if reader.read_until(b'\n', &mut line)? == 0 { @@ -199,7 +202,7 @@ fn is_zero(value: &u64) -> bool { } /// Session totals returned by the routing stats endpoint. -#[derive(Serialize)] +#[derive(Default, Serialize)] pub(crate) struct SessionStatsSnapshot { session_id: String, cost: CostTotals, @@ -224,21 +227,6 @@ struct SessionModelStats { } impl SessionStatsSnapshot { - fn new(session_id: &str) -> Self { - Self { - session_id: session_id.to_string(), - cost: CostTotals::default(), - answer_cost: CostTotals::default(), - routing_cost: CostTotals::default(), - total_calls: 0, - total_prompt_tokens: 0, - total_cached_tokens: 0, - total_cache_creation_tokens: 0, - total_completion_tokens: 0, - models: BTreeMap::new(), - } - } - fn add_record(&mut self, record: &RoutingRecord<'_>, session_id: &str) { if record.session_id.as_deref() != Some(session_id) { return; From c0390abe620ea425654aca1d359c571cac4279f2 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 20 Oct 2026 13:08:28 -0700 Subject: [PATCH 4/8] chore(server): move Anthropic cache pricing into its own module Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-server/src/pricing.rs | 17 +++--------- .../src/pricing/anthropic.rs | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 crates/switchyard-server/src/pricing/anthropic.rs diff --git a/crates/switchyard-server/src/pricing.rs b/crates/switchyard-server/src/pricing.rs index f2f894e8b..2c7797032 100644 --- a/crates/switchyard-server/src/pricing.rs +++ b/crates/switchyard-server/src/pricing.rs @@ -3,6 +3,8 @@ //! This module prices observed calls and keeps missing usage or rates unknown. +mod anthropic; + use std::collections::HashMap; use std::path::Path; @@ -119,23 +121,10 @@ impl Pricing { let input = usage.input_tokens?; let output = usage.output_tokens?; let read = usage.cached_input_tokens().unwrap_or(0); - let write = usage.cache_creation_input_tokens().unwrap_or(0); - let long_write = usage - .cache - .as_ref() - .and_then(|cache| cache.cache_creation_1h_input_tokens) - .unwrap_or(0); - let short_write = write.checked_sub(long_write)?; - let long_rate = if long_write > 0 { - rate.cache_write_1h? - } else { - 0.0 - }; let usd = (input as f64 * rate.input + output as f64 * rate.output + read as f64 * rate.cache_read - + short_write as f64 * rate.cache_write - + long_write as f64 * long_rate) + + anthropic::cache_write_cost(usage, rate.cache_write, rate.cache_write_1h)?) / 1_000_000.0; Some(RecordedCost { usd, diff --git a/crates/switchyard-server/src/pricing/anthropic.rs b/crates/switchyard-server/src/pricing/anthropic.rs new file mode 100644 index 000000000..99b760142 --- /dev/null +++ b/crates/switchyard-server/src/pricing/anthropic.rs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Anthropic prices cache writes at separate rates for five-minute and one-hour lifetimes. + +use switchyard_protocol::Usage; + +/// The caller divides this weighted token sum by one million to obtain USD. +pub(super) fn cache_write_cost( + usage: &Usage, + short_rate: f64, + long_rate: Option, +) -> Option { + let written = usage.cache_creation_input_tokens().unwrap_or(0); + let one_hour = usage + .cache + .as_ref() + .and_then(|cache| cache.cache_creation_1h_input_tokens) + .unwrap_or(0); + let five_minute = written.checked_sub(one_hour)?; + let long_cost = if one_hour > 0 { + one_hour as f64 * long_rate? + } else { + 0.0 + }; + Some(five_minute as f64 * short_rate + long_cost) +} From fd4fe9331be1e1a737154ff205d9f1250d49e47b Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 20 Oct 2026 13:17:57 -0700 Subject: [PATCH 5/8] chore(costs): remove added tests and the standalone guide Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-server/README.md | 15 +- crates/switchyard-server/src/lib.rs | 18 +- crates/switchyard-server/src/pricing.rs | 110 ---------- crates/switchyard-server/src/routing_log.rs | 35 +--- crates/switchyard-server/src/usage_metrics.rs | 41 ---- crates/switchyard-server/tests/cli.rs | 58 ------ crates/switchyard-server/tests/server.rs | 189 ++---------------- .../tests/response_translation.rs | 62 +----- .../tests/stream_translation.rs | 44 ---- docs/cli_reference.md | 2 +- docs/integrations/costs.md | 66 ------ docs/integrations/oh_my_pi.md | 6 +- docs/integrations/pi.md | 8 +- 13 files changed, 32 insertions(+), 622 deletions(-) delete mode 100644 docs/integrations/costs.md diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index d08b224b3..c069a6b14 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -124,10 +124,17 @@ served model. The legacy `proxy_x_session_id` remains a fallback when no normali present. The endpoint returns `404` when the session has no records and is not registered when routing logging is disabled. -Add `--pricing-file prices.json` to record each call's `cost` in USD. The server records the -charge that the upstream reports and otherwise estimates it from the file's rates for the served -model. The session stats then include cost totals. [Record Switchyard session -costs](../../docs/integrations/costs.md) describes the file format. +The routing log saves each call's reported charge in USD. Add `--pricing-file prices.json` +to estimate costs that the upstream does not report. The JSON maps served model IDs to rates +in USD per million tokens: + +```json +{"upstream/model":{"input":1,"output":5,"cache_read":0.1,"cache_write":2}} +``` + +Session stats include cost totals. Missing usage or rates leave cost unknown; `known_usd` +keeps the priced subtotal and `total_usd` is null when any cost is unknown. Rates load once +at startup, and saved costs do not change when prices change. Clients can send `x-switchyard-origin: codex-cli` (or another client label) to include an `origin` field in each routing record. Missing, empty, or non-text header values produce diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index bdcf46757..9110bf596 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -1826,9 +1826,8 @@ targets = ["only"] } /// The observer logs judge calls with their model IDs, classifier tier, and - /// recorded session and turn IDs, even when a call reports no usage. The served - /// response logs routed calls with their final usage, so logging them here would - /// count each routed call twice. + /// recorded session and turn IDs. The served response logs routed calls with + /// their final usage, so logging them here would count each routed call twice. #[test] fn stats_observer_logs_judge_calls_to_the_routing_log() -> Result<(), Box> { @@ -1860,12 +1859,6 @@ targets = ["only"] }; observer(call("judge-model", false)); observer(call("routed-model", true)); - observer(RunObservation::LlmCall(ModelCallObservation { - selected_model: ModelId::from("streamed-judge"), - is_success: true, - duration: Duration::from_millis(3), - usage: None, - })); let snapshot = log .snapshot_session("session-1") @@ -1876,12 +1869,7 @@ targets = ["only"] assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100); assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7); assert!(snapshot["models"].get("routed-model").is_none()); - assert_eq!(snapshot["models"]["streamed-judge"]["calls"], 1); - assert_eq!(snapshot["routing_cost"]["unknown_calls"], 2); - assert!(snapshot["routing_cost"]["total_usd"].is_null()); - let text = std::fs::read_to_string(&log.path)?; - let record: serde_json::Value = - serde_json::from_str(text.lines().next().ok_or("empty log")?)?; + let record: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&log.path)?)?; assert_eq!(record["session_id"], "session-1"); assert_eq!(record["turn_id"], "turn-1"); Ok(()) diff --git a/crates/switchyard-server/src/pricing.rs b/crates/switchyard-server/src/pricing.rs index 2c7797032..4645e05b0 100644 --- a/crates/switchyard-server/src/pricing.rs +++ b/crates/switchyard-server/src/pricing.rs @@ -172,113 +172,3 @@ impl CostTotals { self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rejects_invalid_rate_tables() { - let valid = r#"{"input":1,"output":5,"cache_read":0.1,"cache_write":2}"#; - for rates in [ - r#"{"input":-1,"output":5,"cache_read":0.1,"cache_write":2}"#, - r#"{"input":1,"output":5,"cache_read":0.1}"#, - r#"{"input":1,"output":5,"cache_read":0.1,"cache_write":2,"cache_wirte":2}"#, - r#"{"input":1e999,"output":5,"cache_read":0.1,"cache_write":2}"#, - ] { - assert!( - Pricing::parse(format!("{{\"model\":{rates}}}").as_bytes()).is_err(), - "{rates}" - ); - } - assert!( - Pricing::parse(format!("{{\"model\":{valid},\"model\":{valid}}}").as_bytes()).is_err() - ); - } - - #[test] - fn estimates_disjoint_buckets_and_prefers_reported_charges() - -> Result<(), Box> { - let prices = Pricing::parse( - br#"{"model":{"input":1,"output":5,"cache_read":0.1,"cache_write":2,"cache_write_1h":3},"free":{"input":0,"output":0,"cache_read":0,"cache_write":0}}"#, - )?; - let mut usage = Usage { - input_tokens: Some(50), - output_tokens: Some(10), - reasoning_tokens: Some(4), - cache: Usage::cache_details(Some(20), Some(30)), - ..Usage::default() - }; - usage - .cache - .as_mut() - .ok_or("missing cache detail")? - .cache_creation_1h_input_tokens = Some(10); - assert!( - (prices.cost("model", &usage).ok_or("missing estimate")?.usd - 0.000172).abs() < 1e-12 - ); - assert_eq!( - prices - .cost( - "free", - &Usage { - cache: None, - ..usage.clone() - } - ) - .ok_or("missing free price")? - .usd, - 0.0 - ); - assert!(prices.cost("unknown", &usage).is_none()); - assert!(prices.cost("model", &Usage::default()).is_none()); - assert!(prices.cost("free", &usage).is_none()); - let mut invalid_duration = usage.clone(); - invalid_duration - .cache - .as_mut() - .ok_or("missing cache detail")? - .cache_creation_1h_input_tokens = Some(31); - assert!(prices.cost("model", &invalid_duration).is_none()); - usage.provider_cost = Some(0.0); - assert_eq!( - prices - .cost("unknown", &usage) - .ok_or("missing reported cost")? - .usd, - 0.0 - ); - usage.provider_cost = Some(0.25); - assert_eq!( - prices - .cost("model", &usage) - .ok_or("missing reported cost")? - .usd, - 0.25 - ); - Ok(()) - } - - #[test] - fn partial_and_overflowing_totals_remain_unknown() { - let mut totals = CostTotals::default(); - totals.add(Some(RecordedCost { - usd: 0.25, - source: CostSource::Estimated, - })); - totals.add(None); - assert_eq!(totals.known_usd, 0.25); - assert_eq!(totals.unknown_calls, 1); - assert_eq!(totals.total_usd, None); - totals.add(Some(RecordedCost { - usd: f64::MAX, - source: CostSource::ProviderReported, - })); - totals.add(Some(RecordedCost { - usd: f64::MAX, - source: CostSource::ProviderReported, - })); - assert!(totals.known_usd.is_finite()); - assert_eq!(totals.unknown_calls, 2); - } -} diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs index b0d702a6d..a91586513 100644 --- a/crates/switchyard-server/src/routing_log.rs +++ b/crates/switchyard-server/src/routing_log.rs @@ -327,7 +327,7 @@ mod tests { fs::write( &path, concat!( - r#"{"session_id":"a","model":"m1","fallback_reason":"unavailable","prompt_tokens":10,"completion_tokens":2,"cost":{"usd":0.25,"source":"estimated"}}"#, + r#"{"session_id":"a","model":"m1","fallback_reason":"unavailable","prompt_tokens":10,"completion_tokens":2}"#, "\n", r#"{"session_id":"b","model":"m1","prompt_tokens":99,"completion_tokens":99}"#, "\n", @@ -344,39 +344,6 @@ mod tests { assert_eq!(stats.total_completion_tokens, 2); assert_eq!(stats.models["m1"].calls, 1); assert_eq!(stats.models["unknown"].prompt_tokens, 5); - assert_eq!(stats.cost.known_usd, 0.25); - assert_eq!(stats.cost.estimated_calls, 1); - assert_eq!(stats.cost.unknown_calls, 1); - assert_eq!(stats.cost.total_usd, None); assert!(snapshot(&path, "missing").expect("read log").is_none()); } - - /// One-hour cache writes keep their own count, so a reader can price them at their rate. - #[test] - fn records_name_one_hour_cache_writes_only_when_reported() - -> Result<(), Box> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("routing.jsonl"); - let mut log = RoutingLog::new(&path)?; - let mut usage = Usage { - input_tokens: Some(10), - output_tokens: Some(1), - cache: Usage::cache_details(None, Some(30)), - ..Usage::default() - }; - let context = || RoutingLogContext::from_metadata(&Metadata::default()); - log.append(context(), "m", None, &usage)?; - if let Some(cache) = usage.cache.as_mut() { - cache.cache_creation_1h_input_tokens = Some(20); - } - log.append(context(), "m", None, &usage)?; - let records = fs::read_to_string(&path)? - .lines() - .map(serde_json::from_str) - .collect::, _>>()?; - assert_eq!(records[0]["cache_creation_tokens"], 30); - assert!(records[0].get("cache_creation_1h_tokens").is_none()); - assert_eq!(records[1]["cache_creation_1h_tokens"], 20); - Ok(()) - } } diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs index ad5a6a77f..96a16c9f7 100644 --- a/crates/switchyard-server/src/usage_metrics.rs +++ b/crates/switchyard-server/src/usage_metrics.rs @@ -214,47 +214,6 @@ mod tests { use super::*; - /// A stream error must leave one unpriced record even when no usage was reported. - #[tokio::test] - async fn stream_failure_records_one_unknown_call_and_terminates() - -> Result<(), Box> { - let dir = tempfile::tempdir()?; - let log = SharedRoutingLog::new(dir.path().join("routing.jsonl"))?; - let context = RoutingLogContext::from_metadata(&Metadata { - session_id: Some("failed-stream".to_string()), - ..Metadata::default() - }); - let error = LlmResponseStreamEvent::new(vec![LlmResponseChunk::StreamError { - message: "failed".to_string(), - }]); - let source = stream::iter([Ok(error.clone()), Ok(error)]); - let observed = observe( - Response { - llm_response: LlmResponse::Stream(Box::pin(source)), - metadata: None, - upstream_headers: http::HeaderMap::new(), - }, - "model/worker", - Instant::now(), - StatsAccumulator::default(), - 0.0, - Some((log.clone(), context)), - ); - let LlmResponse::Stream(mut observed) = observed.llm_response else { - return Err("expected stream".into()); - }; - assert!(observed.next().await.is_some()); - assert!(observed.next().await.is_none()); - let snapshot = log - .snapshot_session("failed-stream")? - .ok_or("missing session")?; - let snapshot = serde_json::to_value(snapshot)?; - assert_eq!(snapshot["total_calls"], 1); - assert_eq!(snapshot["cost"]["unknown_calls"], 1); - assert!(snapshot["cost"]["total_usd"].is_null()); - Ok(()) - } - /// An OpenAI Responses client may stop polling immediately after receiving the /// terminal `response.completed` event. Switchyard must record usage and routing /// data before returning that event because the stream wrapper will not resume diff --git a/crates/switchyard-server/tests/cli.rs b/crates/switchyard-server/tests/cli.rs index c60a02ec3..a0e535544 100644 --- a/crates/switchyard-server/tests/cli.rs +++ b/crates/switchyard-server/tests/cli.rs @@ -127,61 +127,3 @@ fn dry_run_rejects_unsendable_configured_headers() -> TestResult { } Ok(()) } - -#[test] -fn pricing_requires_a_log_and_validates_before_serving() -> TestResult { - let directory = tempfile::tempdir()?; - let config = directory.path().join("routes.toml"); - let prices = directory.path().join("prices.json"); - let log = directory.path().join("routing.jsonl"); - fs::write( - &config, - r#" -schema_version = 1 -[llm_clients.upstream] -format = "openai_chat" -base_url = "http://127.0.0.1:1/v1" -[targets.answer] -id = "answer" -llm_client = "upstream" -[routes.direct] -id = "direct" -type = "passthrough" -target = "answer" -"#, - )?; - let output = Command::new(env!("CARGO_BIN_EXE_switchyard-server")) - .arg("--config") - .arg(&config) - .arg("--pricing-file") - .arg(&prices) - .arg("--dry-run") - .output()?; - assert!(!output.status.success()); - assert!(String::from_utf8(output.stderr)?.contains("--routing-log-file")); - assert!(!log.exists()); - for (rates, succeeds) in [ - ( - r#"{"answer":{"input":0,"output":0,"cache_read":0,"cache_write":0}}"#, - true, - ), - (r#"{"answer":{"input":0,"output":0,"cache_read":0}}"#, false), - ] { - fs::write(&prices, rates)?; - let output = Command::new(env!("CARGO_BIN_EXE_switchyard-server")) - .arg("--config") - .arg(&config) - .arg("--pricing-file") - .arg(&prices) - .arg("--routing-log-file") - .arg(&log) - .arg("--dry-run") - .output()?; - assert_eq!(output.status.success(), succeeds); - if !succeeds { - assert!(String::from_utf8(output.stderr)?.contains("invalid pricing file")); - } - assert!(fs::read(&log)?.is_empty()); - } - Ok(()) -} diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index dd4179429..d0fad52ab 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -312,17 +312,11 @@ async fn upstream_chat( ); return Sse::new(stream).into_response(); } - let mut usage = json!({"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17, - "prompt_tokens_details": {"cached_tokens": 7, "cache_creation_tokens": 2}, - "completion_tokens_details": {"reasoning_tokens": 3}}); - if prompt == "reported-cost" { - usage["cost"] = json!(0.00004071); - } let events = [ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(), json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "hello"}}]}).to_string(), json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-partial"}}], "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6, "prompt_tokens_details": {"cached_tokens": 2, "cache_creation_tokens": 1}}}).to_string(), - json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-final"}}], "usage": usage}).to_string(), + json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-final"}}], "usage": {"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17, "prompt_tokens_details": {"cached_tokens": 7, "cache_creation_tokens": 2}, "completion_tokens_details": {"reasoning_tokens": 3}}}).to_string(), json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}).to_string(), "[DONE]".to_string(), ]; @@ -331,15 +325,7 @@ async fn upstream_chat( .into_iter() .map(|data| Ok::(Event::default().data(data))), ); - let mut response = Sse::new(stream).into_response(); - if prompt == "reported-cost" { - // LiteLLM streams report the charge in usage, with a zeroed breakdown header. - response.headers_mut().insert( - "x-litellm-response-cost-original", - HeaderValue::from_static("0.0"), - ); - } - return response; + return Sse::new(stream).into_response(); } if model == "model/advisor" { @@ -470,7 +456,7 @@ async fn upstream_chat( } else { "ok".to_string() }; - let mut response = Json(json!({ + Json(json!({ "id": "chatcmpl-test", "object": "chat.completion", "model": model, @@ -486,15 +472,7 @@ async fn upstream_chat( "prompt_tokens_details": {"cached_tokens": 7} } })) - .into_response(); - if prompt == "reported-cost" { - // Buffered LiteLLM answers report their charge only in this header. - response.headers_mut().insert( - "x-litellm-response-cost", - HeaderValue::from_static("6.141e-05"), - ); - } - response + .into_response() } async fn upstream_messages_requires_forwarded_oauth( @@ -3150,11 +3128,6 @@ recent_turn_window = 1 async fn accepted_escalation_response_is_logged_once_as_the_final_answer() -> TestResult { let upstream = MockUpstream::start().await?; let temp_dir = tempfile::tempdir()?; - let prices = temp_dir.path().join("prices.json"); - std::fs::write( - &prices, - r#"{"model/strong":{"input":10,"output":20,"cache_read":0.2,"cache_write":3},"model/weak":{"input":1,"output":5,"cache_read":0.1,"cache_write":2}}"#, - )?; let state = load_test_config(&format!( r#" schema_version = 1 @@ -3188,8 +3161,7 @@ escalation = {{ confirmations = 1 }} "#, base_url = upstream.base_url ))? - .with_routing_log(temp_dir.path().join("routing.jsonl"))? - .with_pricing_file(prices)?; + .with_routing_log(temp_dir.path().join("routing.jsonl"))?; let app = build_switchyard_router(state); let response = send_with_headers( @@ -3220,16 +3192,6 @@ escalation = {{ confirmations = 1 }} assert_eq!(stats.status, StatusCode::OK); let stats = stats.json()?; assert_eq!(stats["total_calls"], 2); - let answer_cost = stats["answer_cost"]["total_usd"] - .as_f64() - .ok_or("missing answer cost")?; - let routing_cost = stats["routing_cost"]["total_usd"] - .as_f64() - .ok_or("missing routing cost")?; - assert!((answer_cost - 0.0000137).abs() < 1e-12); - assert!((routing_cost - 0.0000714).abs() < 1e-12); - assert_eq!(stats["cost"]["estimated_calls"], 2); - assert_eq!(stats["total_prompt_tokens"], 20); assert_eq!(stats["total_completion_tokens"], 4); assert_eq!(stats["models"]["model/weak"]["calls"], 1); @@ -4737,15 +4699,9 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted let upstream = MockUpstream::start().await?; let temp_dir = tempfile::tempdir()?; let log_path = temp_dir.path().join("routing.jsonl"); - let prices = temp_dir.path().join("prices.json"); - std::fs::write( - &prices, - r#"{"model/strong":{"input":10,"output":20,"cache_read":0.2,"cache_write":3}}"#, - )?; // The fixed seed selects `first` for these requests while keeping `second` enabled. - let state = weighted_random_state(&upstream.base_url, [1000, 1])? - .with_routing_log(&log_path)? - .with_pricing_file(&prices)?; + let state = + weighted_random_state(&upstream.base_url, [1000, 1])?.with_routing_log(&log_path)?; let app = build_switchyard_router(state); let cases = [ ( @@ -4771,14 +4727,7 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted for (path, body) in cases { let previous_call_count = upstream.calls.lock().await.len(); - let response = send_with_headers( - &app, - "POST", - path, - Some(body), - &[("x-switchyard-session-id", "cost-fallback")], - ) - .await?; + let response = send(&app, "POST", path, Some(body)).await?; assert_eq!(response.status, StatusCode::OK); assert_eq!( response @@ -4824,36 +4773,12 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted assert_eq!( records .iter() - .filter(|record| record["model"] == "model/strong") - .count(), - 3 - ); - assert_eq!( - records - .iter() - .filter(|record| record["model"] == "model/weak" && record["cost"].is_null()) + .filter(|record| { + record["model"] == "model/strong" && record.get("fallback_reason").is_none() + }) .count(), 3 ); - let session = send( - &app, - "GET", - "/v1/routing/session-stats?session_id=cost-fallback", - None, - ) - .await? - .json()?; - assert_eq!(session["cost"]["unknown_calls"], 3); - assert_eq!(session["cost"]["estimated_calls"], 3); - assert!(session["cost"]["total_usd"].is_null()); - assert!( - (session["cost"]["known_usd"] - .as_f64() - .ok_or("missing subtotal")? - - 0.0002142) - .abs() - < 1e-12 - ); let previous_call_count = upstream.calls.lock().await.len(); let response = send( @@ -5421,7 +5346,7 @@ async fn advisor_route_streaming_approval_replays_provider_events() -> TestResul assert_eq!(events[1]["choices"][0]["delta"]["content"], "hello"); assert_eq!(events[2]["choices"][0]["delta"]["content"], "-partial"); assert_eq!(events[3]["choices"][0]["delta"]["content"], "-final"); - // The response keeps the provider's usage fields unchanged. + // Provider-specific usage detail rides through untouched. assert_eq!( events[3]["usage"]["prompt_tokens_details"]["cache_creation_tokens"], 2 @@ -5910,93 +5835,3 @@ seed = 17 } Ok(()) } - -#[tokio::test] -async fn session_cost_prefers_reported_charges_and_prices_served_models() -> TestResult { - let upstream = MockUpstream::start().await?; - let directory = tempfile::tempdir()?; - let prices = directory.path().join("prices.json"); - std::fs::write( - &prices, - r#"{"model/a":{"input":1,"output":5,"cache_read":0.1,"cache_write":2},"model/b":{"input":10,"output":20,"cache_read":0.2,"cache_write":3}}"#, - )?; - let state = random_state( - &upstream.base_url, - &[("route-a", &["model/a"]), ("route-b", &["model/b"])], - )? - .with_routing_log(directory.path().join("routing.jsonl"))? - .with_pricing_file(&prices)?; - let app = build_switchyard_router(state); - for (endpoint, stream, model, prompt, expected) in [ - ("/v1/chat/completions", false, "route-a", "hello", 0.0000137), - ( - "/v1/messages", - false, - "route-a", - "reported-cost", - 0.00006141, - ), - ("/v1/responses", false, "route-b", "hello", 0.0000714), - ( - "/v1/chat/completions", - true, - "route-a", - "reported-cost", - 0.00004071, - ), - ("/v1/messages", true, "route-a", "hello", 0.0000327), - ("/v1/responses", true, "route-b", "hello", 0.0001374), - ] { - let session = format!("cost-{model}-{endpoint}-{stream}"); - let mut body = - json!({"model":model,"stream":stream,"messages":[{"role":"user","content":prompt}]}); - if endpoint == "/v1/responses" { - body = json!({"model":model,"stream":stream,"input":prompt}); - } - if endpoint == "/v1/messages" { - body["max_tokens"] = json!(100); - } - let response = send_with_headers( - &app, - "POST", - endpoint, - Some(body), - &[("x-switchyard-session-id", &session)], - ) - .await?; - assert_eq!(response.status, StatusCode::OK, "{}", response.text()?); - let stats = send( - &app, - "GET", - &format!("/v1/routing/session-stats?session_id={session}"), - None, - ) - .await? - .json()?; - assert_eq!(stats["total_calls"], 1); - let source = if prompt == "reported-cost" { - "provider_reported_calls" - } else { - "estimated_calls" - }; - assert_eq!(stats["cost"][source], 1); - assert_eq!( - stats["models"][if model == "route-a" { - "model/a" - } else { - "model/b" - }]["cost"][source], - 1 - ); - let actual = stats["cost"]["total_usd"] - .as_f64() - .ok_or("missing session cost")?; - assert!( - (actual - expected).abs() < 1e-12, - "{endpoint}: {actual} != {expected}" - ); - assert_eq!(stats["routing_cost"]["total_usd"], 0.0); - } - assert_eq!(upstream.calls.lock().await.len(), 6); - Ok(()) -} diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs index 3aed128b5..7d3ce5df5 100644 --- a/crates/switchyard-translation/tests/response_translation.rs +++ b/crates/switchyard-translation/tests/response_translation.rs @@ -6,7 +6,7 @@ pub mod common; use pretty_assertions::assert_eq; -use serde_json::{Value, json}; +use serde_json::json; use switchyard_translation::{ ContentBlock, PreservationPolicy, StopReason, TranslationEngine, TranslationPolicy, WireFormat, }; @@ -1335,63 +1335,3 @@ fn responses_custom_tool_call_output_round_trips_with_request_extensions() -> Te assert_eq!(call["function"]["arguments"], "{\"input\":\"ls -la\"}"); Ok(()) } - -#[test] -fn translated_usage_keeps_charges_and_cache_duration() -> TestResult { - let engine = TranslationEngine::default(); - let body = json!({"id":"msg_cost", "type":"message", "role":"assistant", "model":"claude", - "content":[{"type":"text","text":"ok"}], "stop_reason":"end_turn", - "usage":{"input_tokens":50,"output_tokens":10,"cache_read_input_tokens":20, - "cache_creation_input_tokens":30,"cache_creation":{"ephemeral_1h_input_tokens":10},"cost":0.25}}); - for target in [WireFormat::OpenAiChat, WireFormat::OpenAiResponses] { - let translated = engine - .translate_response( - WireFormat::AnthropicMessages, - target, - &body, - &TranslationPolicy::default(), - )? - .body; - assert_eq!(translated["usage"]["cost"], 0.25); - assert_eq!( - translated["usage"]["cache_creation"]["ephemeral_1h_input_tokens"], - 10 - ); - let detail = if target == WireFormat::OpenAiChat { - "prompt_tokens_details" - } else { - "input_tokens_details" - }; - assert_eq!(translated["usage"][detail]["cache_write_tokens"], 30); - let restored = engine - .translate_response( - target, - WireFormat::AnthropicMessages, - &translated, - &TranslationPolicy::default(), - )? - .body; - assert_eq!(restored["usage"]["cost"], 0.25); - } - for cost in [json!(0), json!(-1), json!("0.25"), Value::Null] { - let mut altered = body.clone(); - altered["usage"]["cost"] = cost.clone(); - let translated = engine - .translate_response( - WireFormat::AnthropicMessages, - WireFormat::OpenAiChat, - &altered, - &TranslationPolicy::default(), - )? - .body; - assert_eq!( - translated["usage"]["cost"], - if cost == json!(0) { - json!(0.0) - } else { - Value::Null - } - ); - } - Ok(()) -} diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs index 635ba19e7..2e4f3e53c 100644 --- a/crates/switchyard-translation/tests/stream_translation.rs +++ b/crates/switchyard-translation/tests/stream_translation.rs @@ -3582,47 +3582,3 @@ fn responses_custom_tool_calls_decode_from_the_finished_item() -> TestResult { } Ok(()) } - -/// The final output-only delta must not erase the charge or cache duration from message_start. -#[test] -fn streamed_usage_keeps_charge_and_duration_after_output_only_delta() -> TestResult { - let engine = TranslationEngine::default(); - for target in [WireFormat::OpenAiChat, WireFormat::OpenAiResponses] { - let mut state = StreamTranslationState::new(WireFormat::AnthropicMessages, target); - let mut events = Vec::new(); - for event in [ - json!({"type":"message_start","message":{"id":"msg_cost","type":"message","role":"assistant","model":"claude","content":[],"usage":{"input_tokens":50,"output_tokens":0,"cache_read_input_tokens":20,"cache_creation_input_tokens":30,"cache_creation":{"ephemeral_1h_input_tokens":10},"cost":0.25}}}), - json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":10}}), - json!({"type":"message_stop"}), - ] { - events.extend(engine.translate_event( - &mut state, - WireFormat::AnthropicMessages, - target, - &event, - )?); - } - events.extend(engine.finish_stream(&mut state, target)?); - let usage = events - .iter() - .rev() - .find_map(|event| { - if target == WireFormat::OpenAiResponses { - event.get("response")?.get("usage") - } else { - event.get("usage") - } - }) - .ok_or("missing final usage")?; - assert_eq!(usage["cost"], 0.25); - assert_eq!(usage["cache_creation"]["ephemeral_1h_input_tokens"], 10); - let detail = if target == WireFormat::OpenAiChat { - "prompt_tokens_details" - } else { - "input_tokens_details" - }; - assert_eq!(usage[detail]["cache_write_tokens"], 30); - assert_eq!(state.usage.output_tokens, Some(10)); - } - Ok(()) -} diff --git a/docs/cli_reference.md b/docs/cli_reference.md index 84fddb034..c9374fbb9 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -20,7 +20,7 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | -| `--pricing-file PATH` | None | Price each routing record from this JSON file of USD rates per million tokens; requires `--routing-log-file`. See [Record Switchyard session costs](integrations/costs.md). | +| `--pricing-file PATH` | None | Estimate unreported call costs from this JSON file of USD rates per million tokens; requires `--routing-log-file`. | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | diff --git a/docs/integrations/costs.md b/docs/integrations/costs.md deleted file mode 100644 index 3809cd872..000000000 --- a/docs/integrations/costs.md +++ /dev/null @@ -1,66 +0,0 @@ -# Record Switchyard session costs - -Switchyard records each observed call's cost in the routing log, including calls -made by classifiers and judges. To estimate costs that the upstream does not -report, start the server with a pricing file: - -```bash -switchyard-server --config routes.toml --routing-log-file routing.jsonl --pricing-file prices.json -``` - -The JSON file maps served model IDs to USD per million tokens. These rates are -illustrative; supply your provider's rates: - -```json -{ - "upstream/model": { - "input": 1, "output": 5, "cache_read": 0.1, "cache_write": 2, - "cache_write_1h": 3 - } -} -``` - -The four base rates are required; `cache_write_1h` is optional. The server rejects -unknown fields, duplicate, empty or padded model IDs, and rates outside 0 to 10000. -Zero means free usage. Use distinct served IDs when gateways charge different -rates for the same model. Rates load once at startup; saved costs do not change -when the pricing file changes. - -A valid charge in `usage.cost` takes precedence, including zero. Buffered answers -can also report a charge in `x-litellm-response-cost`. Otherwise, Switchyard prices -input, output, cache reads and cache writes separately. Reasoning tokens already -count toward output. Reported one-hour cache writes require `cache_write_1h`; -writes without a reported duration use `cache_write`. Missing usage or rates leave -cost unknown. The table does not apply service-tier or long-context surcharges. -Omit `--pricing-file` to record reported charges alone. - -Send `x-switchyard-session-id: my-session` with model requests, then read: - -```bash -curl 'http://localhost:4000/v1/routing/session-stats?session_id=my-session' -jq -c '{model, cost}' routing.jsonl -``` - -A priced log record contains `"cost": {"usd": 0.01, "source": "estimated"}` or -`"source": "provider_reported"`; an unpriced record contains `"cost": null`. -The report includes session and per-model `cost`, plus `answer_cost` and -`routing_cost`. Each total has `known_usd`, `total_usd`, `unknown_calls`, -`estimated_calls`, and `provider_reported_calls`. If any recorded call has unknown -cost, `total_usd` is null and `known_usd` remains the priced subtotal. Old records -without a cost count as unknown. - -Totals cover recorded calls, including failed calls observed by the server. -Classifier and judge calls cannot use header charges. Internal retries, cancelled -requests and abandoned streams may leave charges unrecorded. These totals cannot -establish the provider's bill or replace a client's native budget enforcement. - -When Switchyard translates a buffered response to Chat Completions, it writes -cache-write tokens as `prompt_tokens_details.cache_write_tokens`. Earlier -versions wrote `cache_creation_tokens` there. Streamed Chat output already used -`cache_write_tokens`. Both names are accepted on input, and same-format replay -returns the upstream JSON unchanged. - -The log writer holds one rate table in memory. The log grows per call and has no -size limit or rotation from this option. Failed appends produce a warning and are -not recovered; separate replica logs give separate totals. The session endpoint -scans the log on each read. diff --git a/docs/integrations/oh_my_pi.md b/docs/integrations/oh_my_pi.md index d98e6fc16..2fadf0fa4 100644 --- a/docs/integrations/oh_my_pi.md +++ b/docs/integrations/oh_my_pi.md @@ -93,10 +93,8 @@ replay or compaction. Port 4000 is also the default port for `omp`'s `litellm` provider and for `omp auth-gateway`. If `LITELLM_API_KEY` is set, `omp` probes `http://localhost:4000/v1` as a LiteLLM proxy. In that case, run Switchyard on another port or set -`LITELLM_BASE_URL`. Set `cost` on the model entry for `omp`'s native estimate. -One rate table under a router alias is an approximation when targets have different prices. For a -separate total that includes routing calls, see -[Record Switchyard session costs](costs.md). +`LITELLM_BASE_URL`. Set `cost` on the model entry if you want `omp` to show a non-zero +cost. ## Claude through an LLM gateway diff --git a/docs/integrations/pi.md b/docs/integrations/pi.md index 11ee5f8ae..34bd91323 100644 --- a/docs/integrations/pi.md +++ b/docs/integrations/pi.md @@ -97,10 +97,7 @@ message. When routing picks another target on the next turn, pi treats the chang model switch: it drops thinking signatures and turns off overflow compaction. pi's OpenAI clients keep the local id `switchyard` instead. -Set `cost` on the model entry for pi's native estimate. One rate table under a -router alias is an approximation when targets have different prices. For a -separate total that uses each served model and includes routing calls, see -[Record Switchyard session costs](costs.md). +Set `cost` on the model entry if you want pi to show a non-zero cost. [`benchmark/run-baseline.sh`](../../benchmark/README.md) runs Terminal-Bench tasks with pi through Switchyard when you pass `--agent pi`. @@ -177,9 +174,6 @@ each request's cost with: jq -c '{route_id, model, cost}' PATH ``` -[Record Switchyard session costs](costs.md) describes the pricing file and how the -server totals a session. - ### Thinking With `reasoning: true` on the model entry, pi sends a thinking level on every request, From 7ea13da82a64ec86c2a8a3ccb60736d383cf3997 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 20 Oct 2026 13:22:32 -0700 Subject: [PATCH 6/8] chore(docs): restore pricing-file setup instructions Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-server/README.md | 3 +++ docs/integrations/pi.md | 42 +++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index c069a6b14..6402eec6c 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -136,6 +136,9 @@ Session stats include cost totals. Missing usage or rates leave cost unknown; `k keeps the priced subtotal and `total_usd` is null when any cost is unknown. Rates load once at startup, and saved costs do not change when prices change. +The [pi guide](../../docs/integrations/pi.md#estimate-the-cost) explains how to supply +rates for your own models. + Clients can send `x-switchyard-origin: codex-cli` (or another client label) to include an `origin` field in each routing record. Missing, empty, or non-text header values produce `"origin": null`. The value is supplied by the caller; it is not inferred from `User-Agent`. diff --git a/docs/integrations/pi.md b/docs/integrations/pi.md index 34bd91323..1c71f9aa3 100644 --- a/docs/integrations/pi.md +++ b/docs/integrations/pi.md @@ -155,9 +155,22 @@ second shows `cached_tokens` close to `prompt_tokens`. If the second record show #### Estimate the cost -Start the server with `--pricing-file prices.json` as well, and each record gets a `cost`. -The file maps each record's `model` value to prices in USD per million tokens. This -example uses Anthropic's list prices for Claude Sonnet 5 on 2026-10-02: +The server matches each token count in the routing log to a rate in the pricing file: + +| Tokens in the record | Pricing-file key | +|---|---| +| `prompt_tokens - cached_tokens - cache_creation_tokens` | `input` | +| `cached_tokens` | `cache_read` | +| `cache_creation_tokens - cache_creation_1h_tokens` | `cache_write` | +| `cache_creation_1h_tokens`, when reported | `cache_write_1h` | +| `completion_tokens` | `output` | + +Treat a missing `cache_creation_1h_tokens` count as zero. That count is part of +`cache_creation_tokens`, so it must not be charged twice. + +Write the prices to `prices.json` in USD per million tokens, and key each entry by the +record's `model` value. This example uses Anthropic's list prices for Claude Sonnet 5 on +2026-10-02: ```json { @@ -165,10 +178,27 @@ example uses Anthropic's list prices for Claude Sonnet 5 on 2026-10-02: } ``` +For your own models, copy the entry for each model and replace the key with its exact +`model` value from the routing log. Use the served model ID rather than the routing alias, +and replace the example's numbers with your provider's rates. List recorded model IDs with: + +```bash +jq -r '.model' PATH | sort -u +``` + +Every entry requires `input`, `output`, `cache_read`, and `cache_write`. Set a rate to zero +only when that usage is free. Each rate must be a finite number from 0 to 10000 USD per +million tokens. + The example's `cache_write` price is for a 5-minute cache. A 1-hour cache costs 2 times -the input price instead of 1.25 times; set it as `cache_write_1h`. When the gateway -reports its own charge, the server records that charge instead of the estimate. Read -each request's cost with: +the input price instead of 1.25 times; set `cache_write_1h` to `4.00` for that example. +Include that rate to estimate reported one-hour cache writes; without it, the estimate +remains unknown. + +Start the server with `--pricing-file prices.json` as well as `--routing-log-file PATH`. +The server records the gateway's reported charge when available, even without rates. +Otherwise it uses the pricing file to estimate the cost. If usage is missing or the model +has no rates, the estimate is unknown. Read each request's cost with: ```bash jq -c '{route_id, model, cost}' PATH From a39a8dde287fb460fb1019a7cdaa362490ccdcf7 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 20 Oct 2026 13:45:29 -0700 Subject: [PATCH 7/8] fix(costs): retain routing charges and preserve Chat usage fields Signed-off-by: Elyas Mehtabuddin --- crates/libsy-llm-client/src/client.rs | 10 +++++++- crates/libsy-llm-client/src/run.rs | 18 ++++++++----- crates/switchyard-server/src/usage_metrics.rs | 25 +------------------ .../src/codecs/openai_chat/buffered.rs | 2 +- .../tests/response_translation.rs | 2 +- 5 files changed, 24 insertions(+), 33 deletions(-) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 4c3d82a82..3e4fc675a 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -660,7 +660,7 @@ impl TranslatingLlmClient { })?; // Map a provider's failed generation to 502, even under HTTP 200. // Redact forwarded credentials before returning the provider error. - let agg = + let mut agg = observed_translation::decode_response(&body, wire_format).map_err(|error| { match error { TranslationError::UpstreamFailure { error } => { @@ -676,6 +676,14 @@ impl TranslatingLlmClient { error => LlmClientError::ResponseTranslation(error.to_string()), } })?; + // The gateway may report a buffered charge only in its response header. + if agg.usage.provider_cost.is_none() { + agg.usage.provider_cost = upstream_headers + .get("x-litellm-response-cost") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .filter(|usd| usd.is_finite() && *usd >= 0.0); + } (LlmResponse::Agg(agg), upstream_headers) } }; diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 00bdf76c8..661e05c6b 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -33,7 +33,7 @@ use switchyard_libsy::{ use switchyard_protocol::{ AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message, ModelId, Request, Response, ResponseAccumulator, RoutedDecisionClient, RoutedLlmClient, - RoutingFallbackReason, WireFormat, + RoutingFallbackReason, Usage, WireFormat, }; use switchyard_translation::prepare_request_for_target; @@ -376,10 +376,13 @@ async fn call_one( // the provider's, so it belongs in the routing overhead. let client = clients.route(model_id); let started = Instant::now(); + let mut routing_usage = None; let result = async { let mut response = client?.call(request).await?; if is_routing && let LlmResponse::Stream(chunks) = response.llm_response { - response.llm_response = LlmResponse::Stream(buffer_routing_stream(chunks).await?); + let (chunks, usage) = buffer_routing_stream(chunks).await?; + routing_usage = usage; + response.llm_response = LlmResponse::Stream(chunks); } Ok(response) } @@ -405,19 +408,22 @@ async fn call_one( .as_ref() .ok() .and_then(|response| response.llm_response.as_agg()) - .map(|response| response.usage.clone()), + .map(|response| response.usage.clone()) + .or(routing_usage), }); result } -// Preserve signed provider events while checking the complete routing response. +// Routing logs need the final usage, but replay must keep signed provider events intact. async fn buffer_routing_stream( mut chunks: LlmResponseStream, -) -> std::result::Result { +) -> std::result::Result<(LlmResponseStream, Option), LlmClientError> { let mut events = Vec::new(); + let mut usage = None; while let Some(event) = chunks.try_next().await? { for chunk in event.normalized() { match chunk { + LlmResponseChunk::Usage(latest) => usage = Some(latest.clone()), LlmResponseChunk::DecodeError { message } => { return Err(LlmClientError::ResponseTranslation(message.clone())); } @@ -432,7 +438,7 @@ async fn buffer_routing_stream( } events.push(event); } - Ok(stream::iter(events.into_iter().map(Ok)).boxed()) + Ok((stream::iter(events.into_iter().map(Ok)).boxed(), usage)) } /// Whether a failed candidate is worth routing around. diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs index 96a16c9f7..54c2a1c01 100644 --- a/crates/switchyard-server/src/usage_metrics.rs +++ b/crates/switchyard-server/src/usage_metrics.rs @@ -3,7 +3,6 @@ //! Response usage and full-turn latency metrics for routed requests. -use std::borrow::Cow; use std::time::{Duration, Instant}; use futures_util::StreamExt; @@ -34,8 +33,7 @@ pub(crate) fn observe( LlmResponse::Agg(agg) => { record_terminal(&stats, &agg.usage, &model, started, cache_eligible); if let Some((log, context)) = routing_log { - let usage = with_gateway_charge(&agg.usage, &upstream_headers); - log.append(context, &model, None, &usage); + log.append(context, &model, None, &agg.usage); } LlmResponse::Agg(agg) } @@ -110,27 +108,6 @@ pub(crate) fn observe( } } -/// LiteLLM sends the charge for a buffered response in this header and leaves the cost out of -/// the response body. Its streamed responses omit the header and report `cost` in the final -/// usage instead, which the response codecs read. -const LITELLM_COST_HEADER: &str = "x-litellm-response-cost"; - -/// This function adds the gateway's header charge to `usage` when the body reported none. -fn with_gateway_charge<'a>(usage: &'a Usage, headers: &http::HeaderMap) -> Cow<'a, Usage> { - let charge = headers - .get(LITELLM_COST_HEADER) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.trim().parse::().ok()) - .filter(|usd| usd.is_finite() && *usd >= 0.0); - match charge { - Some(usd) if usage.provider_cost.is_none() => Cow::Owned(Usage { - provider_cost: Some(usd), - ..usage.clone() - }), - _ => Cow::Borrowed(usage), - } -} - // Records a terminal stream failure after the routed call was already counted. fn record_stream_error(stats: &StatsAccumulator, model: &str) { stats.record_response_error(model); diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 4e00f5d0a..0dc064162 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -1292,7 +1292,7 @@ pub(crate) fn encode_openai_usage(usage: &Usage) -> Value { if usage.cached_input_tokens().is_some() || usage.cache_creation_input_tokens().is_some() { value["prompt_tokens_details"] = json!({ "cached_tokens": usage.cached_input_tokens().unwrap_or(0), - "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), + "cache_creation_tokens": usage.cache_creation_input_tokens().unwrap_or(0), }); } if let Some(reasoning_tokens) = usage.reasoning_tokens { diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs index 7d3ce5df5..cef7155dd 100644 --- a/crates/switchyard-translation/tests/response_translation.rs +++ b/crates/switchyard-translation/tests/response_translation.rs @@ -377,7 +377,7 @@ fn responses_reasoning_usage_translates_to_openai_chat_usage_details() -> TestRe json!({"reasoning_tokens": 3}) ); assert_eq!( - output["usage"]["prompt_tokens_details"]["cache_write_tokens"], + output["usage"]["prompt_tokens_details"]["cache_creation_tokens"], 2 ); let decoded = engine From 94e476ebfbc59e4ec090bd31191c8cbea69f6236 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 20 Oct 2026 14:10:14 -0700 Subject: [PATCH 8/8] feat(costs): share recorded spending with the menubar Signed-off-by: Elyas Mehtabuddin --- Cargo.lock | 14 ++ Cargo.toml | 2 + crates/switchyard-costs/Cargo.toml | 19 ++ .../src}/anthropic.rs | 0 .../src/lib.rs} | 80 +++++--- crates/switchyard-costs/src/main.rs | 172 ++++++++++++++++++ crates/switchyard-menubar/Cargo.toml | 2 + crates/switchyard-menubar/README.md | 104 ++++++----- crates/switchyard-menubar/src/app.rs | 15 +- crates/switchyard-menubar/src/config.rs | 25 ++- crates/switchyard-menubar/src/pricing.rs | 121 ++++++------ crates/switchyard-menubar/src/rollup.rs | 116 +++++++++--- crates/switchyard-menubar/src/summary.rs | 65 ++++--- crates/switchyard-server/Cargo.toml | 1 + crates/switchyard-server/README.md | 3 +- crates/switchyard-server/src/lib.rs | 3 +- crates/switchyard-server/src/routing_log.rs | 2 +- scripts/macos/install.sh | 62 ++----- 18 files changed, 539 insertions(+), 267 deletions(-) create mode 100644 crates/switchyard-costs/Cargo.toml rename crates/{switchyard-server/src/pricing => switchyard-costs/src}/anthropic.rs (100%) rename crates/{switchyard-server/src/pricing.rs => switchyard-costs/src/lib.rs} (67%) create mode 100644 crates/switchyard-costs/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 705b86d08..19bf434ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3640,6 +3640,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "switchyard-costs" +version = "0.3.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "switchyard-protocol", + "toml 0.9.12+spec-1.1.0", +] + [[package]] name = "switchyard-desktop" version = "0.3.0" @@ -3723,6 +3734,8 @@ dependencies = [ "objc2-foundation", "serde", "serde_json", + "switchyard-costs", + "switchyard-protocol", "tempfile", "toml 0.9.12+spec-1.1.0", "tray-icon", @@ -3859,6 +3872,7 @@ dependencies = [ "rustls", "serde", "serde_json", + "switchyard-costs", "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", diff --git a/Cargo.toml b/Cargo.toml index e13c41b0d..60228cabd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/switchyard-runner", "crates/switchyard-desktop", "crates/switchyard-menubar", + "crates/switchyard-costs", "crates/switchyard-server", "crates/switchyard-skill-distillation", "crates/switchyard-soak", @@ -49,6 +50,7 @@ regex = "1" reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } +switchyard-costs = { path = "crates/switchyard-costs", version = "0.3.0" } switchyard-libsy = { path = "crates/libsy", version = "0.3.0" } switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.3.0" } switchyard-protocol = { path = "crates/protocol", version = "0.3.0" } diff --git a/crates/switchyard-costs/Cargo.toml b/crates/switchyard-costs/Cargo.toml new file mode 100644 index 000000000..7176256d8 --- /dev/null +++ b/crates/switchyard-costs/Cargo.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-costs" +version.workspace = true +description = "Shared token pricing and recorded cost totals for Switchyard" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +chrono = { version = "0.4", default-features = false, features = ["std"] } +switchyard-protocol.workspace = true +serde.workspace = true +serde_json.workspace = true +toml = "0.9" diff --git a/crates/switchyard-server/src/pricing/anthropic.rs b/crates/switchyard-costs/src/anthropic.rs similarity index 100% rename from crates/switchyard-server/src/pricing/anthropic.rs rename to crates/switchyard-costs/src/anthropic.rs diff --git a/crates/switchyard-server/src/pricing.rs b/crates/switchyard-costs/src/lib.rs similarity index 67% rename from crates/switchyard-server/src/pricing.rs rename to crates/switchyard-costs/src/lib.rs index 4645e05b0..a0001ea19 100644 --- a/crates/switchyard-server/src/pricing.rs +++ b/crates/switchyard-costs/src/lib.rs @@ -11,11 +11,9 @@ use std::path::Path; use serde::{Deserialize, Serialize}; use switchyard_protocol::Usage; -use crate::{ServerError, ServerResult}; - /// Pricing maps served model IDs to token rates in USD per million tokens. -#[derive(Debug, Default)] -pub(crate) struct Pricing(HashMap); +#[derive(Clone, Debug, Default)] +pub struct Pricing(HashMap); impl<'de> Deserialize<'de> for Pricing { fn deserialize>(deserializer: D) -> Result { @@ -45,7 +43,7 @@ impl<'de> Deserialize<'de> for Pricing { } } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] struct Rates { input: f64, @@ -55,34 +53,36 @@ struct Rates { cache_write_1h: Option, } +/// This value identifies how a saved monetary amount was obtained. #[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum CostSource { +pub enum CostSource { + /// The provider supplied this charge. ProviderReported, + /// Configured token rates supplied this estimate. Estimated, } +/// This record stores a charge or estimate for one call. #[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)] -pub(crate) struct RecordedCost { - pub(crate) usd: f64, - pub(crate) source: CostSource, +pub struct RecordedCost { + /// The amount is in USD. + pub usd: f64, + /// The source distinguishes charges from estimates. + pub source: CostSource, } impl Pricing { - pub(crate) fn load(path: &Path) -> ServerResult { - let bytes = std::fs::read(path).map_err(|error| { - ServerError::new(format!( - "failed to read pricing file {}: {error}", - path.display() - )) - })?; - Self::parse(&bytes).map_err(|error| { - ServerError::new(format!("invalid pricing file {}: {error}", path.display())) - }) + /// This method loads and validates a JSON pricing file. + pub fn load(path: &Path) -> Result { + let bytes = std::fs::read(path) + .map_err(|error| format!("read pricing file {}: {error}", path.display()))?; + Self::parse(&bytes) + .map_err(|error| format!("invalid pricing file {}: {error}", path.display())) } /// This function reads a pricing file's JSON and checks every model ID and rate. - fn parse(json: &[u8]) -> Result { + pub fn parse(json: &[u8]) -> Result { let pricing: Self = serde_json::from_slice(json).map_err(|error| error.to_string())?; for (model, rate) in &pricing.0 { if model.trim().is_empty() @@ -107,7 +107,7 @@ impl Pricing { } /// Provider charges take precedence because token rates may omit provider surcharges. - pub(crate) fn cost(&self, model: &str, usage: &Usage) -> Option { + pub fn cost(&self, model: &str, usage: &Usage) -> Option { if let Some(usd) = usage .provider_cost .filter(|value| value.is_finite() && *value >= 0.0) @@ -117,6 +117,11 @@ impl Pricing { source: CostSource::ProviderReported, }); } + self.estimate(model, usage) + } + + /// This method uses token rates alone so baseline estimates exclude provider charges. + pub fn estimate(&self, model: &str, usage: &Usage) -> Option { let rate = self.0.get(model)?; let input = usage.input_tokens?; let output = usage.output_tokens?; @@ -135,12 +140,17 @@ impl Pricing { /// The known subtotal remains visible when any call has unknown cost. #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] -pub(crate) struct CostTotals { - pub(crate) known_usd: f64, - pub(crate) unknown_calls: u64, - pub(crate) estimated_calls: u64, - pub(crate) provider_reported_calls: u64, - pub(crate) total_usd: Option, +pub struct CostTotals { + /// This subtotal includes every known cost. + pub known_usd: f64, + /// This count includes calls whose cost is unknown. + pub unknown_calls: u64, + /// This count includes calls priced from configured rates. + pub estimated_calls: u64, + /// This count includes calls with a provider charge. + pub provider_reported_calls: u64, + /// The total is absent when any call has unknown cost. + pub total_usd: Option, } impl Default for CostTotals { @@ -156,7 +166,8 @@ impl Default for CostTotals { } impl CostTotals { - pub(crate) fn add(&mut self, cost: Option) { + /// This method adds one call and keeps the total unknown if any cost is unknown. + pub fn add(&mut self, cost: Option) { match cost.filter(|cost| { cost.usd.is_finite() && cost.usd >= 0.0 && (self.known_usd + cost.usd).is_finite() }) { @@ -171,4 +182,17 @@ impl CostTotals { } self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); } + + /// This method combines disjoint groups of calls without repricing them. + pub fn merge(&mut self, other: &Self) { + if (self.known_usd + other.known_usd).is_finite() { + self.known_usd += other.known_usd; + } else { + self.unknown_calls += 1; + } + self.unknown_calls += other.unknown_calls; + self.estimated_calls += other.estimated_calls; + self.provider_reported_calls += other.provider_reported_calls; + self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); + } } diff --git a/crates/switchyard-costs/src/main.rs b/crates/switchyard-costs/src/main.rs new file mode 100644 index 000000000..dc3f7b7f4 --- /dev/null +++ b/crates/switchyard-costs/src/main.rs @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! This command migrates old settings and logs without changing the originals. + +use std::io::{BufRead, Write}; +use std::path::Path; + +use serde::Deserialize; +use serde_json::{Value, json}; +use switchyard_costs::{Pricing, RecordedCost}; +use switchyard_protocol::Usage; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct OldRate { + input_per_mtok: f64, + output_per_mtok: f64, + cached_input_per_mtok: Option, +} + +#[derive(Deserialize)] +struct Record { + ts: String, + #[serde(default, rename = "tier")] + _tier: String, + model: String, + prompt_tokens: Option, + completion_tokens: Option, + cached_tokens: Option, + cache_creation_tokens: Option, + cache_creation_1h_tokens: Option, + // The nested option distinguishes an absent cost from an explicit unknown cost. + #[serde(default, deserialize_with = "present_cost")] + cost: Option>, +} + +// An explicit null remains unknown instead of being reconstructed from token rates. +fn present_cost<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result>, D::Error> { + Option::deserialize(deserializer).map(Some) +} + +// Only records without a cost field receive reconstructed estimates. +fn migrate( + settings: &Path, + log: &Path, + destination: &Path, +) -> Result<(), Box> { + let mut config: toml::Table = std::fs::read_to_string(settings)?.parse()?; + let rates = match config.remove("prices") { + Some(old) => { + if config.contains_key("pricing_file") { + return Err("settings contain both prices and pricing_file".into()); + } + let old: std::collections::BTreeMap = old.try_into()?; + Value::Object(old.into_iter().map(|(model, rate)| (model, json!({ + "input":rate.input_per_mtok, "output":rate.output_per_mtok, + "cache_read":rate.cached_input_per_mtok.unwrap_or(rate.input_per_mtok), + "cache_write":rate.input_per_mtok, + }))).collect()) + } + None => return Err("migration requires the old settings with a [prices] table".into()), + }; + let rates = serde_json::to_vec_pretty(&rates)?; + let pricing = Pricing::parse(&rates)?; + let destination = std::path::absolute(destination)?; + config.insert( + "pricing_file".into(), + toml::Value::String( + destination + .join("prices.json") + .to_string_lossy() + .into_owned(), + ), + ); + config.insert( + "routing_log".into(), + toml::Value::String( + destination + .join("routing.jsonl") + .to_string_lossy() + .into_owned(), + ), + ); + // Creating a fresh directory prevents overwriting inputs or earlier migration results. + std::fs::create_dir(&destination)?; + let result = (|| -> Result<(), Box> { + let mut output = + std::io::BufWriter::new(std::fs::File::create(destination.join("routing.jsonl"))?); + let mut source = std::io::BufReader::new(std::fs::File::open(log)?); + let mut line = String::new(); + loop { + line.clear(); + if source.read_line(&mut line)? == 0 { + break; + } + if !line.ends_with('\n') { + return Err("unfinished log record; stop the writer before migrating".into()); + } + // Typed parsing catches duplicate known fields before Value could overwrite them. + let record: Record = serde_json::from_str(&line)?; + chrono::DateTime::parse_from_rfc3339(&record.ts)?; + if record.model.trim().is_empty() { + return Err("log record has an empty model".into()); + } + if let Some(Some(cost)) = record.cost + && (!cost.usd.is_finite() || cost.usd < 0.0) + { + return Err("log record has an invalid cost".into()); + } + let mut value: Value = serde_json::from_str(&line)?; + if !value.is_object() { + return Err("log record must be a JSON object".into()); + } + if record.cost.is_none() { + let read = record.cached_tokens.unwrap_or(0); + let written = record.cache_creation_tokens.unwrap_or(0); + let mut cache = Usage::cache_details(Some(read), Some(written)); + if let Some(tokens) = record.cache_creation_1h_tokens { + cache + .get_or_insert_with(Default::default) + .cache_creation_1h_input_tokens = Some(tokens); + } + let usage = Usage { + input_tokens: record + .prompt_tokens + .and_then(|input| input.checked_sub(read)?.checked_sub(written)), + output_tokens: record.completion_tokens, + cache, + ..Usage::default() + }; + value["cost"] = serde_json::to_value(pricing.estimate(&record.model, &usage))?; + } + serde_json::to_writer(&mut output, &value)?; + output.write_all(b"\n")?; + } + output.flush()?; + std::fs::write(destination.join("prices.json"), rates)?; + std::fs::write( + destination.join("menubar.toml"), + toml::to_string_pretty(&config)?, + )?; + Ok(()) + })(); + if result.is_err() { + std::fs::remove_dir_all(&destination)?; + } + result +} + +fn main() -> std::process::ExitCode { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if args.len() != 3 { + eprintln!( + "Usage: switchyard-costs OLD_MENUBAR_TOML ROUTING_JSONL NEW_DIRECTORY\nStop the server and menubar first. Outputs use the rates in the old settings." + ); + return std::process::ExitCode::FAILURE; + } + match migrate( + Path::new(&args[0]), + Path::new(&args[1]), + Path::new(&args[2]), + ) { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(error) => { + eprintln!("switchyard-costs: {error}"); + std::process::ExitCode::FAILURE + } + } +} diff --git a/crates/switchyard-menubar/Cargo.toml b/crates/switchyard-menubar/Cargo.toml index d12551f0d..b839ad9c9 100644 --- a/crates/switchyard-menubar/Cargo.toml +++ b/crates/switchyard-menubar/Cargo.toml @@ -14,6 +14,8 @@ rust-version.workspace = true publish = false [dependencies] +switchyard-costs.workspace = true +switchyard-protocol.workspace = true chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } serde.workspace = true serde_json.workspace = true diff --git a/crates/switchyard-menubar/README.md b/crates/switchyard-menubar/README.md index 4cc99d208..8525a5878 100644 --- a/crates/switchyard-menubar/README.md +++ b/crates/switchyard-menubar/README.md @@ -36,55 +36,69 @@ routing_log = "~/.switchyard/routing.jsonl" baseline_model = "gpt-5.6-sol" refresh_seconds = 30 -[prices."gpt-5.6-sol"] -input_per_mtok = 1.25 -cached_input_per_mtok = 0.125 -output_per_mtok = 10.0 +pricing_file = "~/.switchyard/prices.json" ``` +The server's `--pricing-file` and the menubar's `pricing_file` use the same JSON file. +Key each entry by the model ID in the routing log. Rates are in USD per million tokens: + +```json +{"gpt-5.6-sol":{"input":1.25,"cache_read":0.125,"cache_write":1.25,"output":10}} +``` + +These example rates preserve the installer's previous assumptions. Replace them with +what you pay. The [pi guide](../../docs/integrations/pi.md#estimate-the-cost) explains +cache-write rates and how to add other models. Rates load at startup; restart both +applications after editing them. + ## How savings are computed -The server writes one JSONL record per call, with the model that answered and -its token counts. - -- **Actual** is every call priced at the model that served it, including - Switchyard's own classifier calls. -- **Baseline** is the caller-facing calls only, priced as if each had used - `baseline_model`. Classifier calls have no baseline counterpart, because - without Switchyard they would not happen. - -Savings are the difference, so routing overhead counts against the figure and -a bad day shows a negative number. Dollar figures stay hidden until every -model seen has a price, so a partial table cannot mislead. - -Codex's "Approve for me" reviews count too. The server config that the -installer writes sends them to `codex-auto-review` on the ChatGPT backend. With -a ChatGPT login, Codex sends these reviews to `codex-auto-review` even without -Switchyard, so the installer prices that model at the `baseline_model` rates. -Each review then adds the same amount to the actual cost and to the baseline, -so reviews do not change the dollar amount saved. They do lower the percentage -a little, because the baseline grows. If you change `baseline_model`, update -this price to match. - -The menu bar needs a price for every model ID that the routing log records. -The macOS installer includes prices for the default models, including -`gpt-6-luna`, `gpt-6-sol`, and `gpt-6.1-sol`. Existing `menubar.toml` files are -preserved during reinstall, so add any missing model prices to that file by -hand. - -Codex's "Approve for me" reviews need a price too. Use the reviewer target's -`id`, which is the model ID recorded in the routing log. The default reviewer -route uses `codex-auto-review`. With a ChatGPT login, these reviews also happen -without Switchyard. Add that ID to the price table at the `baseline_model` -rates, including the cached input rate. Each review then adds the same cost to -actual and baseline. Dollars saved stay the same; the percentage drops as the -baseline grows. Update this price when you change `baseline_model`. - -If a reviewer uses another model ID, add a price for that ID. Any period with -an unpriced model hides its savings and shows the missing-price hint. - -These are list-price estimates. On a ChatGPT login there is no per-token bill, -so read them as "what this traffic would have cost at API rates". +Actual spending is the sum of the costs saved in the routing log, including routing +calls. Changing prices does not change saved costs. The baseline is an estimate of +what the answer traffic would have cost using `baseline_model`; it excludes routing +calls. The server and menubar share the code for cost totals and token pricing, including +cache-read and cache-write rates. +The baseline assumes the same token and cache counts as the observed answer traffic. + +The menubar shows estimated savings only when actual costs and the baseline estimate +are complete. A recorded `cost: null` keeps spending unknown. A missing baseline rate +or required one-hour cache-write rate keeps the baseline unknown. Missing or invalid answer +usage also keeps the baseline unknown. Token counts still +appear when savings cannot be calculated. The log can be read while the server is offline. + +Codex's `codex-auto-review` model keeps the installer's baseline rates by default. +Update that entry when changing `baseline_model` if you want those reviews to have +the same estimated cost with and without Switchyard. ChatGPT subscription traffic +may have no per-token bill; token-rate estimates are comparisons at the supplied rates. + +## Migrate older settings and logs + +Inline `[prices]` settings and log records without a `cost` field require migration. +Stop the server and menubar before migrating. From the repository root, run: + +```sh +cargo run -p switchyard-costs -- \ + ~/.switchyard/menubar.toml ~/.switchyard/routing.jsonl ~/.switchyard/migrated-costs +``` + +The destination directory must not exist. The command writes `menubar.toml`, +`prices.json`, and `routing.jsonl` there and leaves the originals untouched. It preserves +existing costs, including null, and adds estimates only to records without a cost field. +Missing usage or rates leave the reconstructed cost unknown. Malformed or unfinished +records stop migration; failed migration outputs are removed. + +The command uses the rates in the old settings. It cannot recover historical provider +charges or historical prices. It maps `input_per_mtok` to `input` and `cache_write`, +`output_per_mtok` to `output`, and `cached_input_per_mtok` to `cache_read` (or the input +rate when omitted). This preserves the old cache-write assumption. Review those rates +for your providers; add `cache_write_1h` when needed. Reconstruction is always marked +`estimated`, even when it uses historical prices. + +Review the generated files, then start the menubar with the generated settings path. +Those settings point at the generated log and pricing file. Update the server's +LaunchAgent to use those same paths for `--routing-log-file` and `--pricing-file` before +restarting it. Keep the originals as backups. Reinstalling preserves existing files; +it does not migrate them. ## Codex profiles diff --git a/crates/switchyard-menubar/src/app.rs b/crates/switchyard-menubar/src/app.rs index 158411f40..3695ce9b0 100644 --- a/crates/switchyard-menubar/src/app.rs +++ b/crates/switchyard-menubar/src/app.rs @@ -12,15 +12,10 @@ use crate::summary::{Row, build}; /// Reads new log entries, probes the server, and builds the menu body. pub fn refresh(config: &Config, log: &mut rollup::Reader) -> Vec { - let usage = log - .read(&config.routing_log, Local::now().date_naive()) - .unwrap_or_else(|error| { - eprintln!( - "switchyard-menubar: read {}: {error}", - config.routing_log.display() - ); - rollup::Usage::default() - }); + let usage = match log.read(&config.routing_log, Local::now().date_naive()) { + Ok(usage) => usage, + Err(error) => return vec![Row::Label(format!("Cannot read routing log: {error}"))], + }; build(probe(&config.server_url), &usage, config) } @@ -36,7 +31,7 @@ mod tests { std::fs::write( &path, format!( - r#"{{"ts":"{ts}","model":"luna","tier":"","prompt_tokens":1000,"cached_tokens":0,"completion_tokens":100,"reasoning_tokens":0}}"# + r#"{{"ts":"{ts}","model":"luna","tier":"","prompt_tokens":1000,"cached_tokens":0,"completion_tokens":100,"reasoning_tokens":0,"cost":null}}"# ) + "\n", ) .expect("write log"); diff --git a/crates/switchyard-menubar/src/config.rs b/crates/switchyard-menubar/src/config.rs index 8b29b4b6a..cebc785bf 100644 --- a/crates/switchyard-menubar/src/config.rs +++ b/crates/switchyard-menubar/src/config.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; use serde::Deserialize; -use crate::pricing::PriceTable; +use switchyard_costs::Pricing; /// Everything the menu bar needs to find the server and price its traffic. /// Missing keys fall back to [`Config::default`]. @@ -26,8 +26,11 @@ pub struct Config { pub refresh_seconds: u64, /// Model the traffic is assumed to have used without Switchyard. pub baseline_model: String, - /// Per-model rates. Dollar figures are hidden while a seen model is absent. - pub prices: PriceTable, + /// The shared JSON file supplies rates for the baseline estimate. + pub pricing_file: Option, + /// Rates load once when the settings are read. + #[serde(skip)] + pub prices: Pricing, } impl Default for Config { @@ -39,7 +42,8 @@ impl Default for Config { launchd_label: "com.nvidia.switchyard.server".to_string(), refresh_seconds: 30, baseline_model: "gpt-5.6-sol".to_string(), - prices: PriceTable::new(), + pricing_file: None, + prices: Pricing::default(), } } } @@ -53,6 +57,11 @@ impl Config { Err(error) if error.kind() == std::io::ErrorKind::NotFound => Self::default(), Err(error) => return Err(format!("read {}: {error}", path.display())), }; + if let Some(path) = config.pricing_file.as_ref() { + let path = expand_home(path); + config.prices = Pricing::load(&path)?; + config.pricing_file = Some(path); + } config.routing_log = expand_home(&config.routing_log); config.config_file = expand_home(&config.config_file); Ok(config) @@ -100,10 +109,6 @@ mod tests { server_url = "http://127.0.0.1:9000" baseline_model = "sol" -[prices.luna] -input_per_mtok = 0.25 -cached_input_per_mtok = 0.025 -output_per_mtok = 2.0 "#, ) .expect("write settings"); @@ -112,7 +117,7 @@ output_per_mtok = 2.0 assert_eq!(config.server_url, "http://127.0.0.1:9000"); assert_eq!(config.baseline_model, "sol"); - assert_eq!(config.prices["luna"].cached_input_per_mtok, Some(0.025)); + assert!(config.pricing_file.is_none()); assert_eq!(config.refresh_seconds, 30, "unset keys keep their default"); } @@ -123,7 +128,7 @@ output_per_mtok = 2.0 let config = Config::load(&dir.path().join("absent.toml")).expect("load"); assert_eq!(config.server_url, "http://127.0.0.1:4123"); - assert!(config.prices.is_empty()); + assert!(config.pricing_file.is_none()); let empty = dir.path().join("empty.toml"); std::fs::write(&empty, "").expect("write settings"); let loaded = Config::load(&empty).expect("load empty settings"); diff --git a/crates/switchyard-menubar/src/pricing.rs b/crates/switchyard-menubar/src/pricing.rs index e551bdd71..e2cf1c98d 100644 --- a/crates/switchyard-menubar/src/pricing.rs +++ b/crates/switchyard-menubar/src/pricing.rs @@ -4,37 +4,10 @@ //! Counterfactual cost: what routed traffic cost against what it would have //! cost had every call gone to one capable model. -use std::collections::BTreeMap; +use switchyard_costs::{CostTotals, Pricing}; +use switchyard_protocol::{InputCacheUsage, Usage}; -use serde::Deserialize; - -use crate::rollup::{ModelTokens, Totals}; - -const PER_MILLION: f64 = 1_000_000.0; - -/// Per-million-token rates for one model. -#[derive(Clone, Copy, Debug, Deserialize, PartialEq)] -pub struct ModelPrice { - pub input_per_mtok: f64, - /// Rate for cache reads. Falls back to the full input rate. - #[serde(default)] - pub cached_input_per_mtok: Option, - /// Rate for generated tokens, reasoning included. - pub output_per_mtok: f64, -} - -impl ModelPrice { - fn cost(&self, tokens: &ModelTokens) -> f64 { - let cached_rate = self.cached_input_per_mtok.unwrap_or(self.input_per_mtok); - (tokens.input as f64 * self.input_per_mtok - + tokens.cached_input as f64 * cached_rate - + tokens.output as f64 * self.output_per_mtok) - / PER_MILLION - } -} - -/// Rates keyed by the model id the server records in the routing log. -pub type PriceTable = BTreeMap; +use crate::rollup::Totals; /// What the traffic cost, and what it would have cost without routing. #[derive(Clone, Copy, Debug, PartialEq)] @@ -59,44 +32,52 @@ impl Savings { /// Prices a period against the baseline model. /// -/// Returns `None` when any model seen has no entry in the table, since a -/// partial bill would understate cost and overstate savings. -pub fn estimate(totals: &Totals, prices: &PriceTable, baseline_model: &str) -> Option { - let baseline_price = prices.get(baseline_model)?; - - let mut actual = 0.0; - let mut baseline = 0.0; - for (model, tokens) in &totals.routed { - actual += prices.get(model)?.cost(tokens); - baseline += baseline_price.cost(tokens); +/// Returns `None` when recorded spending or the baseline estimate is incomplete. +pub fn estimate(totals: &Totals, prices: &Pricing, baseline_model: &str) -> Option { + if totals.baseline_unknown { + return None; } - // Classifier calls are Switchyard's own cost. They have no baseline - // counterpart, so they only ever reduce the savings figure. - for (model, tokens) in &totals.classifier { - actual += prices.get(model)?.cost(tokens); + let mut baseline = CostTotals::default(); + for tokens in totals.routed.values() { + let usage = Usage { + input_tokens: Some(tokens.input), + output_tokens: Some(tokens.output), + cache: Some(Box::new(InputCacheUsage { + cached_input_tokens: Some(tokens.cached_input), + cache_creation_input_tokens: Some(tokens.cache_write), + cache_creation_1h_input_tokens: Some(tokens.cache_write_1h), + })), + ..Usage::default() + }; + baseline.add(prices.estimate(baseline_model, &usage)); } - - Some(Savings { actual, baseline }) + Some(Savings { + actual: totals.cost.total_usd?, + baseline: baseline.total_usd?, + }) } #[cfg(test)] mod tests { use super::*; - - fn price(input: f64, output: f64) -> ModelPrice { - ModelPrice { - input_per_mtok: input, - cached_input_per_mtok: None, - output_per_mtok: output, - } + use crate::rollup::ModelTokens; + + fn table() -> Pricing { + Pricing::parse( + br#"{ + "sol":{"input":1.25,"cache_read":1.25,"cache_write":1.25,"output":10}, + "luna":{"input":0.25,"cache_read":0.25,"cache_write":0.25,"output":2}, + "terra":{"input":0.05,"cache_read":0.05,"cache_write":0.05,"output":0.4} + }"#, + ) + .expect("rates") } - fn table() -> PriceTable { - PriceTable::from([ - ("sol".to_string(), price(1.25, 10.0)), - ("luna".to_string(), price(0.25, 2.0)), - ("terra".to_string(), price(0.05, 0.4)), - ]) + fn record_cost(totals: &mut Totals, usd: f64) { + totals.cost.add(Some(switchyard_costs::RecordedCost { + usd, + source: switchyard_costs::CostSource::Estimated, + })); } fn tokens(input: u64, output: u64) -> ModelTokens { @@ -105,6 +86,7 @@ mod tests { input, cached_input: 0, output, + ..ModelTokens::default() } } @@ -115,6 +97,7 @@ mod tests { .routed .insert("luna".to_string(), tokens(1_000_000, 100_000)); + record_cost(&mut totals, 0.45); let savings = estimate(&totals, &table(), "sol").expect("priced"); assert!((savings.actual - 0.45).abs() < 1e-9); @@ -128,11 +111,13 @@ mod tests { totals .routed .insert("luna".to_string(), tokens(1_000_000, 100_000)); + record_cost(&mut totals, 0.45); let before = estimate(&totals, &table(), "sol").expect("priced"); totals .classifier .insert("terra".to_string(), tokens(1_000_000, 10_000)); + record_cost(&mut totals, 0.054); let after = estimate(&totals, &table(), "sol").expect("priced"); assert_eq!(after.baseline, before.baseline); @@ -141,14 +126,9 @@ mod tests { #[test] fn cache_reads_use_the_cached_rate() { - let prices = PriceTable::from([( - "sol".to_string(), - ModelPrice { - input_per_mtok: 1.0, - cached_input_per_mtok: Some(0.1), - output_per_mtok: 0.0, - }, - )]); + let prices = + Pricing::parse(br#"{"sol":{"input":1,"cache_read":0.1,"cache_write":1,"output":0}}"#) + .expect("rates"); let mut totals = Totals::default(); totals.routed.insert( "sol".to_string(), @@ -157,23 +137,26 @@ mod tests { input: 1_000_000, cached_input: 1_000_000, output: 0, + ..ModelTokens::default() }, ); + record_cost(&mut totals, 1.1); let savings = estimate(&totals, &prices, "sol").expect("priced"); assert!((savings.actual - 1.1).abs() < 1e-9); } #[test] - fn an_unpriced_model_suppresses_the_estimate() { + fn an_unknown_cost_suppresses_savings() { let mut totals = Totals::default(); totals .routed .insert("unknown".to_string(), tokens(1_000, 100)); + totals.cost.add(None); assert!(estimate(&totals, &table(), "sol").is_none()); - assert!(estimate(&Totals::default(), &table(), "absent").is_none()); + assert!(estimate(&totals, &table(), "absent").is_none()); } #[test] diff --git a/crates/switchyard-menubar/src/rollup.rs b/crates/switchyard-menubar/src/rollup.rs index c05a34780..0641ccb73 100644 --- a/crates/switchyard-menubar/src/rollup.rs +++ b/crates/switchyard-menubar/src/rollup.rs @@ -10,6 +10,7 @@ use std::path::Path; use chrono::{DateTime, Local, NaiveDate}; use serde::Deserialize; +use switchyard_costs::{CostTotals, RecordedCost}; /// Tier value the server writes for classifier and judge calls. const CLASSIFIER_TIER: &str = "classifier"; @@ -18,19 +19,22 @@ const CLASSIFIER_TIER: &str = "classifier"; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct ModelTokens { pub requests: u64, - /// Input tokens billed at the full rate. Cache writes bill at that rate - /// too, so they are counted here rather than tracked separately. + /// Input tokens exclude cache reads and writes. pub input: u64, /// Input tokens served from the provider cache. pub cached_input: u64, /// Generated tokens, including reasoning. pub output: u64, + /// Cache-write tokens are separate from uncached input. + pub cache_write: u64, + /// One-hour writes are a subset of all cache writes. + pub cache_write_1h: u64, } impl ModelTokens { /// Every token the provider counted, cached reads included. pub fn total(&self) -> u64 { - self.input + self.cached_input + self.output + self.input + self.cached_input + self.cache_write + self.output } fn add(&mut self, other: &Self) { @@ -38,6 +42,8 @@ impl ModelTokens { self.input += other.input; self.cached_input += other.cached_input; self.output += other.output; + self.cache_write += other.cache_write; + self.cache_write_1h += other.cache_write_1h; } } @@ -45,8 +51,12 @@ impl ModelTokens { /// /// The split matters for savings: without Switchyard the `routed` calls would /// still have happened, but the `classifier` calls would not exist at all. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct Totals { + /// Actual spending comes only from the saved cost of each call. + pub cost: CostTotals, + /// Missing or invalid answer usage keeps savings unknown after totals are merged. + pub baseline_unknown: bool, /// Calls that served the caller, keyed by the model that answered. pub routed: BTreeMap, /// Switchyard's own classifier and judge calls, keyed by model. @@ -55,6 +65,10 @@ pub struct Totals { impl Totals { fn add(&mut self, record: &Record) { + self.cost.add(record.cost); + if record.tier != CLASSIFIER_TIER { + self.baseline_unknown |= !record.has_baseline_usage(); + } let bucket = if record.tier == CLASSIFIER_TIER { &mut self.classifier } else { @@ -67,6 +81,8 @@ impl Totals { } fn merge(&mut self, other: &Self) { + self.cost.merge(&other.cost); + self.baseline_unknown |= other.baseline_unknown; for (model, tokens) in &other.routed { self.routed.entry(model.clone()).or_default().add(tokens); } @@ -113,22 +129,48 @@ struct Record { #[serde(default)] tier: String, #[serde(default)] - prompt_tokens: u64, + prompt_tokens: Option, + #[serde(default)] + cached_tokens: Option, + #[serde(default)] + completion_tokens: Option, #[serde(default)] - cached_tokens: u64, + cache_creation_tokens: Option, #[serde(default)] - completion_tokens: u64, + cache_creation_1h_tokens: Option, + // A missing field requires migration; explicit null is a valid unknown cost. + #[serde(deserialize_with = "Option::deserialize")] + cost: Option, } impl Record { + // Each record must be valid before aggregation can conceal inconsistent cache counts. + fn has_baseline_usage(&self) -> bool { + let Some(input) = self.prompt_tokens else { + return false; + }; + self.completion_tokens.is_some() + && input + .checked_sub(self.cached_tokens.unwrap_or(0)) + .and_then(|input| input.checked_sub(self.cache_creation_tokens.unwrap_or(0))) + .is_some() + && self.cache_creation_1h_tokens.unwrap_or(0) <= self.cache_creation_tokens.unwrap_or(0) + } + fn tokens(&self) -> ModelTokens { ModelTokens { requests: 1, // `prompt_tokens` is the whole input, cache reads included. - input: self.prompt_tokens.saturating_sub(self.cached_tokens), - cached_input: self.cached_tokens, + input: self + .prompt_tokens + .unwrap_or(0) + .saturating_sub(self.cached_tokens.unwrap_or(0)) + .saturating_sub(self.cache_creation_tokens.unwrap_or(0)), + cached_input: self.cached_tokens.unwrap_or(0), // Completion tokens already include reasoning tokens. - output: self.completion_tokens, + output: self.completion_tokens.unwrap_or(0), + cache_write: self.cache_creation_tokens.unwrap_or(0), + cache_write_1h: self.cache_creation_1h_tokens.unwrap_or(0), } } } @@ -176,10 +218,22 @@ impl Reader { if line.last() != Some(&b'\n') { break; } - self.offset += bytes as u64; - let Ok(record) = serde_json::from_slice::(&line) else { - continue; + let record = match serde_json::from_slice::(&line) { + Ok(record) => record, + Err(error) if error.is_data() => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "invalid routing record: {error}; migrate older logs before reading them" + ), + )); + } + Err(_) => { + self.offset += bytes as u64; + continue; + } }; + self.offset += bytes as u64; let Ok(ts) = DateTime::parse_from_rfc3339(&record.ts) else { continue; }; @@ -232,7 +286,7 @@ mod tests { fn line(local_ts: &str, model: &str, tier: &str, prompt: u64, completion: u64) -> String { let offset = Local::now().offset().to_string(); format!( - r#"{{"ts":"{local_ts}{offset}","model":"{model}","tier":"{tier}","prompt_tokens":{prompt},"cached_tokens":0,"completion_tokens":{completion},"reasoning_tokens":0}}"# + r#"{{"ts":"{local_ts}{offset}","model":"{model}","tier":"{tier}","prompt_tokens":{prompt},"cached_tokens":0,"completion_tokens":{completion},"reasoning_tokens":0,"cost":null}}"# ) } @@ -266,7 +320,7 @@ mod tests { #[test] fn separates_cache_reads_from_full_price_input() { let record: Record = serde_json::from_str( - r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":1000,"cached_tokens":600,"completion_tokens":50,"reasoning_tokens":25}"#, + r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":1000,"cached_tokens":600,"completion_tokens":50,"reasoning_tokens":25,"cost":null}"#, ) .expect("parse record"); let tokens = record.tokens(); @@ -279,27 +333,39 @@ mod tests { #[test] fn counts_completion_tokens_once_and_prices_them() { let record: Record = serde_json::from_str( - r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":12,"cached_tokens":0,"completion_tokens":5,"reasoning_tokens":3}"#, + r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":12,"cached_tokens":0,"cache_creation_tokens":10,"completion_tokens":5,"reasoning_tokens":3,"cost":{"usd":0.000022,"source":"provider_reported"}}"#, ) .expect("parse record"); let tokens = record.tokens(); - assert_eq!(tokens.input, 12); + assert_eq!(tokens.input, 2); assert_eq!(tokens.output, 5); assert_eq!(tokens.total(), 17); let mut totals = Totals::default(); totals.add(&record); - let prices = crate::pricing::PriceTable::from([( - "m".to_string(), - crate::pricing::ModelPrice { - input_per_mtok: 1.0, - cached_input_per_mtok: None, - output_per_mtok: 2.0, - }, - )]); + let prices = switchyard_costs::Pricing::parse( + br#"{"m":{"input":1,"cache_read":1,"cache_write":1,"cache_write_1h":3,"output":2}}"#, + ) + .expect("rates"); let savings = crate::pricing::estimate(&totals, &prices, "m").expect("priced"); assert!((savings.actual - 0.000022).abs() < 1e-12); + // A known zero charge still needs valid usage for the hypothetical baseline. + for fields in [ + r#""prompt_tokens":null,"completion_tokens":5"#, + r#""completion_tokens":5"#, + r#""prompt_tokens":2,"cached_tokens":3,"completion_tokens":5"#, + r#""prompt_tokens":20,"cache_creation_tokens":2,"cache_creation_1h_tokens":3,"completion_tokens":5"#, + ] { + let record: Record = serde_json::from_str(&format!( + r#"{{"ts":"2026-09-28T10:00:00Z","model":"m",{fields},"cost":{{"usd":0,"source":"provider_reported"}}}}"# + )).expect("record with incomplete baseline usage"); + let mut incomplete = Totals::default(); + incomplete.add(&record); + incomplete.merge(&totals); + assert_eq!(incomplete.cost.total_usd, Some(0.000022)); + assert!(crate::pricing::estimate(&incomplete, &prices, "m").is_none()); + } } #[test] diff --git a/crates/switchyard-menubar/src/summary.rs b/crates/switchyard-menubar/src/summary.rs index 9c18cb6e2..024d42f01 100644 --- a/crates/switchyard-menubar/src/summary.rs +++ b/crates/switchyard-menubar/src/summary.rs @@ -51,7 +51,9 @@ pub fn build(status: ServerStatus, usage: &Usage, config: &Config) -> Vec { if estimate(&usage.week, &config.prices, &config.baseline_model).is_none() { rows.push(Row::Separator); - rows.push(label("Add prices to menubar.toml to see savings")); + rows.push(label( + "Savings need complete recorded costs and baseline rates in prices.json", + )); } rows } @@ -64,7 +66,7 @@ fn period(name: &str, totals: &Totals, config: &Config) -> Vec { tokens(totals.tokens()) ))]; if let Some(savings) = estimate(totals, &config.prices, &config.baseline_model) { - rows.push(label(format!(" Saved {}", saved(&savings)))); + rows.push(label(format!(" Estimated savings {}", saved(&savings)))); } rows } @@ -123,8 +125,8 @@ fn money(value: f64) -> String { #[cfg(test)] mod tests { use super::*; - use crate::pricing::{ModelPrice, PriceTable}; use crate::rollup::ModelTokens; + use switchyard_costs::{CostSource, Pricing, RecordedCost}; fn counted(input: u64, output: u64, requests: u64) -> ModelTokens { ModelTokens { @@ -132,10 +134,11 @@ mod tests { input, cached_input: 0, output, + ..ModelTokens::default() } } - fn config(prices: PriceTable) -> Config { + fn config(prices: Pricing) -> Config { Config { baseline_model: "sol".to_string(), prices, @@ -143,25 +146,8 @@ mod tests { } } - fn priced() -> PriceTable { - PriceTable::from([ - ( - "sol".to_string(), - ModelPrice { - input_per_mtok: 1.25, - cached_input_per_mtok: None, - output_per_mtok: 10.0, - }, - ), - ( - "luna".to_string(), - ModelPrice { - input_per_mtok: 0.25, - cached_input_per_mtok: None, - output_per_mtok: 2.0, - }, - ), - ]) + fn priced() -> Pricing { + Pricing::parse(br#"{"sol":{"input":1.25,"cache_read":1.25,"cache_write":1.25,"output":10},"luna":{"input":0.25,"cache_read":0.25,"cache_write":0.25,"output":2}}"#).expect("rates") } fn labels(rows: &[Row]) -> Vec { @@ -187,6 +173,14 @@ mod tests { week.routed .insert("luna".to_string(), counted(4_000_000, 400_000, 12)); + today.cost.add(Some(RecordedCost { + usd: 0.45, + source: CostSource::Estimated, + })); + week.cost.add(Some(RecordedCost { + usd: 1.80, + source: CostSource::Estimated, + })); let rows = build( ServerStatus::Running, &usage_of(today, week), @@ -196,9 +190,9 @@ mod tests { assert_eq!(labels[0], "Server: running · http://127.0.0.1:4123"); assert_eq!(labels[1], "Today — 3 requests · 1.1M tokens"); - assert_eq!(labels[2], " Saved $1.80 (80% of $2.25)"); + assert_eq!(labels[2], " Estimated savings $1.80 (80% of $2.25)"); assert_eq!(labels[3], "This week — 12 requests · 4.4M tokens"); - assert_eq!(labels[4], " Saved $7.20 (80% of $9.00)"); + assert_eq!(labels[4], " Estimated savings $7.20 (80% of $9.00)"); } #[test] @@ -209,19 +203,20 @@ mod tests { week.classifier .insert("sol".to_string(), counted(100, 0, 1)); - for prices in [ - PriceTable::new(), - PriceTable::from([("sol".to_string(), priced()["sol"])]), - PriceTable::from([("luna".to_string(), priced()["luna"])]), - ] { + week.cost.add(None); + for prices in [Pricing::default(), priced()] { let rows = build( ServerStatus::Running, &usage_of(week.clone(), week.clone()), &config(prices), ); let labels = labels(&rows); - assert!(labels.iter().all(|label| !label.contains("Saved"))); - assert!(labels.iter().any(|label| label.contains("menubar.toml"))); + assert!( + labels + .iter() + .all(|label| !label.contains("Estimated savings")) + ); + assert!(labels.iter().any(|label| label.contains("costs"))); } } @@ -263,13 +258,17 @@ mod tests { .classifier .insert("sol".to_string(), counted(1_000_000, 0, 1)); + today.cost.add(Some(RecordedCost { + usd: 1.25125, + source: CostSource::Estimated, + })); let rows = build( ServerStatus::Running, &usage_of(today.clone(), today), &config(priced()), ); - assert!(labels(&rows)[2].starts_with(" Saved -$1.25")); + assert!(labels(&rows)[2].starts_with(" Estimated savings -$1.25")); } #[test] diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml index 2ef504d7a..3601dd083 100644 --- a/crates/switchyard-server/Cargo.toml +++ b/crates/switchyard-server/Cargo.toml @@ -20,6 +20,7 @@ publish = ["crates-io"] prefill-router = ["switchyard-runner/prefill-router"] [dependencies] +switchyard-costs.workspace = true async-stream.workspace = true async-trait.workspace = true axum = "0.8" diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 6402eec6c..aa454a460 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -137,7 +137,8 @@ keeps the priced subtotal and `total_usd` is null when any cost is unknown. Rate at startup, and saved costs do not change when prices change. The [pi guide](../../docs/integrations/pi.md#estimate-the-cost) explains how to supply -rates for your own models. +rates for your own models. The menubar reads the same file through its `pricing_file` +setting; its actual spending comes from recorded costs. Clients can send `x-switchyard-origin: codex-cli` (or another client label) to include an `origin` field in each routing record. Missing, empty, or non-text header values produce diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 9110bf596..4fed60a30 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -8,7 +8,6 @@ pub mod config; mod metrics; mod observability; mod observed_translation; -mod pricing; mod redaction; mod response; mod routing_log; @@ -240,7 +239,7 @@ impl ServerState { /// This method loads rates in USD per million tokens for saved routing records. pub fn with_pricing_file(self, path: impl AsRef) -> ServerResult { - let prices = pricing::Pricing::load(path.as_ref())?; + let prices = switchyard_costs::Pricing::load(path.as_ref()).map_err(ServerError::new)?; let log = self .routing_log .as_ref() diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs index a91586513..01dddfb6b 100644 --- a/crates/switchyard-server/src/routing_log.rs +++ b/crates/switchyard-server/src/routing_log.rs @@ -14,9 +14,9 @@ use humantime::format_rfc3339_millis; use serde::{Deserialize, Serialize}; use switchyard_protocol::{Metadata, ModelId, Usage}; -use crate::pricing::{CostTotals, Pricing, RecordedCost}; use crate::usage_metrics::token_usage; use crate::{ServerError, ServerResult}; +use switchyard_costs::{CostTotals, Pricing, RecordedCost}; const LEGACY_SESSION_ID_HEADER: &str = "proxy_x_session_id"; const ORIGIN_HEADER: &str = "x-switchyard-origin"; diff --git a/scripts/macos/install.sh b/scripts/macos/install.sh index ae7b3a9dd..f3cff8135 100755 --- a/scripts/macos/install.sh +++ b/scripts/macos/install.sh @@ -55,8 +55,8 @@ step "Writing server config" write_once "$SY_HOME/composite.toml" < "$REPO_ROOT/scripts/config/composite.toml" step "Writing menu bar settings" -# Rates are list prices per million tokens and are only used to estimate -# savings. Edit them to match what you actually pay. +# The server and menubar share rates in USD per million tokens. +# Edit prices.json to match what you actually pay. TOML_SY_HOME="$(toml_escape_basic_string "$SY_HOME")" write_once "$SY_HOME/menubar.toml" <$SY_PORT --routing-log-file $XML_SY_HOME/routing.jsonl + --pricing-file + $XML_SY_HOME/prices.json RunAtLoad