diff --git a/Cargo.lock b/Cargo.lock index 705b86d082..19bf434ba5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3640,6 +3640,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "switchyard-costs" +version = "0.3.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "switchyard-protocol", + "toml 0.9.12+spec-1.1.0", +] + [[package]] name = "switchyard-desktop" version = "0.3.0" @@ -3723,6 +3734,8 @@ dependencies = [ "objc2-foundation", "serde", "serde_json", + "switchyard-costs", + "switchyard-protocol", "tempfile", "toml 0.9.12+spec-1.1.0", "tray-icon", @@ -3859,6 +3872,7 @@ dependencies = [ "rustls", "serde", "serde_json", + "switchyard-costs", "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", diff --git a/Cargo.toml b/Cargo.toml index e13c41b0d5..60228cabdf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/switchyard-runner", "crates/switchyard-desktop", "crates/switchyard-menubar", + "crates/switchyard-costs", "crates/switchyard-server", "crates/switchyard-skill-distillation", "crates/switchyard-soak", @@ -49,6 +50,7 @@ regex = "1" reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } +switchyard-costs = { path = "crates/switchyard-costs", version = "0.3.0" } switchyard-libsy = { path = "crates/libsy", version = "0.3.0" } switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.3.0" } switchyard-protocol = { path = "crates/protocol", version = "0.3.0" } diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 4c3d82a823..3e4fc675ae 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -660,7 +660,7 @@ impl TranslatingLlmClient { })?; // Map a provider's failed generation to 502, even under HTTP 200. // Redact forwarded credentials before returning the provider error. - let agg = + let mut agg = observed_translation::decode_response(&body, wire_format).map_err(|error| { match error { TranslationError::UpstreamFailure { error } => { @@ -676,6 +676,14 @@ impl TranslatingLlmClient { error => LlmClientError::ResponseTranslation(error.to_string()), } })?; + // The gateway may report a buffered charge only in its response header. + if agg.usage.provider_cost.is_none() { + agg.usage.provider_cost = upstream_headers + .get("x-litellm-response-cost") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .filter(|usd| usd.is_finite() && *usd >= 0.0); + } (LlmResponse::Agg(agg), upstream_headers) } }; diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 00bdf76c87..661e05c6ba 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -33,7 +33,7 @@ use switchyard_libsy::{ use switchyard_protocol::{ AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message, ModelId, Request, Response, ResponseAccumulator, RoutedDecisionClient, RoutedLlmClient, - RoutingFallbackReason, WireFormat, + RoutingFallbackReason, Usage, WireFormat, }; use switchyard_translation::prepare_request_for_target; @@ -376,10 +376,13 @@ async fn call_one( // the provider's, so it belongs in the routing overhead. let client = clients.route(model_id); let started = Instant::now(); + let mut routing_usage = None; let result = async { let mut response = client?.call(request).await?; if is_routing && let LlmResponse::Stream(chunks) = response.llm_response { - response.llm_response = LlmResponse::Stream(buffer_routing_stream(chunks).await?); + let (chunks, usage) = buffer_routing_stream(chunks).await?; + routing_usage = usage; + response.llm_response = LlmResponse::Stream(chunks); } Ok(response) } @@ -405,19 +408,22 @@ async fn call_one( .as_ref() .ok() .and_then(|response| response.llm_response.as_agg()) - .map(|response| response.usage.clone()), + .map(|response| response.usage.clone()) + .or(routing_usage), }); result } -// Preserve signed provider events while checking the complete routing response. +// Routing logs need the final usage, but replay must keep signed provider events intact. async fn buffer_routing_stream( mut chunks: LlmResponseStream, -) -> std::result::Result { +) -> std::result::Result<(LlmResponseStream, Option), LlmClientError> { let mut events = Vec::new(); + let mut usage = None; while let Some(event) = chunks.try_next().await? { for chunk in event.normalized() { match chunk { + LlmResponseChunk::Usage(latest) => usage = Some(latest.clone()), LlmResponseChunk::DecodeError { message } => { return Err(LlmClientError::ResponseTranslation(message.clone())); } @@ -432,7 +438,7 @@ async fn buffer_routing_stream( } events.push(event); } - Ok(stream::iter(events.into_iter().map(Ok)).boxed()) + Ok((stream::iter(events.into_iter().map(Ok)).boxed(), usage)) } /// Whether a failed candidate is worth routing around. diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 27a5f38622..ec16a2c0fe 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -996,6 +996,7 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya total_tokens: Some(25), reasoning_tokens: Some(2), cache: Usage::cache_details(Some(3), Some(4)), + provider_cost: None, }, }) as Arc; let mut request = request_with_metadata("obs-session-1", "obs-corr-1"); diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index 9429354e5c..cb3731aef4 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -559,13 +559,16 @@ pub struct LlmRequest { /// Normalized token usage counts. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Usage { + /// The upstream provider reports this monetary charge in USD. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_cost: Option, /// Non-cached input tokens. Provider codecs normalize aggregate OpenAI /// input counts by subtracting the cache detail fields. pub input_tokens: Option, /// Cache-read and cache-creation token detail, when reported. #[serde(flatten)] pub cache: Option>, - /// Generated output tokens, excluding reasoning detail when the provider reports it separately. + /// The output count includes reasoning tokens when the provider counts them. pub output_tokens: Option, /// Provider-reported or codec-computed total token count. /// @@ -579,6 +582,9 @@ pub struct Usage { /// Optional cache-token detail kept out of the common, cache-free usage allocation. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct InputCacheUsage { + /// The provider reports these cache-write tokens at the one-hour rate. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_1h_input_tokens: Option, /// Input tokens read from a provider cache. pub cached_input_tokens: Option, /// Input tokens written into a provider cache. @@ -595,6 +601,7 @@ impl Usage { return None; } Some(Box::new(InputCacheUsage { + cache_creation_1h_input_tokens: None, cached_input_tokens, cache_creation_input_tokens, })) diff --git a/crates/switchyard-costs/Cargo.toml b/crates/switchyard-costs/Cargo.toml new file mode 100644 index 0000000000..7176256d84 --- /dev/null +++ b/crates/switchyard-costs/Cargo.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-costs" +version.workspace = true +description = "Shared token pricing and recorded cost totals for Switchyard" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +chrono = { version = "0.4", default-features = false, features = ["std"] } +switchyard-protocol.workspace = true +serde.workspace = true +serde_json.workspace = true +toml = "0.9" diff --git a/crates/switchyard-costs/src/anthropic.rs b/crates/switchyard-costs/src/anthropic.rs new file mode 100644 index 0000000000..99b7601429 --- /dev/null +++ b/crates/switchyard-costs/src/anthropic.rs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Anthropic prices cache writes at separate rates for five-minute and one-hour lifetimes. + +use switchyard_protocol::Usage; + +/// The caller divides this weighted token sum by one million to obtain USD. +pub(super) fn cache_write_cost( + usage: &Usage, + short_rate: f64, + long_rate: Option, +) -> Option { + let written = usage.cache_creation_input_tokens().unwrap_or(0); + let one_hour = usage + .cache + .as_ref() + .and_then(|cache| cache.cache_creation_1h_input_tokens) + .unwrap_or(0); + let five_minute = written.checked_sub(one_hour)?; + let long_cost = if one_hour > 0 { + one_hour as f64 * long_rate? + } else { + 0.0 + }; + Some(five_minute as f64 * short_rate + long_cost) +} diff --git a/crates/switchyard-costs/src/lib.rs b/crates/switchyard-costs/src/lib.rs new file mode 100644 index 0000000000..a0001ea194 --- /dev/null +++ b/crates/switchyard-costs/src/lib.rs @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! This module prices observed calls and keeps missing usage or rates unknown. + +mod anthropic; + +use std::collections::HashMap; +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use switchyard_protocol::Usage; + +/// Pricing maps served model IDs to token rates in USD per million tokens. +#[derive(Clone, Debug, Default)] +pub struct Pricing(HashMap); + +impl<'de> Deserialize<'de> for Pricing { + fn deserialize>(deserializer: D) -> Result { + // This visitor rejects duplicate IDs before a map can overwrite a rate table. + struct UniqueModels; + impl<'de> serde::de::Visitor<'de> for UniqueModels { + type Value = Pricing; + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a map of unique model IDs to token rates") + } + fn visit_map>( + self, + mut map: M, + ) -> Result { + let mut rates = HashMap::new(); + while let Some((model, rate)) = map.next_entry::()? { + if rates.insert(model.clone(), rate).is_some() { + return Err(serde::de::Error::custom(format!( + "duplicate pricing model {model:?}" + ))); + } + } + Ok(Pricing(rates)) + } + } + deserializer.deserialize_map(UniqueModels) + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Rates { + input: f64, + output: f64, + cache_read: f64, + cache_write: f64, + cache_write_1h: Option, +} + +/// This value identifies how a saved monetary amount was obtained. +#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CostSource { + /// The provider supplied this charge. + ProviderReported, + /// Configured token rates supplied this estimate. + Estimated, +} + +/// This record stores a charge or estimate for one call. +#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)] +pub struct RecordedCost { + /// The amount is in USD. + pub usd: f64, + /// The source distinguishes charges from estimates. + pub source: CostSource, +} + +impl Pricing { + /// This method loads and validates a JSON pricing file. + pub fn load(path: &Path) -> Result { + let bytes = std::fs::read(path) + .map_err(|error| format!("read pricing file {}: {error}", path.display()))?; + Self::parse(&bytes) + .map_err(|error| format!("invalid pricing file {}: {error}", path.display())) + } + + /// This function reads a pricing file's JSON and checks every model ID and rate. + pub fn parse(json: &[u8]) -> Result { + let pricing: Self = serde_json::from_slice(json).map_err(|error| error.to_string())?; + for (model, rate) in &pricing.0 { + if model.trim().is_empty() + || model.trim() != model + || [ + Some(rate.input), + Some(rate.output), + Some(rate.cache_read), + Some(rate.cache_write), + rate.cache_write_1h, + ] + .into_iter() + .flatten() + .any(|value| !value.is_finite() || !(0.0..=10000.0).contains(&value)) + { + return Err(format!( + "invalid pricing for model {model:?}: use a nonempty model ID and rates from 0 to 10000 USD per million tokens" + )); + } + } + Ok(pricing) + } + + /// Provider charges take precedence because token rates may omit provider surcharges. + pub fn cost(&self, model: &str, usage: &Usage) -> Option { + if let Some(usd) = usage + .provider_cost + .filter(|value| value.is_finite() && *value >= 0.0) + { + return Some(RecordedCost { + usd, + source: CostSource::ProviderReported, + }); + } + self.estimate(model, usage) + } + + /// This method uses token rates alone so baseline estimates exclude provider charges. + pub fn estimate(&self, model: &str, usage: &Usage) -> Option { + let rate = self.0.get(model)?; + let input = usage.input_tokens?; + let output = usage.output_tokens?; + let read = usage.cached_input_tokens().unwrap_or(0); + let usd = (input as f64 * rate.input + + output as f64 * rate.output + + read as f64 * rate.cache_read + + anthropic::cache_write_cost(usage, rate.cache_write, rate.cache_write_1h)?) + / 1_000_000.0; + Some(RecordedCost { + usd, + source: CostSource::Estimated, + }) + } +} + +/// The known subtotal remains visible when any call has unknown cost. +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub struct CostTotals { + /// This subtotal includes every known cost. + pub known_usd: f64, + /// This count includes calls whose cost is unknown. + pub unknown_calls: u64, + /// This count includes calls priced from configured rates. + pub estimated_calls: u64, + /// This count includes calls with a provider charge. + pub provider_reported_calls: u64, + /// The total is absent when any call has unknown cost. + pub total_usd: Option, +} + +impl Default for CostTotals { + fn default() -> Self { + Self { + known_usd: 0.0, + unknown_calls: 0, + estimated_calls: 0, + provider_reported_calls: 0, + total_usd: Some(0.0), + } + } +} + +impl CostTotals { + /// This method adds one call and keeps the total unknown if any cost is unknown. + pub fn add(&mut self, cost: Option) { + match cost.filter(|cost| { + cost.usd.is_finite() && cost.usd >= 0.0 && (self.known_usd + cost.usd).is_finite() + }) { + Some(cost) => { + self.known_usd += cost.usd; + match cost.source { + CostSource::Estimated => self.estimated_calls += 1, + CostSource::ProviderReported => self.provider_reported_calls += 1, + } + } + None => self.unknown_calls += 1, + } + self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); + } + + /// This method combines disjoint groups of calls without repricing them. + pub fn merge(&mut self, other: &Self) { + if (self.known_usd + other.known_usd).is_finite() { + self.known_usd += other.known_usd; + } else { + self.unknown_calls += 1; + } + self.unknown_calls += other.unknown_calls; + self.estimated_calls += other.estimated_calls; + self.provider_reported_calls += other.provider_reported_calls; + self.total_usd = (self.unknown_calls == 0).then_some(self.known_usd); + } +} diff --git a/crates/switchyard-costs/src/main.rs b/crates/switchyard-costs/src/main.rs new file mode 100644 index 0000000000..dc3f7b7f4e --- /dev/null +++ b/crates/switchyard-costs/src/main.rs @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! This command migrates old settings and logs without changing the originals. + +use std::io::{BufRead, Write}; +use std::path::Path; + +use serde::Deserialize; +use serde_json::{Value, json}; +use switchyard_costs::{Pricing, RecordedCost}; +use switchyard_protocol::Usage; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct OldRate { + input_per_mtok: f64, + output_per_mtok: f64, + cached_input_per_mtok: Option, +} + +#[derive(Deserialize)] +struct Record { + ts: String, + #[serde(default, rename = "tier")] + _tier: String, + model: String, + prompt_tokens: Option, + completion_tokens: Option, + cached_tokens: Option, + cache_creation_tokens: Option, + cache_creation_1h_tokens: Option, + // The nested option distinguishes an absent cost from an explicit unknown cost. + #[serde(default, deserialize_with = "present_cost")] + cost: Option>, +} + +// An explicit null remains unknown instead of being reconstructed from token rates. +fn present_cost<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result>, D::Error> { + Option::deserialize(deserializer).map(Some) +} + +// Only records without a cost field receive reconstructed estimates. +fn migrate( + settings: &Path, + log: &Path, + destination: &Path, +) -> Result<(), Box> { + let mut config: toml::Table = std::fs::read_to_string(settings)?.parse()?; + let rates = match config.remove("prices") { + Some(old) => { + if config.contains_key("pricing_file") { + return Err("settings contain both prices and pricing_file".into()); + } + let old: std::collections::BTreeMap = old.try_into()?; + Value::Object(old.into_iter().map(|(model, rate)| (model, json!({ + "input":rate.input_per_mtok, "output":rate.output_per_mtok, + "cache_read":rate.cached_input_per_mtok.unwrap_or(rate.input_per_mtok), + "cache_write":rate.input_per_mtok, + }))).collect()) + } + None => return Err("migration requires the old settings with a [prices] table".into()), + }; + let rates = serde_json::to_vec_pretty(&rates)?; + let pricing = Pricing::parse(&rates)?; + let destination = std::path::absolute(destination)?; + config.insert( + "pricing_file".into(), + toml::Value::String( + destination + .join("prices.json") + .to_string_lossy() + .into_owned(), + ), + ); + config.insert( + "routing_log".into(), + toml::Value::String( + destination + .join("routing.jsonl") + .to_string_lossy() + .into_owned(), + ), + ); + // Creating a fresh directory prevents overwriting inputs or earlier migration results. + std::fs::create_dir(&destination)?; + let result = (|| -> Result<(), Box> { + let mut output = + std::io::BufWriter::new(std::fs::File::create(destination.join("routing.jsonl"))?); + let mut source = std::io::BufReader::new(std::fs::File::open(log)?); + let mut line = String::new(); + loop { + line.clear(); + if source.read_line(&mut line)? == 0 { + break; + } + if !line.ends_with('\n') { + return Err("unfinished log record; stop the writer before migrating".into()); + } + // Typed parsing catches duplicate known fields before Value could overwrite them. + let record: Record = serde_json::from_str(&line)?; + chrono::DateTime::parse_from_rfc3339(&record.ts)?; + if record.model.trim().is_empty() { + return Err("log record has an empty model".into()); + } + if let Some(Some(cost)) = record.cost + && (!cost.usd.is_finite() || cost.usd < 0.0) + { + return Err("log record has an invalid cost".into()); + } + let mut value: Value = serde_json::from_str(&line)?; + if !value.is_object() { + return Err("log record must be a JSON object".into()); + } + if record.cost.is_none() { + let read = record.cached_tokens.unwrap_or(0); + let written = record.cache_creation_tokens.unwrap_or(0); + let mut cache = Usage::cache_details(Some(read), Some(written)); + if let Some(tokens) = record.cache_creation_1h_tokens { + cache + .get_or_insert_with(Default::default) + .cache_creation_1h_input_tokens = Some(tokens); + } + let usage = Usage { + input_tokens: record + .prompt_tokens + .and_then(|input| input.checked_sub(read)?.checked_sub(written)), + output_tokens: record.completion_tokens, + cache, + ..Usage::default() + }; + value["cost"] = serde_json::to_value(pricing.estimate(&record.model, &usage))?; + } + serde_json::to_writer(&mut output, &value)?; + output.write_all(b"\n")?; + } + output.flush()?; + std::fs::write(destination.join("prices.json"), rates)?; + std::fs::write( + destination.join("menubar.toml"), + toml::to_string_pretty(&config)?, + )?; + Ok(()) + })(); + if result.is_err() { + std::fs::remove_dir_all(&destination)?; + } + result +} + +fn main() -> std::process::ExitCode { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if args.len() != 3 { + eprintln!( + "Usage: switchyard-costs OLD_MENUBAR_TOML ROUTING_JSONL NEW_DIRECTORY\nStop the server and menubar first. Outputs use the rates in the old settings." + ); + return std::process::ExitCode::FAILURE; + } + match migrate( + Path::new(&args[0]), + Path::new(&args[1]), + Path::new(&args[2]), + ) { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(error) => { + eprintln!("switchyard-costs: {error}"); + std::process::ExitCode::FAILURE + } + } +} diff --git a/crates/switchyard-menubar/Cargo.toml b/crates/switchyard-menubar/Cargo.toml index d12551f0d7..b839ad9c97 100644 --- a/crates/switchyard-menubar/Cargo.toml +++ b/crates/switchyard-menubar/Cargo.toml @@ -14,6 +14,8 @@ rust-version.workspace = true publish = false [dependencies] +switchyard-costs.workspace = true +switchyard-protocol.workspace = true chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } serde.workspace = true serde_json.workspace = true diff --git a/crates/switchyard-menubar/README.md b/crates/switchyard-menubar/README.md index 4cc99d2083..8525a5878e 100644 --- a/crates/switchyard-menubar/README.md +++ b/crates/switchyard-menubar/README.md @@ -36,55 +36,69 @@ routing_log = "~/.switchyard/routing.jsonl" baseline_model = "gpt-5.6-sol" refresh_seconds = 30 -[prices."gpt-5.6-sol"] -input_per_mtok = 1.25 -cached_input_per_mtok = 0.125 -output_per_mtok = 10.0 +pricing_file = "~/.switchyard/prices.json" ``` +The server's `--pricing-file` and the menubar's `pricing_file` use the same JSON file. +Key each entry by the model ID in the routing log. Rates are in USD per million tokens: + +```json +{"gpt-5.6-sol":{"input":1.25,"cache_read":0.125,"cache_write":1.25,"output":10}} +``` + +These example rates preserve the installer's previous assumptions. Replace them with +what you pay. The [pi guide](../../docs/integrations/pi.md#estimate-the-cost) explains +cache-write rates and how to add other models. Rates load at startup; restart both +applications after editing them. + ## How savings are computed -The server writes one JSONL record per call, with the model that answered and -its token counts. - -- **Actual** is every call priced at the model that served it, including - Switchyard's own classifier calls. -- **Baseline** is the caller-facing calls only, priced as if each had used - `baseline_model`. Classifier calls have no baseline counterpart, because - without Switchyard they would not happen. - -Savings are the difference, so routing overhead counts against the figure and -a bad day shows a negative number. Dollar figures stay hidden until every -model seen has a price, so a partial table cannot mislead. - -Codex's "Approve for me" reviews count too. The server config that the -installer writes sends them to `codex-auto-review` on the ChatGPT backend. With -a ChatGPT login, Codex sends these reviews to `codex-auto-review` even without -Switchyard, so the installer prices that model at the `baseline_model` rates. -Each review then adds the same amount to the actual cost and to the baseline, -so reviews do not change the dollar amount saved. They do lower the percentage -a little, because the baseline grows. If you change `baseline_model`, update -this price to match. - -The menu bar needs a price for every model ID that the routing log records. -The macOS installer includes prices for the default models, including -`gpt-6-luna`, `gpt-6-sol`, and `gpt-6.1-sol`. Existing `menubar.toml` files are -preserved during reinstall, so add any missing model prices to that file by -hand. - -Codex's "Approve for me" reviews need a price too. Use the reviewer target's -`id`, which is the model ID recorded in the routing log. The default reviewer -route uses `codex-auto-review`. With a ChatGPT login, these reviews also happen -without Switchyard. Add that ID to the price table at the `baseline_model` -rates, including the cached input rate. Each review then adds the same cost to -actual and baseline. Dollars saved stay the same; the percentage drops as the -baseline grows. Update this price when you change `baseline_model`. - -If a reviewer uses another model ID, add a price for that ID. Any period with -an unpriced model hides its savings and shows the missing-price hint. - -These are list-price estimates. On a ChatGPT login there is no per-token bill, -so read them as "what this traffic would have cost at API rates". +Actual spending is the sum of the costs saved in the routing log, including routing +calls. Changing prices does not change saved costs. The baseline is an estimate of +what the answer traffic would have cost using `baseline_model`; it excludes routing +calls. The server and menubar share the code for cost totals and token pricing, including +cache-read and cache-write rates. +The baseline assumes the same token and cache counts as the observed answer traffic. + +The menubar shows estimated savings only when actual costs and the baseline estimate +are complete. A recorded `cost: null` keeps spending unknown. A missing baseline rate +or required one-hour cache-write rate keeps the baseline unknown. Missing or invalid answer +usage also keeps the baseline unknown. Token counts still +appear when savings cannot be calculated. The log can be read while the server is offline. + +Codex's `codex-auto-review` model keeps the installer's baseline rates by default. +Update that entry when changing `baseline_model` if you want those reviews to have +the same estimated cost with and without Switchyard. ChatGPT subscription traffic +may have no per-token bill; token-rate estimates are comparisons at the supplied rates. + +## Migrate older settings and logs + +Inline `[prices]` settings and log records without a `cost` field require migration. +Stop the server and menubar before migrating. From the repository root, run: + +```sh +cargo run -p switchyard-costs -- \ + ~/.switchyard/menubar.toml ~/.switchyard/routing.jsonl ~/.switchyard/migrated-costs +``` + +The destination directory must not exist. The command writes `menubar.toml`, +`prices.json`, and `routing.jsonl` there and leaves the originals untouched. It preserves +existing costs, including null, and adds estimates only to records without a cost field. +Missing usage or rates leave the reconstructed cost unknown. Malformed or unfinished +records stop migration; failed migration outputs are removed. + +The command uses the rates in the old settings. It cannot recover historical provider +charges or historical prices. It maps `input_per_mtok` to `input` and `cache_write`, +`output_per_mtok` to `output`, and `cached_input_per_mtok` to `cache_read` (or the input +rate when omitted). This preserves the old cache-write assumption. Review those rates +for your providers; add `cache_write_1h` when needed. Reconstruction is always marked +`estimated`, even when it uses historical prices. + +Review the generated files, then start the menubar with the generated settings path. +Those settings point at the generated log and pricing file. Update the server's +LaunchAgent to use those same paths for `--routing-log-file` and `--pricing-file` before +restarting it. Keep the originals as backups. Reinstalling preserves existing files; +it does not migrate them. ## Codex profiles diff --git a/crates/switchyard-menubar/src/app.rs b/crates/switchyard-menubar/src/app.rs index 158411f40d..3695ce9b0f 100644 --- a/crates/switchyard-menubar/src/app.rs +++ b/crates/switchyard-menubar/src/app.rs @@ -12,15 +12,10 @@ use crate::summary::{Row, build}; /// Reads new log entries, probes the server, and builds the menu body. pub fn refresh(config: &Config, log: &mut rollup::Reader) -> Vec { - let usage = log - .read(&config.routing_log, Local::now().date_naive()) - .unwrap_or_else(|error| { - eprintln!( - "switchyard-menubar: read {}: {error}", - config.routing_log.display() - ); - rollup::Usage::default() - }); + let usage = match log.read(&config.routing_log, Local::now().date_naive()) { + Ok(usage) => usage, + Err(error) => return vec![Row::Label(format!("Cannot read routing log: {error}"))], + }; build(probe(&config.server_url), &usage, config) } @@ -36,7 +31,7 @@ mod tests { std::fs::write( &path, format!( - r#"{{"ts":"{ts}","model":"luna","tier":"","prompt_tokens":1000,"cached_tokens":0,"completion_tokens":100,"reasoning_tokens":0}}"# + r#"{{"ts":"{ts}","model":"luna","tier":"","prompt_tokens":1000,"cached_tokens":0,"completion_tokens":100,"reasoning_tokens":0,"cost":null}}"# ) + "\n", ) .expect("write log"); diff --git a/crates/switchyard-menubar/src/config.rs b/crates/switchyard-menubar/src/config.rs index 8b29b4b6ac..cebc785bf3 100644 --- a/crates/switchyard-menubar/src/config.rs +++ b/crates/switchyard-menubar/src/config.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; use serde::Deserialize; -use crate::pricing::PriceTable; +use switchyard_costs::Pricing; /// Everything the menu bar needs to find the server and price its traffic. /// Missing keys fall back to [`Config::default`]. @@ -26,8 +26,11 @@ pub struct Config { pub refresh_seconds: u64, /// Model the traffic is assumed to have used without Switchyard. pub baseline_model: String, - /// Per-model rates. Dollar figures are hidden while a seen model is absent. - pub prices: PriceTable, + /// The shared JSON file supplies rates for the baseline estimate. + pub pricing_file: Option, + /// Rates load once when the settings are read. + #[serde(skip)] + pub prices: Pricing, } impl Default for Config { @@ -39,7 +42,8 @@ impl Default for Config { launchd_label: "com.nvidia.switchyard.server".to_string(), refresh_seconds: 30, baseline_model: "gpt-5.6-sol".to_string(), - prices: PriceTable::new(), + pricing_file: None, + prices: Pricing::default(), } } } @@ -53,6 +57,11 @@ impl Config { Err(error) if error.kind() == std::io::ErrorKind::NotFound => Self::default(), Err(error) => return Err(format!("read {}: {error}", path.display())), }; + if let Some(path) = config.pricing_file.as_ref() { + let path = expand_home(path); + config.prices = Pricing::load(&path)?; + config.pricing_file = Some(path); + } config.routing_log = expand_home(&config.routing_log); config.config_file = expand_home(&config.config_file); Ok(config) @@ -100,10 +109,6 @@ mod tests { server_url = "http://127.0.0.1:9000" baseline_model = "sol" -[prices.luna] -input_per_mtok = 0.25 -cached_input_per_mtok = 0.025 -output_per_mtok = 2.0 "#, ) .expect("write settings"); @@ -112,7 +117,7 @@ output_per_mtok = 2.0 assert_eq!(config.server_url, "http://127.0.0.1:9000"); assert_eq!(config.baseline_model, "sol"); - assert_eq!(config.prices["luna"].cached_input_per_mtok, Some(0.025)); + assert!(config.pricing_file.is_none()); assert_eq!(config.refresh_seconds, 30, "unset keys keep their default"); } @@ -123,7 +128,7 @@ output_per_mtok = 2.0 let config = Config::load(&dir.path().join("absent.toml")).expect("load"); assert_eq!(config.server_url, "http://127.0.0.1:4123"); - assert!(config.prices.is_empty()); + assert!(config.pricing_file.is_none()); let empty = dir.path().join("empty.toml"); std::fs::write(&empty, "").expect("write settings"); let loaded = Config::load(&empty).expect("load empty settings"); diff --git a/crates/switchyard-menubar/src/pricing.rs b/crates/switchyard-menubar/src/pricing.rs index e551bdd719..e2cf1c98d2 100644 --- a/crates/switchyard-menubar/src/pricing.rs +++ b/crates/switchyard-menubar/src/pricing.rs @@ -4,37 +4,10 @@ //! Counterfactual cost: what routed traffic cost against what it would have //! cost had every call gone to one capable model. -use std::collections::BTreeMap; +use switchyard_costs::{CostTotals, Pricing}; +use switchyard_protocol::{InputCacheUsage, Usage}; -use serde::Deserialize; - -use crate::rollup::{ModelTokens, Totals}; - -const PER_MILLION: f64 = 1_000_000.0; - -/// Per-million-token rates for one model. -#[derive(Clone, Copy, Debug, Deserialize, PartialEq)] -pub struct ModelPrice { - pub input_per_mtok: f64, - /// Rate for cache reads. Falls back to the full input rate. - #[serde(default)] - pub cached_input_per_mtok: Option, - /// Rate for generated tokens, reasoning included. - pub output_per_mtok: f64, -} - -impl ModelPrice { - fn cost(&self, tokens: &ModelTokens) -> f64 { - let cached_rate = self.cached_input_per_mtok.unwrap_or(self.input_per_mtok); - (tokens.input as f64 * self.input_per_mtok - + tokens.cached_input as f64 * cached_rate - + tokens.output as f64 * self.output_per_mtok) - / PER_MILLION - } -} - -/// Rates keyed by the model id the server records in the routing log. -pub type PriceTable = BTreeMap; +use crate::rollup::Totals; /// What the traffic cost, and what it would have cost without routing. #[derive(Clone, Copy, Debug, PartialEq)] @@ -59,44 +32,52 @@ impl Savings { /// Prices a period against the baseline model. /// -/// Returns `None` when any model seen has no entry in the table, since a -/// partial bill would understate cost and overstate savings. -pub fn estimate(totals: &Totals, prices: &PriceTable, baseline_model: &str) -> Option { - let baseline_price = prices.get(baseline_model)?; - - let mut actual = 0.0; - let mut baseline = 0.0; - for (model, tokens) in &totals.routed { - actual += prices.get(model)?.cost(tokens); - baseline += baseline_price.cost(tokens); +/// Returns `None` when recorded spending or the baseline estimate is incomplete. +pub fn estimate(totals: &Totals, prices: &Pricing, baseline_model: &str) -> Option { + if totals.baseline_unknown { + return None; } - // Classifier calls are Switchyard's own cost. They have no baseline - // counterpart, so they only ever reduce the savings figure. - for (model, tokens) in &totals.classifier { - actual += prices.get(model)?.cost(tokens); + let mut baseline = CostTotals::default(); + for tokens in totals.routed.values() { + let usage = Usage { + input_tokens: Some(tokens.input), + output_tokens: Some(tokens.output), + cache: Some(Box::new(InputCacheUsage { + cached_input_tokens: Some(tokens.cached_input), + cache_creation_input_tokens: Some(tokens.cache_write), + cache_creation_1h_input_tokens: Some(tokens.cache_write_1h), + })), + ..Usage::default() + }; + baseline.add(prices.estimate(baseline_model, &usage)); } - - Some(Savings { actual, baseline }) + Some(Savings { + actual: totals.cost.total_usd?, + baseline: baseline.total_usd?, + }) } #[cfg(test)] mod tests { use super::*; - - fn price(input: f64, output: f64) -> ModelPrice { - ModelPrice { - input_per_mtok: input, - cached_input_per_mtok: None, - output_per_mtok: output, - } + use crate::rollup::ModelTokens; + + fn table() -> Pricing { + Pricing::parse( + br#"{ + "sol":{"input":1.25,"cache_read":1.25,"cache_write":1.25,"output":10}, + "luna":{"input":0.25,"cache_read":0.25,"cache_write":0.25,"output":2}, + "terra":{"input":0.05,"cache_read":0.05,"cache_write":0.05,"output":0.4} + }"#, + ) + .expect("rates") } - fn table() -> PriceTable { - PriceTable::from([ - ("sol".to_string(), price(1.25, 10.0)), - ("luna".to_string(), price(0.25, 2.0)), - ("terra".to_string(), price(0.05, 0.4)), - ]) + fn record_cost(totals: &mut Totals, usd: f64) { + totals.cost.add(Some(switchyard_costs::RecordedCost { + usd, + source: switchyard_costs::CostSource::Estimated, + })); } fn tokens(input: u64, output: u64) -> ModelTokens { @@ -105,6 +86,7 @@ mod tests { input, cached_input: 0, output, + ..ModelTokens::default() } } @@ -115,6 +97,7 @@ mod tests { .routed .insert("luna".to_string(), tokens(1_000_000, 100_000)); + record_cost(&mut totals, 0.45); let savings = estimate(&totals, &table(), "sol").expect("priced"); assert!((savings.actual - 0.45).abs() < 1e-9); @@ -128,11 +111,13 @@ mod tests { totals .routed .insert("luna".to_string(), tokens(1_000_000, 100_000)); + record_cost(&mut totals, 0.45); let before = estimate(&totals, &table(), "sol").expect("priced"); totals .classifier .insert("terra".to_string(), tokens(1_000_000, 10_000)); + record_cost(&mut totals, 0.054); let after = estimate(&totals, &table(), "sol").expect("priced"); assert_eq!(after.baseline, before.baseline); @@ -141,14 +126,9 @@ mod tests { #[test] fn cache_reads_use_the_cached_rate() { - let prices = PriceTable::from([( - "sol".to_string(), - ModelPrice { - input_per_mtok: 1.0, - cached_input_per_mtok: Some(0.1), - output_per_mtok: 0.0, - }, - )]); + let prices = + Pricing::parse(br#"{"sol":{"input":1,"cache_read":0.1,"cache_write":1,"output":0}}"#) + .expect("rates"); let mut totals = Totals::default(); totals.routed.insert( "sol".to_string(), @@ -157,23 +137,26 @@ mod tests { input: 1_000_000, cached_input: 1_000_000, output: 0, + ..ModelTokens::default() }, ); + record_cost(&mut totals, 1.1); let savings = estimate(&totals, &prices, "sol").expect("priced"); assert!((savings.actual - 1.1).abs() < 1e-9); } #[test] - fn an_unpriced_model_suppresses_the_estimate() { + fn an_unknown_cost_suppresses_savings() { let mut totals = Totals::default(); totals .routed .insert("unknown".to_string(), tokens(1_000, 100)); + totals.cost.add(None); assert!(estimate(&totals, &table(), "sol").is_none()); - assert!(estimate(&Totals::default(), &table(), "absent").is_none()); + assert!(estimate(&totals, &table(), "absent").is_none()); } #[test] diff --git a/crates/switchyard-menubar/src/rollup.rs b/crates/switchyard-menubar/src/rollup.rs index c05a34780f..0641ccb739 100644 --- a/crates/switchyard-menubar/src/rollup.rs +++ b/crates/switchyard-menubar/src/rollup.rs @@ -10,6 +10,7 @@ use std::path::Path; use chrono::{DateTime, Local, NaiveDate}; use serde::Deserialize; +use switchyard_costs::{CostTotals, RecordedCost}; /// Tier value the server writes for classifier and judge calls. const CLASSIFIER_TIER: &str = "classifier"; @@ -18,19 +19,22 @@ const CLASSIFIER_TIER: &str = "classifier"; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct ModelTokens { pub requests: u64, - /// Input tokens billed at the full rate. Cache writes bill at that rate - /// too, so they are counted here rather than tracked separately. + /// Input tokens exclude cache reads and writes. pub input: u64, /// Input tokens served from the provider cache. pub cached_input: u64, /// Generated tokens, including reasoning. pub output: u64, + /// Cache-write tokens are separate from uncached input. + pub cache_write: u64, + /// One-hour writes are a subset of all cache writes. + pub cache_write_1h: u64, } impl ModelTokens { /// Every token the provider counted, cached reads included. pub fn total(&self) -> u64 { - self.input + self.cached_input + self.output + self.input + self.cached_input + self.cache_write + self.output } fn add(&mut self, other: &Self) { @@ -38,6 +42,8 @@ impl ModelTokens { self.input += other.input; self.cached_input += other.cached_input; self.output += other.output; + self.cache_write += other.cache_write; + self.cache_write_1h += other.cache_write_1h; } } @@ -45,8 +51,12 @@ impl ModelTokens { /// /// The split matters for savings: without Switchyard the `routed` calls would /// still have happened, but the `classifier` calls would not exist at all. -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct Totals { + /// Actual spending comes only from the saved cost of each call. + pub cost: CostTotals, + /// Missing or invalid answer usage keeps savings unknown after totals are merged. + pub baseline_unknown: bool, /// Calls that served the caller, keyed by the model that answered. pub routed: BTreeMap, /// Switchyard's own classifier and judge calls, keyed by model. @@ -55,6 +65,10 @@ pub struct Totals { impl Totals { fn add(&mut self, record: &Record) { + self.cost.add(record.cost); + if record.tier != CLASSIFIER_TIER { + self.baseline_unknown |= !record.has_baseline_usage(); + } let bucket = if record.tier == CLASSIFIER_TIER { &mut self.classifier } else { @@ -67,6 +81,8 @@ impl Totals { } fn merge(&mut self, other: &Self) { + self.cost.merge(&other.cost); + self.baseline_unknown |= other.baseline_unknown; for (model, tokens) in &other.routed { self.routed.entry(model.clone()).or_default().add(tokens); } @@ -113,22 +129,48 @@ struct Record { #[serde(default)] tier: String, #[serde(default)] - prompt_tokens: u64, + prompt_tokens: Option, + #[serde(default)] + cached_tokens: Option, + #[serde(default)] + completion_tokens: Option, #[serde(default)] - cached_tokens: u64, + cache_creation_tokens: Option, #[serde(default)] - completion_tokens: u64, + cache_creation_1h_tokens: Option, + // A missing field requires migration; explicit null is a valid unknown cost. + #[serde(deserialize_with = "Option::deserialize")] + cost: Option, } impl Record { + // Each record must be valid before aggregation can conceal inconsistent cache counts. + fn has_baseline_usage(&self) -> bool { + let Some(input) = self.prompt_tokens else { + return false; + }; + self.completion_tokens.is_some() + && input + .checked_sub(self.cached_tokens.unwrap_or(0)) + .and_then(|input| input.checked_sub(self.cache_creation_tokens.unwrap_or(0))) + .is_some() + && self.cache_creation_1h_tokens.unwrap_or(0) <= self.cache_creation_tokens.unwrap_or(0) + } + fn tokens(&self) -> ModelTokens { ModelTokens { requests: 1, // `prompt_tokens` is the whole input, cache reads included. - input: self.prompt_tokens.saturating_sub(self.cached_tokens), - cached_input: self.cached_tokens, + input: self + .prompt_tokens + .unwrap_or(0) + .saturating_sub(self.cached_tokens.unwrap_or(0)) + .saturating_sub(self.cache_creation_tokens.unwrap_or(0)), + cached_input: self.cached_tokens.unwrap_or(0), // Completion tokens already include reasoning tokens. - output: self.completion_tokens, + output: self.completion_tokens.unwrap_or(0), + cache_write: self.cache_creation_tokens.unwrap_or(0), + cache_write_1h: self.cache_creation_1h_tokens.unwrap_or(0), } } } @@ -176,10 +218,22 @@ impl Reader { if line.last() != Some(&b'\n') { break; } - self.offset += bytes as u64; - let Ok(record) = serde_json::from_slice::(&line) else { - continue; + let record = match serde_json::from_slice::(&line) { + Ok(record) => record, + Err(error) if error.is_data() => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "invalid routing record: {error}; migrate older logs before reading them" + ), + )); + } + Err(_) => { + self.offset += bytes as u64; + continue; + } }; + self.offset += bytes as u64; let Ok(ts) = DateTime::parse_from_rfc3339(&record.ts) else { continue; }; @@ -232,7 +286,7 @@ mod tests { fn line(local_ts: &str, model: &str, tier: &str, prompt: u64, completion: u64) -> String { let offset = Local::now().offset().to_string(); format!( - r#"{{"ts":"{local_ts}{offset}","model":"{model}","tier":"{tier}","prompt_tokens":{prompt},"cached_tokens":0,"completion_tokens":{completion},"reasoning_tokens":0}}"# + r#"{{"ts":"{local_ts}{offset}","model":"{model}","tier":"{tier}","prompt_tokens":{prompt},"cached_tokens":0,"completion_tokens":{completion},"reasoning_tokens":0,"cost":null}}"# ) } @@ -266,7 +320,7 @@ mod tests { #[test] fn separates_cache_reads_from_full_price_input() { let record: Record = serde_json::from_str( - r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":1000,"cached_tokens":600,"completion_tokens":50,"reasoning_tokens":25}"#, + r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":1000,"cached_tokens":600,"completion_tokens":50,"reasoning_tokens":25,"cost":null}"#, ) .expect("parse record"); let tokens = record.tokens(); @@ -279,27 +333,39 @@ mod tests { #[test] fn counts_completion_tokens_once_and_prices_them() { let record: Record = serde_json::from_str( - r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":12,"cached_tokens":0,"completion_tokens":5,"reasoning_tokens":3}"#, + r#"{"ts":"2026-09-28T10:00:00.000Z","model":"m","prompt_tokens":12,"cached_tokens":0,"cache_creation_tokens":10,"completion_tokens":5,"reasoning_tokens":3,"cost":{"usd":0.000022,"source":"provider_reported"}}"#, ) .expect("parse record"); let tokens = record.tokens(); - assert_eq!(tokens.input, 12); + assert_eq!(tokens.input, 2); assert_eq!(tokens.output, 5); assert_eq!(tokens.total(), 17); let mut totals = Totals::default(); totals.add(&record); - let prices = crate::pricing::PriceTable::from([( - "m".to_string(), - crate::pricing::ModelPrice { - input_per_mtok: 1.0, - cached_input_per_mtok: None, - output_per_mtok: 2.0, - }, - )]); + let prices = switchyard_costs::Pricing::parse( + br#"{"m":{"input":1,"cache_read":1,"cache_write":1,"cache_write_1h":3,"output":2}}"#, + ) + .expect("rates"); let savings = crate::pricing::estimate(&totals, &prices, "m").expect("priced"); assert!((savings.actual - 0.000022).abs() < 1e-12); + // A known zero charge still needs valid usage for the hypothetical baseline. + for fields in [ + r#""prompt_tokens":null,"completion_tokens":5"#, + r#""completion_tokens":5"#, + r#""prompt_tokens":2,"cached_tokens":3,"completion_tokens":5"#, + r#""prompt_tokens":20,"cache_creation_tokens":2,"cache_creation_1h_tokens":3,"completion_tokens":5"#, + ] { + let record: Record = serde_json::from_str(&format!( + r#"{{"ts":"2026-09-28T10:00:00Z","model":"m",{fields},"cost":{{"usd":0,"source":"provider_reported"}}}}"# + )).expect("record with incomplete baseline usage"); + let mut incomplete = Totals::default(); + incomplete.add(&record); + incomplete.merge(&totals); + assert_eq!(incomplete.cost.total_usd, Some(0.000022)); + assert!(crate::pricing::estimate(&incomplete, &prices, "m").is_none()); + } } #[test] diff --git a/crates/switchyard-menubar/src/summary.rs b/crates/switchyard-menubar/src/summary.rs index 9c18cb6e26..024d42f016 100644 --- a/crates/switchyard-menubar/src/summary.rs +++ b/crates/switchyard-menubar/src/summary.rs @@ -51,7 +51,9 @@ pub fn build(status: ServerStatus, usage: &Usage, config: &Config) -> Vec { if estimate(&usage.week, &config.prices, &config.baseline_model).is_none() { rows.push(Row::Separator); - rows.push(label("Add prices to menubar.toml to see savings")); + rows.push(label( + "Savings need complete recorded costs and baseline rates in prices.json", + )); } rows } @@ -64,7 +66,7 @@ fn period(name: &str, totals: &Totals, config: &Config) -> Vec { tokens(totals.tokens()) ))]; if let Some(savings) = estimate(totals, &config.prices, &config.baseline_model) { - rows.push(label(format!(" Saved {}", saved(&savings)))); + rows.push(label(format!(" Estimated savings {}", saved(&savings)))); } rows } @@ -123,8 +125,8 @@ fn money(value: f64) -> String { #[cfg(test)] mod tests { use super::*; - use crate::pricing::{ModelPrice, PriceTable}; use crate::rollup::ModelTokens; + use switchyard_costs::{CostSource, Pricing, RecordedCost}; fn counted(input: u64, output: u64, requests: u64) -> ModelTokens { ModelTokens { @@ -132,10 +134,11 @@ mod tests { input, cached_input: 0, output, + ..ModelTokens::default() } } - fn config(prices: PriceTable) -> Config { + fn config(prices: Pricing) -> Config { Config { baseline_model: "sol".to_string(), prices, @@ -143,25 +146,8 @@ mod tests { } } - fn priced() -> PriceTable { - PriceTable::from([ - ( - "sol".to_string(), - ModelPrice { - input_per_mtok: 1.25, - cached_input_per_mtok: None, - output_per_mtok: 10.0, - }, - ), - ( - "luna".to_string(), - ModelPrice { - input_per_mtok: 0.25, - cached_input_per_mtok: None, - output_per_mtok: 2.0, - }, - ), - ]) + fn priced() -> Pricing { + Pricing::parse(br#"{"sol":{"input":1.25,"cache_read":1.25,"cache_write":1.25,"output":10},"luna":{"input":0.25,"cache_read":0.25,"cache_write":0.25,"output":2}}"#).expect("rates") } fn labels(rows: &[Row]) -> Vec { @@ -187,6 +173,14 @@ mod tests { week.routed .insert("luna".to_string(), counted(4_000_000, 400_000, 12)); + today.cost.add(Some(RecordedCost { + usd: 0.45, + source: CostSource::Estimated, + })); + week.cost.add(Some(RecordedCost { + usd: 1.80, + source: CostSource::Estimated, + })); let rows = build( ServerStatus::Running, &usage_of(today, week), @@ -196,9 +190,9 @@ mod tests { assert_eq!(labels[0], "Server: running · http://127.0.0.1:4123"); assert_eq!(labels[1], "Today — 3 requests · 1.1M tokens"); - assert_eq!(labels[2], " Saved $1.80 (80% of $2.25)"); + assert_eq!(labels[2], " Estimated savings $1.80 (80% of $2.25)"); assert_eq!(labels[3], "This week — 12 requests · 4.4M tokens"); - assert_eq!(labels[4], " Saved $7.20 (80% of $9.00)"); + assert_eq!(labels[4], " Estimated savings $7.20 (80% of $9.00)"); } #[test] @@ -209,19 +203,20 @@ mod tests { week.classifier .insert("sol".to_string(), counted(100, 0, 1)); - for prices in [ - PriceTable::new(), - PriceTable::from([("sol".to_string(), priced()["sol"])]), - PriceTable::from([("luna".to_string(), priced()["luna"])]), - ] { + week.cost.add(None); + for prices in [Pricing::default(), priced()] { let rows = build( ServerStatus::Running, &usage_of(week.clone(), week.clone()), &config(prices), ); let labels = labels(&rows); - assert!(labels.iter().all(|label| !label.contains("Saved"))); - assert!(labels.iter().any(|label| label.contains("menubar.toml"))); + assert!( + labels + .iter() + .all(|label| !label.contains("Estimated savings")) + ); + assert!(labels.iter().any(|label| label.contains("costs"))); } } @@ -263,13 +258,17 @@ mod tests { .classifier .insert("sol".to_string(), counted(1_000_000, 0, 1)); + today.cost.add(Some(RecordedCost { + usd: 1.25125, + source: CostSource::Estimated, + })); let rows = build( ServerStatus::Running, &usage_of(today.clone(), today), &config(priced()), ); - assert!(labels(&rows)[2].starts_with(" Saved -$1.25")); + assert!(labels(&rows)[2].starts_with(" Estimated savings -$1.25")); } #[test] diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 0870348323..e561ff8c9d 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -1415,6 +1415,7 @@ mod tests { output_tokens: Some(7), total_tokens: Some(23), reasoning_tokens: Some(5), + provider_cost: None, }), }; diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml index 2ef504d7ab..3601dd0832 100644 --- a/crates/switchyard-server/Cargo.toml +++ b/crates/switchyard-server/Cargo.toml @@ -20,6 +20,7 @@ publish = ["crates-io"] prefill-router = ["switchyard-runner/prefill-router"] [dependencies] +switchyard-costs.workspace = true async-stream.workspace = true async-trait.workspace = true axum = "0.8" diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 08335197f5..aa454a4607 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -124,6 +124,22 @@ served model. The legacy `proxy_x_session_id` remains a fallback when no normali present. The endpoint returns `404` when the session has no records and is not registered when routing logging is disabled. +The routing log saves each call's reported charge in USD. Add `--pricing-file prices.json` +to estimate costs that the upstream does not report. The JSON maps served model IDs to rates +in USD per million tokens: + +```json +{"upstream/model":{"input":1,"output":5,"cache_read":0.1,"cache_write":2}} +``` + +Session stats include cost totals. Missing usage or rates leave cost unknown; `known_usd` +keeps the priced subtotal and `total_usd` is null when any cost is unknown. Rates load once +at startup, and saved costs do not change when prices change. + +The [pi guide](../../docs/integrations/pi.md#estimate-the-cost) explains how to supply +rates for your own models. The menubar reads the same file through its `pricing_file` +setting; its actual spending comes from recorded costs. + Clients can send `x-switchyard-origin: codex-cli` (or another client label) to include an `origin` field in each routing record. Missing, empty, or non-text header values produce `"origin": null`. The value is supplied by the caller; it is not inferred from `User-Agent`. diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index 231d4e2bc4..9694457921 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -52,6 +52,10 @@ pub(crate) struct ServerArgs { #[arg(long, value_name = "PATH")] routing_log_file: Option, + /// This JSON file maps served model IDs to rates in USD per million tokens. + #[arg(long, value_name = "PATH", requires = "routing_log_file")] + pricing_file: Option, + /// TLS certificate path in PEM format. #[arg(long, requires = "tls_key")] tls_cert: Option, @@ -72,6 +76,9 @@ impl ServerArgs { if let Some(path) = self.routing_log_file { state = state.with_routing_log(path)?; } + if let Some(path) = self.pricing_file { + state = state.with_pricing_file(path)?; + } let tls = match (self.tls_cert, self.tls_key) { (Some(cert), Some(key)) => { if !cert.exists() || !key.exists() { diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 9e3ac630a2..4fed60a308 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -237,6 +237,17 @@ impl ServerState { Ok(self) } + /// This method loads rates in USD per million tokens for saved routing records. + pub fn with_pricing_file(self, path: impl AsRef) -> ServerResult { + let prices = switchyard_costs::Pricing::load(path.as_ref()).map_err(ServerError::new)?; + let log = self + .routing_log + .as_ref() + .ok_or_else(|| ServerError::new("pricing requires a routing log"))?; + log.writer.lock().pricing = prices; + Ok(self) + } + /// Returns the route model IDs served by the configured algorithms. pub fn models(&self) -> impl Iterator { self.runner.models().map(|model| model.id.as_str()) @@ -429,13 +440,11 @@ struct RequestStart(Instant); /// Routing-log marker distinguishing classifier and judge calls from answer calls. const CLASSIFIER_TIER: &str = "classifier"; -/// Maps answer-call observations to backend stats, routing calls to classifier/judge -/// stats, and records routing overhead once the algorithm run completes. +/// This observer updates backend stats and records classifier and judge calls. /// -/// Successful classifier/judge calls are also appended to `classifier_log` when one is -/// configured, so per-session routing snapshots account for judge token overhead. Routed -/// calls stay off this path: the served call is logged with its terminal usage in -/// [`usage_metrics::observe`], which would make a second append here a double count. +/// The routing log also records failed answer calls with unknown cost. Successful +/// answer calls are recorded by [`usage_metrics::observe`] with their terminal usage; +/// recording them here would count each answer twice. fn stats_observer( stats: StatsAccumulator, classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>, @@ -447,22 +456,28 @@ fn stats_observer( if call.is_success { stats.record_success(&call.selected_model, latency_ms); } else { + if let Some((log, context)) = classifier_log.as_ref() { + log.append( + context.clone(), + &call.selected_model, + None, + &Usage::default(), + ); + } stats.record_error(&call.selected_model); } } RunObservation::LlmCall(call) | RunObservation::DecisionCall(call) => { let latency_ms = call.duration.as_secs_f64() * 1_000.0; + if let Some((log, context)) = classifier_log.as_ref() { + log.append( + context.clone(), + &call.selected_model, + Some(CLASSIFIER_TIER), + call.usage.as_ref().unwrap_or(&Usage::default()), + ); + } if call.is_success { - if let (Some((log, context)), Some(usage)) = - (classifier_log.as_ref(), call.usage.as_ref()) - { - log.append( - context.clone(), - &call.selected_model, - Some(CLASSIFIER_TIER), - usage, - ); - } stats.record_classifier_success( call.selected_model, call.usage.as_ref().map(usage_metrics::token_usage), diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs index 674db1b6c5..01dddfb6b3 100644 --- a/crates/switchyard-server/src/routing_log.rs +++ b/crates/switchyard-server/src/routing_log.rs @@ -16,6 +16,7 @@ use switchyard_protocol::{Metadata, ModelId, Usage}; use crate::usage_metrics::token_usage; use crate::{ServerError, ServerResult}; +use switchyard_costs::{CostTotals, Pricing, RecordedCost}; const LEGACY_SESSION_ID_HEADER: &str = "proxy_x_session_id"; const ORIGIN_HEADER: &str = "x-switchyard-origin"; @@ -23,7 +24,10 @@ const TASK_HEADER: &str = "x-switchyard-intake-task"; const TRIAL_ID_HEADER: &str = "x-switchyard-trial-id"; /// Append-only writer for one routing JSONL file. -pub(crate) struct RoutingLog(fs::File); +pub(crate) struct RoutingLog { + file: fs::File, + pub(crate) pricing: Pricing, +} impl RoutingLog { pub(crate) fn new(path: impl Into) -> ServerResult { @@ -39,7 +43,10 @@ impl RoutingLog { .append(true) .open(&path) .map_err(|error| routing_log_error(&path, error))?; - Ok(Self(file)) + Ok(Self { + file, + pricing: Pricing::default(), + }) } pub(crate) fn append( @@ -49,8 +56,15 @@ impl RoutingLog { tier: Option<&str>, usage: &Usage, ) -> std::io::Result<()> { + let cost = self.pricing.cost(model, usage); + let cache_creation_1h_tokens = usage + .cache + .as_ref() + .and_then(|cache| cache.cache_creation_1h_input_tokens) + .unwrap_or(0); let usage = token_usage(usage); let record = RoutingRecord { + cost, ts: format_rfc3339_millis(SystemTime::now()).to_string().into(), route_id: context.route_id.into(), algorithm: context.algorithm.into(), @@ -64,6 +78,7 @@ impl RoutingLog { prompt_tokens: usage.prompt_tokens, cached_tokens: usage.cached_tokens, cache_creation_tokens: usage.cache_creation_tokens, + cache_creation_1h_tokens, completion_tokens: usage.completion_tokens, reasoning_tokens: usage.reasoning_tokens, total_tokens: usage.prompt_tokens.saturating_add(usage.completion_tokens), @@ -71,7 +86,7 @@ impl RoutingLog { let mut line = serde_json::to_vec(&record).map_err(std::io::Error::other)?; line.push(b'\n'); - self.0.write_all(&line) + self.file.write_all(&line) } } @@ -82,7 +97,10 @@ pub(crate) fn snapshot( ) -> std::io::Result> { let mut reader = BufReader::with_capacity(64 * 1024, fs::File::open(path)?); let mut line = Vec::new(); - let mut snapshot = SessionStatsSnapshot::new(session_id); + let mut snapshot = SessionStatsSnapshot { + session_id: session_id.to_string(), + ..Default::default() + }; loop { line.clear(); if reader.read_until(b'\n', &mut line)? == 0 { @@ -151,6 +169,8 @@ impl RoutingLogContext { #[derive(Default, Deserialize, Serialize)] #[serde(default)] struct RoutingRecord<'a> { + #[serde(default)] + cost: Option, ts: Cow<'a, str>, route_id: Cow<'a, str>, algorithm: Cow<'a, str>, @@ -169,15 +189,25 @@ struct RoutingRecord<'a> { prompt_tokens: u64, cached_tokens: u64, cache_creation_tokens: u64, + /// The part of `cache_creation_tokens` written at the one-hour rate, when reported. + #[serde(skip_serializing_if = "is_zero")] + cache_creation_1h_tokens: u64, completion_tokens: u64, reasoning_tokens: u64, total_tokens: u64, } +fn is_zero(value: &u64) -> bool { + *value == 0 +} + /// Session totals returned by the routing stats endpoint. -#[derive(Serialize)] +#[derive(Default, Serialize)] pub(crate) struct SessionStatsSnapshot { session_id: String, + cost: CostTotals, + answer_cost: CostTotals, + routing_cost: CostTotals, total_calls: u64, total_prompt_tokens: u64, total_cached_tokens: u64, @@ -188,6 +218,7 @@ pub(crate) struct SessionStatsSnapshot { #[derive(Default, Serialize)] struct SessionModelStats { + cost: CostTotals, calls: u64, prompt_tokens: u64, cached_tokens: u64, @@ -196,18 +227,6 @@ struct SessionModelStats { } impl SessionStatsSnapshot { - fn new(session_id: &str) -> Self { - Self { - session_id: session_id.to_string(), - total_calls: 0, - total_prompt_tokens: 0, - total_cached_tokens: 0, - total_cache_creation_tokens: 0, - total_completion_tokens: 0, - models: BTreeMap::new(), - } - } - fn add_record(&mut self, record: &RoutingRecord<'_>, session_id: &str) { if record.session_id.as_deref() != Some(session_id) { return; @@ -217,6 +236,13 @@ impl SessionStatsSnapshot { model => model, }; let stats = self.models.entry(ModelId::from(model)).or_default(); + stats.cost.add(record.cost); + self.cost.add(record.cost); + if record.tier == "classifier" { + self.routing_cost.add(record.cost); + } else { + self.answer_cost.add(record.cost); + } stats.calls = stats.calls.saturating_add(1); stats.prompt_tokens = stats.prompt_tokens.saturating_add(record.prompt_tokens); stats.cached_tokens = stats.cached_tokens.saturating_add(record.cached_tokens); diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs index 045f51c86c..54c2a1c011 100644 --- a/crates/switchyard-server/src/usage_metrics.rs +++ b/crates/switchyard-server/src/usage_metrics.rs @@ -66,6 +66,10 @@ pub(crate) fn observe( } } } + if failed && !recorded + && let Some((log, context)) = routing_log.as_ref() { + log.append(context.clone(), &model, None, &Usage::default()); + } if failed { record_stream_error(&stats, &model); } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 315d2470c6..d0fad52abc 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -4769,10 +4769,16 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted .lines() .map(serde_json::from_str::) .collect::, _>>()?; - assert_eq!(records.len(), 3); - assert!(records.iter().all(|record| { - record["model"] == "model/strong" && record.get("fallback_reason").is_none() - })); + assert_eq!(records.len(), 6); + assert_eq!( + records + .iter() + .filter(|record| { + record["model"] == "model/strong" && record.get("fallback_reason").is_none() + }) + .count(), + 3 + ); let previous_call_count = upstream.calls.lock().await.len(); let response = send( diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 88aa8a324e..9282c71ccf 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -1256,8 +1256,13 @@ fn decode_anthropic_usage(value: Option<&Value>) -> Usage { .and_then(Value::as_u64); let output_tokens = value.get("output_tokens").and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(value), input_tokens, - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + value, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens, total_tokens: input_tokens.zip(output_tokens).map(|(input, output)| { input @@ -1289,7 +1294,7 @@ fn encode_anthropic_usage(usage: &Usage) -> Value { if let Some(thinking_tokens) = usage.reasoning_tokens { value["output_tokens_details"] = json!({"thinking_tokens": thinking_tokens}); } - value + crate::codecs::common::encode_usage_metadata(value, usage) } // Maps Anthropic stop reasons to normalized stop reasons. diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs index 908941a89f..f901d875e1 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs @@ -628,6 +628,20 @@ fn capture_anthropic_usage(state: &mut StreamTranslationState, usage: &Value) { if let Some(value) = usage.get("input_tokens").and_then(Value::as_u64) { state.usage.input_tokens = Some(value); } + if let Some(cost) = crate::codecs::common::provider_cost(usage) { + state.usage.provider_cost = Some(cost); + } + if let Some(tokens) = usage + .get("cache_creation") + .and_then(|details| details.get("ephemeral_1h_input_tokens")) + .and_then(Value::as_u64) + { + state + .usage + .cache + .get_or_insert_with(Default::default) + .cache_creation_1h_input_tokens = Some(tokens); + } if let Some(value) = usage.get("output_tokens").and_then(Value::as_u64) { state.usage.output_tokens = Some(value); } @@ -676,7 +690,7 @@ fn anthropic_stream_usage(state: &StreamTranslationState) -> Value { json!({"thinking_tokens": value}), ); } - Value::Object(usage) + crate::codecs::common::encode_usage_metadata(Value::Object(usage), &state.usage) } // Converts any upstream message ID into an Anthropic-looking message ID. diff --git a/crates/switchyard-translation/src/codecs/common.rs b/crates/switchyard-translation/src/codecs/common.rs index ddd795f806..303c90321b 100644 --- a/crates/switchyard-translation/src/codecs/common.rs +++ b/crates/switchyard-translation/src/codecs/common.rs @@ -231,6 +231,51 @@ pub(crate) fn provider_extensions( extensions } +/// Invalid charges remain unknown so accounting can use configured token rates. +pub(crate) fn provider_cost(value: &Map) -> Option { + value + .get("cost") + .and_then(Value::as_f64) + .filter(|cost| cost.is_finite() && *cost >= 0.0) +} + +/// Cache duration must survive decoding because one-hour writes can have a different rate. +pub(crate) fn cache_usage_details( + value: &Map, + cached: Option, + created: Option, +) -> Option> { + let one_hour = value + .get("cache_creation") + .and_then(|details| details.get("ephemeral_1h_input_tokens")) + .and_then(Value::as_u64); + let mut cache = crate::llm::Usage::cache_details(cached, created); + if let Some(tokens) = one_hour { + cache + .get_or_insert_with(Default::default) + .cache_creation_1h_input_tokens = Some(tokens); + } + cache +} + +/// These fields preserve provider charges and cache duration across response formats. +pub(crate) fn encode_usage_metadata(mut value: Value, usage: &crate::llm::Usage) -> Value { + if let Some(cost) = usage + .provider_cost + .filter(|cost| cost.is_finite() && *cost >= 0.0) + { + value["cost"] = serde_json::json!(cost); + } + if let Some(tokens) = usage + .cache + .as_ref() + .and_then(|cache| cache.cache_creation_1h_input_tokens) + { + value["cache_creation"] = serde_json::json!({"ephemeral_1h_input_tokens": tokens}); + } + value +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 907518fe93..0dc0641625 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -1248,6 +1248,7 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage { }) .and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(value), input_tokens: value .get("prompt_tokens") .and_then(Value::as_u64) @@ -1256,7 +1257,11 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage { .saturating_sub(cached_input_tokens.unwrap_or(0)) .saturating_sub(cache_creation_input_tokens.unwrap_or(0)) }), - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + value, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens: value.get("completion_tokens").and_then(Value::as_u64), total_tokens: value.get("total_tokens").and_then(Value::as_u64), reasoning_tokens: value @@ -1295,7 +1300,7 @@ pub(crate) fn encode_openai_usage(usage: &Usage) -> Value { "reasoning_tokens": reasoning_tokens, }); } - value + crate::codecs::common::encode_usage_metadata(value, usage) } /// Maps OpenAI finish reasons to normalized stop reasons. diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs index c86749728b..02cec4b6e5 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs @@ -5,6 +5,7 @@ use serde_json::{Map, Value, json}; +use super::buffered::{decode_openai_usage, encode_openai_usage}; use crate::LlmResponseChunk; use crate::codecs::common::{first_nonempty_string, reasoning_text_from_details}; use crate::codecs::stream::{ @@ -12,7 +13,6 @@ use crate::codecs::stream::{ target_model_or_source_model, }; use crate::format::{FormatId, WireFormat}; -use crate::llm::Usage; /// Stream codec for OpenAI Chat Completions chunks. pub struct OpenAiChatStreamCodec; @@ -84,8 +84,8 @@ fn decode_openai_chat_stream( }); } - if let Some(usage) = object.get("usage").and_then(Value::as_object) { - let usage = openai_usage(usage); + if let Some(usage) = object.get("usage").filter(|usage| usage.is_object()) { + let usage = decode_openai_usage(Some(usage)); state.usage = usage.clone(); state.saw_backend_usage = true; out.push(LlmResponseChunk::Usage(usage)); @@ -324,7 +324,9 @@ fn encode_openai_chat_stream( state, json!({}), Some(openai_finish_reason(reason.as_deref())), - state.saw_backend_usage.then(|| openai_usage_value(state)), + state + .saw_backend_usage + .then(|| encode_openai_usage(&state.usage)), )] } LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => { @@ -346,48 +348,12 @@ fn finish_openai_chat_stream(state: &mut StreamTranslationState) -> Vec { state, json!({}), Some(openai_finish_reason(state.stop_reason.as_deref())), - state.saw_backend_usage.then(|| openai_usage_value(state)), + state + .saw_backend_usage + .then(|| encode_openai_usage(&state.usage)), )] } -// Normalizes OpenAI token usage fields. -fn openai_usage(usage: &Map) -> Usage { - let cached_input_tokens = usage - .get("prompt_tokens_details") - .and_then(|details| details.get("cached_tokens")) - .and_then(Value::as_u64); - let cache_creation_input_tokens = usage - .get("prompt_tokens_details") - .and_then(|details| { - details - .get("cache_write_tokens") - .or_else(|| details.get("cache_creation_tokens")) - }) - .and_then(Value::as_u64); - Usage { - input_tokens: usage - .get("prompt_tokens") - .and_then(Value::as_u64) - .map(|tokens| { - tokens - .saturating_sub(cached_input_tokens.unwrap_or(0)) - .saturating_sub(cache_creation_input_tokens.unwrap_or(0)) - }), - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), - output_tokens: usage.get("completion_tokens").and_then(Value::as_u64), - total_tokens: usage.get("total_tokens").and_then(Value::as_u64), - reasoning_tokens: usage - .get("completion_tokens_details") - .and_then(|details| details.get("reasoning_tokens")) - .or_else(|| { - usage - .get("output_tokens_details") - .and_then(|details| details.get("reasoning_tokens")) - }) - .and_then(Value::as_u64), - } -} - // Builds a single OpenAI Chat stream chunk payload. fn openai_stream_chunk( state: &StreamTranslationState, @@ -414,7 +380,12 @@ fn openai_stream_chunk( // Builds the usage-only chunk OpenAI emits after the terminal choices chunk. fn openai_usage_chunk(state: &StreamTranslationState) -> Value { - let mut payload = openai_stream_chunk(state, json!({}), None, Some(openai_usage_value(state))); + let mut payload = openai_stream_chunk( + state, + json!({}), + None, + Some(encode_openai_usage(&state.usage)), + ); payload["choices"] = json!([]); payload } @@ -449,34 +420,6 @@ fn openai_tool_call_chunk( ) } -// Builds OpenAI usage payloads from normalized and provider-extra state. -fn openai_usage_value(state: &StreamTranslationState) -> Value { - let cache_creation_tokens = state.usage.cache_creation_input_tokens().unwrap_or(0); - let cache_read_tokens = state.usage.cached_input_tokens().unwrap_or(0); - let prompt_tokens = - state.usage.input_tokens.unwrap_or(0) + cache_creation_tokens + cache_read_tokens; - let completion_tokens = state.usage.output_tokens.unwrap_or(0); - let mut usage = json!({ - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": state.usage.total_tokens.unwrap_or(prompt_tokens + completion_tokens), - }); - if let Some(reasoning_tokens) = state.usage.reasoning_tokens { - usage["completion_tokens_details"] = json!({ - "reasoning_tokens": reasoning_tokens, - }); - } - if state.usage.cache_creation_input_tokens().is_some() - || state.usage.cached_input_tokens().is_some() - { - usage["prompt_tokens_details"] = json!({ - "cached_tokens": cache_read_tokens, - "cache_creation_tokens": cache_creation_tokens, - }); - } - usage -} - // Converts any upstream message ID into an OpenAI-looking stream ID. fn openai_stream_id(state: &StreamTranslationState) -> String { let Some(id) = state.message_id.as_deref() else { diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 43fab28069..294e597408 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -2100,8 +2100,13 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage { .or_else(|| value.get("completion_tokens")) .and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(value), input_tokens, - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + value, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens, total_tokens: value .get("total_tokens") @@ -2124,7 +2129,7 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage { } // Encodes normalized usage into Responses usage JSON. -fn encode_responses_usage(usage: &Usage) -> Value { +pub(super) fn encode_responses_usage(usage: &Usage) -> Value { let input_tokens = usage.input_tokens.unwrap_or(0) + usage.cached_input_tokens().unwrap_or(0) + usage.cache_creation_input_tokens().unwrap_or(0); @@ -2132,19 +2137,22 @@ fn encode_responses_usage(usage: &Usage) -> Value { // upstream that reports no cache or reasoning breakdown means zero, not "unknown"; omitting // them instead yields a payload that fails to deserialize into the OpenAI SDK's // ResponseUsage, which types both as non-optional. - json!({ - "input_tokens": input_tokens, - "output_tokens": usage.output_tokens.unwrap_or(0), - "total_tokens": usage - .total_tokens - .or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0))) - .unwrap_or(0), - "input_tokens_details": { - "cached_tokens": usage.cached_input_tokens().unwrap_or(0), - "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), - }, - "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)}, - }) + crate::codecs::common::encode_usage_metadata( + json!({ + "input_tokens": input_tokens, + "output_tokens": usage.output_tokens.unwrap_or(0), + "total_tokens": usage + .total_tokens + .or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0))) + .unwrap_or(0), + "input_tokens_details": { + "cached_tokens": usage.cached_input_tokens().unwrap_or(0), + "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), + }, + "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)}, + }), + usage, + ) } /// Re-emits captured request extensions that the Responses format accepts. diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index 35f3a1411a..b726bea71a 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -17,6 +17,8 @@ use crate::codecs::stream::{ use crate::format::{FormatId, WireFormat}; use crate::llm::Usage; +use super::buffered::encode_responses_usage; + /// Stream codec for OpenAI Responses API events. pub struct OpenAiResponsesStreamCodec; @@ -850,7 +852,7 @@ fn responses_stream_response( tool_choice: "auto", tools: Vec::new(), top_p: None, - usage: responses_usage_value(&state.usage), + usage: encode_responses_usage(&state.usage), } } @@ -1163,8 +1165,13 @@ fn responses_usage(usage: &serde_json::Map) -> Usage { }); let output_tokens = usage.get("output_tokens").and_then(Value::as_u64); Usage { + provider_cost: crate::codecs::common::provider_cost(usage), input_tokens, - cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens), + cache: crate::codecs::common::cache_usage_details( + usage, + cached_input_tokens, + cache_creation_input_tokens, + ), output_tokens, total_tokens: usage .get("total_tokens") @@ -1182,27 +1189,6 @@ fn responses_usage(usage: &serde_json::Map) -> Usage { } } -// Builds OpenAI Responses usage payloads from normalized usage. -fn responses_usage_value(usage: &Usage) -> Value { - let input_tokens = usage.input_tokens.unwrap_or(0) - + usage.cached_input_tokens().unwrap_or(0) - + usage.cache_creation_input_tokens().unwrap_or(0); - // Both detail objects are always present, for the same reason as the buffered encoder: the - // Responses schema types them as required, so a missing breakdown serializes as zero. - json!({ - "input_tokens": input_tokens, - "output_tokens": usage.output_tokens.unwrap_or(0), - "total_tokens": usage.total_tokens.unwrap_or_else(|| { - input_tokens + usage.output_tokens.unwrap_or(0) - }), - "input_tokens_details": { - "cached_tokens": usage.cached_input_tokens().unwrap_or(0), - "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0), - }, - "output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)}, - }) -} - // Longest response-id discriminator embedded verbatim in a synthesized item id. OpenAI rejects // item ids over 64 characters, and some upstreams issue response ids several hundred characters // long, so anything longer is replaced by a fixed-width digest. diff --git a/docs/cli_reference.md b/docs/cli_reference.md index e5da32455a..c9374fbb9c 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -20,6 +20,7 @@ switchyard-server --config [options] | `--shutdown-timeout SHUTDOWN_TIMEOUT` | `30s` | Maximum time active requests may drain during shutdown. | | `--dry-run` | Off | Validate the deployment without binding a socket. | | `--routing-log-file PATH` | None | Append durable per-request routing records to this JSONL file. | +| `--pricing-file PATH` | None | Estimate unreported call costs from this JSON file of USD rates per million tokens; requires `--routing-log-file`. | | `--tls-cert PATH` | None | PEM certificate path; requires `--tls-key`. | | `--tls-key PATH` | None | PEM private-key path; requires `--tls-cert`. | | `-h, --help` | — | Print command help. | diff --git a/docs/integrations/pi.md b/docs/integrations/pi.md index 75a2853b39..1c71f9aa3d 100644 --- a/docs/integrations/pi.md +++ b/docs/integrations/pi.md @@ -155,15 +155,18 @@ second shows `cached_tokens` close to `prompt_tokens`. If the second record show #### Estimate the cost -The routing log records token counts, not prices. To estimate what a request cost, -multiply each count in its record by the matching price and add the results: +The server matches each token count in the routing log to a rate in the pricing file: -| Tokens in the record | Price | +| Tokens in the record | Pricing-file key | |---|---| -| `prompt_tokens - cached_tokens - cache_creation_tokens` | Input | -| `cached_tokens` | Cache read | -| `cache_creation_tokens` | Cache write | -| `completion_tokens` | Output | +| `prompt_tokens - cached_tokens - cache_creation_tokens` | `input` | +| `cached_tokens` | `cache_read` | +| `cache_creation_tokens - cache_creation_1h_tokens` | `cache_write` | +| `cache_creation_1h_tokens`, when reported | `cache_write_1h` | +| `completion_tokens` | `output` | + +Treat a missing `cache_creation_1h_tokens` count as zero. That count is part of +`cache_creation_tokens`, so it must not be charged twice. Write the prices to `prices.json` in USD per million tokens, and key each entry by the record's `model` value. This example uses Anthropic's list prices for Claude Sonnet 5 on @@ -175,25 +178,30 @@ record's `model` value. This example uses Anthropic's list prices for Claude Son } ``` +For your own models, copy the entry for each model and replace the key with its exact +`model` value from the routing log. Use the served model ID rather than the routing alias, +and replace the example's numbers with your provider's rates. List recorded model IDs with: + +```bash +jq -r '.model' PATH | sort -u +``` + +Every entry requires `input`, `output`, `cache_read`, and `cache_write`. Set a rate to zero +only when that usage is free. Each rate must be a finite number from 0 to 10000 USD per +million tokens. + The example's `cache_write` price is for a 5-minute cache. A 1-hour cache costs 2 times -the input price instead of 1.25 times. The routing log does not say which cache the -gateway used. A gateway may also charge its own prices, so the result is an estimate, not -the gateway's bill. This command prints one estimated cost per record, or a warning for a -model that has no entry in `prices.json`: +the input price instead of 1.25 times; set `cache_write_1h` to `4.00` for that example. +Include that rate to estimate reported one-hour cache writes; without it, the estimate +remains unknown. + +Start the server with `--pricing-file prices.json` as well as `--routing-log-file PATH`. +The server records the gateway's reported charge when available, even without rates. +Otherwise it uses the pricing file to estimate the cost. If usage is missing or the model +has no rates, the estimate is unknown. Read each request's cost with: ```bash -jq -r --slurpfile prices prices.json ' - . as $r - | ($prices[0][$r.model // ""]) as $p - | if $p == null then - "warning: no price for model \($r.model); add it to prices.json" - else - ((($r.prompt_tokens // 0) - ($r.cached_tokens // 0) - ($r.cache_creation_tokens // 0)) * $p.input - + ($r.cached_tokens // 0) * $p.cache_read - + ($r.cache_creation_tokens // 0) * $p.cache_write - + ($r.completion_tokens // 0) * $p.output) / 1000000 - | "\($r.route_id) \($r.model) estimated $\(. * 1000000 | round / 1000000)" - end' PATH +jq -c '{route_id, model, cost}' PATH ``` ### Thinking diff --git a/scripts/macos/install.sh b/scripts/macos/install.sh index ae7b3a9ddb..f3cff8135b 100755 --- a/scripts/macos/install.sh +++ b/scripts/macos/install.sh @@ -55,8 +55,8 @@ step "Writing server config" write_once "$SY_HOME/composite.toml" < "$REPO_ROOT/scripts/config/composite.toml" step "Writing menu bar settings" -# Rates are list prices per million tokens and are only used to estimate -# savings. Edit them to match what you actually pay. +# The server and menubar share rates in USD per million tokens. +# Edit prices.json to match what you actually pay. TOML_SY_HOME="$(toml_escape_basic_string "$SY_HOME")" write_once "$SY_HOME/menubar.toml" <$SY_PORT --routing-log-file $XML_SY_HOME/routing.jsonl + --pricing-file + $XML_SY_HOME/prices.json RunAtLoad