From 302b65467d7fbd756375cb35d1e21b9b04561372 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:16:08 +0700 Subject: [PATCH 01/41] Pin bundled pricing tables --- rust/src/core/cost_pricing_tests.rs | 111 ++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/rust/src/core/cost_pricing_tests.rs b/rust/src/core/cost_pricing_tests.rs index 884efd92c4..f8168aeec4 100644 --- a/rust/src/core/cost_pricing_tests.rs +++ b/rust/src/core/cost_pricing_tests.rs @@ -803,3 +803,114 @@ fn gpt54_and_gpt55_bill_the_whole_request_at_long_context_rates_above_272k() { assert!(CostUsagePricing::codex_fast_cost_usd("gpt-5.5-priority", 272_001, 0, 1).is_none()); assert!(CostUsagePricing::codex_fast_cost_usd("gpt-5.4-fast", 272_000, 0, 1).is_some()); } + +fn sorted_keys

(table: &HashMap<&'static str, P>) -> Vec<&'static str> { + let mut keys: Vec<_> = table.keys().copied().collect(); + keys.sort_unstable(); + keys +} + +fn codex_pricing_rows() -> String { + sorted_keys(&CODEX_PRICING) + .into_iter() + .map(|key| { + let p = CODEX_PRICING[key]; + let long = p.long_context.map(|l| { + ( + l.input_cost_per_token, + l.output_cost_per_token, + l.cache_read_input_cost_per_token, + l.cache_write_input_cost_per_token, + ) + }); + format!( + "{key} {:?} {:?} {:?} {:?} {:?} {:?}\n", + p.input_cost_per_token, + p.output_cost_per_token, + p.cache_read_input_cost_per_token, + p.cache_write_input_cost_per_token, + p.display_label, + long, + ) + }) + .collect() +} + +fn claude_pricing_rows() -> String { + sorted_keys(&CLAUDE_PRICING) + .into_iter() + .map(|key| { + let p = CLAUDE_PRICING[key]; + format!( + "{key} {:?} {:?} {:?} {:?} {:?} {:?} {:?} {:?} {:?}\n", + p.input_cost_per_token, + p.output_cost_per_token, + p.cache_creation_input_cost_per_token, + p.cache_read_input_cost_per_token, + p.threshold_tokens, + p.input_cost_per_token_above_threshold, + p.output_cost_per_token_above_threshold, + p.cache_creation_input_cost_per_token_above_threshold, + p.cache_read_input_cost_per_token_above_threshold, + ) + }) + .collect() +} + +#[test] +fn bundled_pricing_tables_keep_every_model_and_rate() { + assert_eq!( + codex_pricing_rows(), + "\ +gpt-5 1.25e-6 1e-5 1.25e-7 None None None +gpt-5-codex 1.25e-6 1e-5 1.25e-7 None None None +gpt-5-mini 2.5e-7 2e-6 2.5e-8 None None None +gpt-5-nano 5e-8 4e-7 5e-9 None None None +gpt-5-pro 1.5e-5 0.00012 1.5e-5 None None None +gpt-5.1 1.25e-6 1e-5 1.25e-7 None None None +gpt-5.1-codex 1.25e-6 1e-5 1.25e-7 None None None +gpt-5.1-codex-max 1.25e-6 1e-5 1.25e-7 None None None +gpt-5.1-codex-mini 2.5e-7 2e-6 2.5e-8 None None None +gpt-5.2 1.75e-6 1.4e-5 1.75e-7 None None None +gpt-5.2-codex 1.75e-6 1.4e-5 1.75e-7 None None None +gpt-5.2-pro 2.1e-5 0.000168 2.1e-5 None None None +gpt-5.3-codex 1.75e-6 1.4e-5 1.75e-7 None None None +gpt-5.3-codex-spark 0.0 0.0 0.0 None Some(\"Research Preview\") None +gpt-5.4 2.5e-6 1.5e-5 2.5e-7 None None Some((5e-6, 2.25e-5, 5e-7, None)) +gpt-5.4-codex 2.5e-6 1.5e-5 2.5e-7 None None Some((5e-6, 2.25e-5, 5e-7, None)) +gpt-5.4-mini 7.5e-7 4.5e-6 7.5e-8 None None None +gpt-5.4-mini-codex 7.5e-7 4.5e-6 7.5e-8 None None None +gpt-5.4-nano 2e-7 1.25e-6 2e-8 None None None +gpt-5.4-nano-codex 2e-7 1.25e-6 2e-8 None None None +gpt-5.4-pro 3e-5 0.00018 3e-5 None None None +gpt-5.5 5e-6 3e-5 5e-7 None None Some((1e-5, 4.5e-5, 1e-6, None)) +gpt-5.5-cyber 1.25e-5 7.5e-5 1.25e-6 None None None +gpt-5.5-pro 3e-5 0.00018 3e-5 None None None +gpt-5.6-cyber 1.25e-5 7.5e-5 1.25e-6 Some(1.5625e-5) None None +gpt-5.6-luna 2e-7 1.2e-6 2e-8 Some(2.5e-7) None Some((4e-7, 1.8e-6, 4e-8, Some(5e-7))) +gpt-5.6-sol 4e-6 2e-5 4e-7 Some(5e-6) None Some((8e-6, 3e-5, 8e-7, Some(1e-5))) +gpt-5.6-terra 2e-6 1.2e-5 2e-7 Some(2.5e-6) None Some((4e-6, 1.8e-5, 4e-7, Some(5e-6))) +gpt-6-astra 1e-5 5e-5 1e-6 Some(1.25e-5) None Some((2e-5, 7.5e-5, 2e-6, Some(2.5e-5))) +" + ); + assert_eq!( + claude_pricing_rows(), + "\ +claude-fable-5 1e-5 5e-5 1.25e-5 1e-6 None None None None None +claude-haiku-4-5 1e-6 5e-6 1.25e-6 1e-7 None None None None None +claude-haiku-4-5-20251001 1e-6 5e-6 1.25e-6 1e-7 None None None None None +claude-opus-4-1 1.5e-5 7.5e-5 1.875e-5 1.5e-6 None None None None None +claude-opus-4-20250514 1.5e-5 7.5e-5 1.875e-5 1.5e-6 None None None None None +claude-opus-4-5 5e-6 2.5e-5 6.25e-6 5e-7 None None None None None +claude-opus-4-5-20251101 5e-6 2.5e-5 6.25e-6 5e-7 None None None None None +claude-opus-4-6 5e-6 2.5e-5 6.25e-6 5e-7 None None None None None +claude-opus-4-6-20260205 5e-6 2.5e-5 6.25e-6 5e-7 None None None None None +claude-opus-4-7 5e-6 2.5e-5 6.25e-6 5e-7 None None None None None +claude-opus-4-8 5e-6 2.5e-5 6.25e-6 5e-7 None None None None None +claude-sonnet-4-20250514 3e-6 1.5e-5 3.75e-6 3e-7 Some(200000) Some(6e-6) Some(2.25e-5) Some(7.5e-6) Some(6e-7) +claude-sonnet-4-5 3e-6 1.5e-5 3.75e-6 3e-7 Some(200000) Some(6e-6) Some(2.25e-5) Some(7.5e-6) Some(6e-7) +claude-sonnet-4-5-20250929 3e-6 1.5e-5 3.75e-6 3e-7 Some(200000) Some(6e-6) Some(2.25e-5) Some(7.5e-6) Some(6e-7) +claude-sonnet-4-6 3e-6 1.5e-5 3.75e-6 3e-7 Some(200000) Some(6e-6) Some(2.25e-5) Some(7.5e-6) Some(6e-7) +" + ); +} From c5f035dcbe01433f37d3e14fa9fec7c0b9129ba6 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:20:41 +0700 Subject: [PATCH 02/41] Build pricing tables from const rate rows RCO-14: replace the 44 hand-written HashMap inserts with const-fn rows and shared rate constants. The pin test from the previous commit holds every key and rate. Also inline the one-caller codex_cost_usd_at_date_with_pricing_snapshot. --- rust/src/core/cost_pricing.rs | 769 +++++++--------------------- rust/src/core/cost_pricing/codex.rs | 23 +- 2 files changed, 185 insertions(+), 607 deletions(-) diff --git a/rust/src/core/cost_pricing.rs b/rust/src/core/cost_pricing.rs index 3b807702d6..bde8af627e 100755 --- a/rust/src/core/cost_pricing.rs +++ b/rust/src/core/cost_pricing.rs @@ -60,589 +60,190 @@ pub struct ClaudePricing { pub cache_read_input_cost_per_token_above_threshold: Option, } -/// Codex model pricing table -static CODEX_PRICING: LazyLock> = LazyLock::new(|| { - let mut m = HashMap::new(); - - // GPT-5 pricing - m.insert( - "gpt-5", - CodexPricing { - input_cost_per_token: 1.25e-6, - output_cost_per_token: 1e-5, - cache_read_input_cost_per_token: 1.25e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5-codex", - CodexPricing { - input_cost_per_token: 1.25e-6, - output_cost_per_token: 1e-5, - cache_read_input_cost_per_token: 1.25e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5-mini", - CodexPricing { - input_cost_per_token: 2.5e-7, - output_cost_per_token: 2e-6, - cache_read_input_cost_per_token: 2.5e-8, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5-nano", - CodexPricing { - input_cost_per_token: 5e-8, - output_cost_per_token: 4e-7, - cache_read_input_cost_per_token: 5e-9, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5-pro", - CodexPricing { - input_cost_per_token: 1.5e-5, - output_cost_per_token: 1.2e-4, - cache_read_input_cost_per_token: 1.5e-5, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.1", - CodexPricing { - input_cost_per_token: 1.25e-6, - output_cost_per_token: 1e-5, - cache_read_input_cost_per_token: 1.25e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.1-codex", - CodexPricing { - input_cost_per_token: 1.25e-6, - output_cost_per_token: 1e-5, - cache_read_input_cost_per_token: 1.25e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.1-codex-max", - CodexPricing { - input_cost_per_token: 1.25e-6, - output_cost_per_token: 1e-5, - cache_read_input_cost_per_token: 1.25e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.1-codex-mini", - CodexPricing { - input_cost_per_token: 2.5e-7, - output_cost_per_token: 2e-6, - cache_read_input_cost_per_token: 2.5e-8, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.2", - CodexPricing { - input_cost_per_token: 1.75e-6, - output_cost_per_token: 1.4e-5, - cache_read_input_cost_per_token: 1.75e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.2-codex", - CodexPricing { - input_cost_per_token: 1.75e-6, - output_cost_per_token: 1.4e-5, - cache_read_input_cost_per_token: 1.75e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.2-pro", - CodexPricing { - input_cost_per_token: 2.1e-5, - output_cost_per_token: 1.68e-4, - cache_read_input_cost_per_token: 2.1e-5, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.3-codex", - CodexPricing { - input_cost_per_token: 1.75e-6, - output_cost_per_token: 1.4e-5, - cache_read_input_cost_per_token: 1.75e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.3-codex-spark", - CodexPricing { - input_cost_per_token: 0.0, - output_cost_per_token: 0.0, - cache_read_input_cost_per_token: 0.0, - cache_write_input_cost_per_token: None, - display_label: Some("Research Preview"), - long_context: None, - }, - ); - - // GPT-5.4 pricing (updated to match upstream 0.22). Like upstream, the - // whole request bills 2x input / 1.5x output above 272K input tokens. - m.insert( - "gpt-5.4", - CodexPricing { - input_cost_per_token: 2.5e-6, - output_cost_per_token: 1.5e-5, - cache_read_input_cost_per_token: 2.5e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: Some(CodexLongContextRates { - input_cost_per_token: 5e-6, - output_cost_per_token: 2.25e-5, - cache_read_input_cost_per_token: 5e-7, - cache_write_input_cost_per_token: None, - }), - }, - ); - m.insert( - "gpt-5.4-codex", - CodexPricing { - input_cost_per_token: 2.5e-6, - output_cost_per_token: 1.5e-5, - cache_read_input_cost_per_token: 2.5e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: Some(CodexLongContextRates { - input_cost_per_token: 5e-6, - output_cost_per_token: 2.25e-5, - cache_read_input_cost_per_token: 5e-7, - cache_write_input_cost_per_token: None, - }), - }, - ); - - // GPT-5.4 Mini pricing (updated to match upstream 0.22) - m.insert( - "gpt-5.4-mini", - CodexPricing { - input_cost_per_token: 7.5e-7, - output_cost_per_token: 4.5e-6, - cache_read_input_cost_per_token: 7.5e-8, +impl CodexPricing { + /// Standard per-token rates; cache writes bill at the input rate and there + /// is no long-context tier. + const fn new(input: f64, output: f64, cache_read: f64) -> Self { + Self { + input_cost_per_token: input, + output_cost_per_token: output, + cache_read_input_cost_per_token: cache_read, cache_write_input_cost_per_token: None, display_label: None, long_context: None, - }, - ); - m.insert( - "gpt-5.4-mini-codex", - CodexPricing { - input_cost_per_token: 7.5e-7, - output_cost_per_token: 4.5e-6, - cache_read_input_cost_per_token: 7.5e-8, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - - // GPT-5.4 Nano pricing (updated to match upstream 0.22) - m.insert( - "gpt-5.4-nano", - CodexPricing { - input_cost_per_token: 2e-7, - output_cost_per_token: 1.25e-6, - cache_read_input_cost_per_token: 2e-8, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.4-nano-codex", - CodexPricing { - input_cost_per_token: 2e-7, - output_cost_per_token: 1.25e-6, - cache_read_input_cost_per_token: 2e-8, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - - // GPT-5.4 Pro - m.insert( - "gpt-5.4-pro", - CodexPricing { - input_cost_per_token: 3e-5, - output_cost_per_token: 1.8e-4, - cache_read_input_cost_per_token: 3e-5, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.5", - CodexPricing { - input_cost_per_token: 5e-6, - output_cost_per_token: 3e-5, - cache_read_input_cost_per_token: 5e-7, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: Some(CodexLongContextRates { - input_cost_per_token: 1e-5, - output_cost_per_token: 4.5e-5, - cache_read_input_cost_per_token: 1e-6, - cache_write_input_cost_per_token: None, - }), - }, - ); - m.insert( - "gpt-5.5-pro", - CodexPricing { - input_cost_per_token: 3e-5, - output_cost_per_token: 1.8e-4, - cache_read_input_cost_per_token: 3e-5, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - // GPT-5.6 Sol/Terra/Luna (OpenAI pricing page and model cards), in - // upstream `gpt56Pricing` order (input, cache read, cache write, output). - // Above 272K input tokens the whole request bills 2x input / 1.5x output; - // cache writes bill at 1.25x uncached input. Sol was repriced from $5/$30 - // to $4/$20 on 2026-08-21. Dated usage before a model's repricing keeps - // the rates in `codex_pricing::codex_historical_pricing`. - m.insert( - "gpt-5.6-sol", - codex_pricing::gpt56_pricing((4e-6, 4e-7, 5e-6, 2e-5), (8e-6, 8e-7, 1e-5, 3e-5)), - ); - m.insert( - "gpt-5.6-terra", - codex_pricing::gpt56_pricing((2e-6, 2e-7, 2.5e-6, 1.2e-5), (4e-6, 4e-7, 5e-6, 1.8e-5)), - ); - m.insert( - "gpt-5.6-luna", - codex_pricing::gpt56_pricing((2e-7, 2e-8, 2.5e-7, 1.2e-6), (4e-7, 4e-8, 5e-7, 1.8e-6)), - ); - // Daybreak Cyber models (OpenAI pricing page). No long-context tier is - // published, and gpt-5.5-cyber lists no cache-write rate, so its writes - // bill at the input rate. - m.insert( - "gpt-5.6-cyber", - CodexPricing { - input_cost_per_token: 1.25e-5, - output_cost_per_token: 7.5e-5, - cache_read_input_cost_per_token: 1.25e-6, - cache_write_input_cost_per_token: Some(1.5625e-5), - display_label: None, - long_context: None, - }, - ); - m.insert( - "gpt-5.5-cyber", - CodexPricing { - input_cost_per_token: 1.25e-5, - output_cost_per_token: 7.5e-5, - cache_read_input_cost_per_token: 1.25e-6, - cache_write_input_cost_per_token: None, - display_label: None, - long_context: None, - }, - ); - // GPT-6 Astra pricing (OpenAI model card and pricing table). - // Long-context rates apply to the whole request above 272K input tokens; - // cache writes bill at 1.25x uncached input in both tiers. - m.insert( - "gpt-6-astra", - CodexPricing { - input_cost_per_token: 1e-5, - output_cost_per_token: 5e-5, - cache_read_input_cost_per_token: 1e-6, - cache_write_input_cost_per_token: Some(1.25e-5), - display_label: None, - long_context: Some(CodexLongContextRates { - input_cost_per_token: 2e-5, - output_cost_per_token: 7.5e-5, - cache_read_input_cost_per_token: 2e-6, - cache_write_input_cost_per_token: Some(2.5e-5), - }), - }, - ); - - m -}); + } + } -/// Claude model pricing table -static CLAUDE_PRICING: LazyLock> = LazyLock::new(|| { - let mut m = HashMap::new(); - - // Fable 5 - m.insert( - "claude-fable-5", - ClaudePricing { - input_cost_per_token: 1e-5, - output_cost_per_token: 5e-5, - cache_creation_input_cost_per_token: 1.25e-5, - cache_read_input_cost_per_token: 1e-6, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - - // Haiku 4.5 - m.insert( - "claude-haiku-4-5", - ClaudePricing { - input_cost_per_token: 1e-6, - output_cost_per_token: 5e-6, - cache_creation_input_cost_per_token: 1.25e-6, - cache_read_input_cost_per_token: 1e-7, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - m.insert( - "claude-haiku-4-5-20251001", - ClaudePricing { - input_cost_per_token: 1e-6, - output_cost_per_token: 5e-6, - cache_creation_input_cost_per_token: 1.25e-6, - cache_read_input_cost_per_token: 1e-7, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - - // Opus 4.6 - m.insert( - "claude-opus-4-6", - ClaudePricing { - input_cost_per_token: 5e-6, - output_cost_per_token: 2.5e-5, - cache_creation_input_cost_per_token: 6.25e-6, - cache_read_input_cost_per_token: 5e-7, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - m.insert( - "claude-opus-4-6-20260205", - ClaudePricing { - input_cost_per_token: 5e-6, - output_cost_per_token: 2.5e-5, - cache_creation_input_cost_per_token: 6.25e-6, - cache_read_input_cost_per_token: 5e-7, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - - // Opus 4.7 (same pricing as Opus 4.6) - m.insert( - "claude-opus-4-7", - ClaudePricing { - input_cost_per_token: 5e-6, - output_cost_per_token: 2.5e-5, - cache_creation_input_cost_per_token: 6.25e-6, - cache_read_input_cost_per_token: 5e-7, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - - // Opus 4.8 (same pricing as Opus 4.5/4.6/4.7) - m.insert( - "claude-opus-4-8", - ClaudePricing { - input_cost_per_token: 5e-6, - output_cost_per_token: 2.5e-5, - cache_creation_input_cost_per_token: 6.25e-6, - cache_read_input_cost_per_token: 5e-7, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - - // Opus 4.5 - m.insert( - "claude-opus-4-5", - ClaudePricing { - input_cost_per_token: 5e-6, - output_cost_per_token: 2.5e-5, - cache_creation_input_cost_per_token: 6.25e-6, - cache_read_input_cost_per_token: 5e-7, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - m.insert( - "claude-opus-4-5-20251101", - ClaudePricing { - input_cost_per_token: 5e-6, - output_cost_per_token: 2.5e-5, - cache_creation_input_cost_per_token: 6.25e-6, - cache_read_input_cost_per_token: 5e-7, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - - // Sonnet 4.5 (with tiered pricing at 200k tokens) - m.insert( - "claude-sonnet-4-5", - ClaudePricing { - input_cost_per_token: 3e-6, - output_cost_per_token: 1.5e-5, - cache_creation_input_cost_per_token: 3.75e-6, - cache_read_input_cost_per_token: 3e-7, - threshold_tokens: Some(200_000), - input_cost_per_token_above_threshold: Some(6e-6), - output_cost_per_token_above_threshold: Some(2.25e-5), - cache_creation_input_cost_per_token_above_threshold: Some(7.5e-6), - cache_read_input_cost_per_token_above_threshold: Some(6e-7), - }, - ); - m.insert( - "claude-sonnet-4-5-20250929", - ClaudePricing { - input_cost_per_token: 3e-6, - output_cost_per_token: 1.5e-5, - cache_creation_input_cost_per_token: 3.75e-6, - cache_read_input_cost_per_token: 3e-7, - threshold_tokens: Some(200_000), - input_cost_per_token_above_threshold: Some(6e-6), - output_cost_per_token_above_threshold: Some(2.25e-5), - cache_creation_input_cost_per_token_above_threshold: Some(7.5e-6), - cache_read_input_cost_per_token_above_threshold: Some(6e-7), - }, - ); - - // Sonnet 4.6 (same pricing as Sonnet 4.5, with 200k tier) - m.insert( - "claude-sonnet-4-6", - ClaudePricing { - input_cost_per_token: 3e-6, - output_cost_per_token: 1.5e-5, - cache_creation_input_cost_per_token: 3.75e-6, - cache_read_input_cost_per_token: 3e-7, - threshold_tokens: Some(200_000), - input_cost_per_token_above_threshold: Some(6e-6), - output_cost_per_token_above_threshold: Some(2.25e-5), - cache_creation_input_cost_per_token_above_threshold: Some(7.5e-6), - cache_read_input_cost_per_token_above_threshold: Some(6e-7), - }, - ); - - // Opus 4 - m.insert( - "claude-opus-4-20250514", - ClaudePricing { - input_cost_per_token: 1.5e-5, - output_cost_per_token: 7.5e-5, - cache_creation_input_cost_per_token: 1.875e-5, - cache_read_input_cost_per_token: 1.5e-6, - threshold_tokens: None, - input_cost_per_token_above_threshold: None, - output_cost_per_token_above_threshold: None, - cache_creation_input_cost_per_token_above_threshold: None, - cache_read_input_cost_per_token_above_threshold: None, - }, - ); - m.insert( - "claude-opus-4-1", - ClaudePricing { - input_cost_per_token: 1.5e-5, - output_cost_per_token: 7.5e-5, - cache_creation_input_cost_per_token: 1.875e-5, - cache_read_input_cost_per_token: 1.5e-6, + const fn with_cache_write(mut self, cache_write: f64) -> Self { + self.cache_write_input_cost_per_token = Some(cache_write); + self + } + + const fn with_label(mut self, label: &'static str) -> Self { + self.display_label = Some(label); + self + } + + const fn with_long_context( + mut self, + input: f64, + output: f64, + cache_read: f64, + cache_write: Option, + ) -> Self { + self.long_context = Some(CodexLongContextRates { + input_cost_per_token: input, + output_cost_per_token: output, + cache_read_input_cost_per_token: cache_read, + cache_write_input_cost_per_token: cache_write, + }); + self + } +} + +impl ClaudePricing { + /// Untiered per-token rates. + const fn new(input: f64, output: f64, cache_creation: f64, cache_read: f64) -> Self { + Self { + input_cost_per_token: input, + output_cost_per_token: output, + cache_creation_input_cost_per_token: cache_creation, + cache_read_input_cost_per_token: cache_read, threshold_tokens: None, input_cost_per_token_above_threshold: None, output_cost_per_token_above_threshold: None, cache_creation_input_cost_per_token_above_threshold: None, cache_read_input_cost_per_token_above_threshold: None, - }, - ); - - // Sonnet 4 - m.insert( - "claude-sonnet-4-20250514", - ClaudePricing { - input_cost_per_token: 3e-6, - output_cost_per_token: 1.5e-5, - cache_creation_input_cost_per_token: 3.75e-6, - cache_read_input_cost_per_token: 3e-7, - threshold_tokens: Some(200_000), - input_cost_per_token_above_threshold: Some(6e-6), - output_cost_per_token_above_threshold: Some(2.25e-5), - cache_creation_input_cost_per_token_above_threshold: Some(7.5e-6), - cache_read_input_cost_per_token_above_threshold: Some(6e-7), - }, - ); - - m + } + } + + /// Rates for requests above 200k tokens, in the same order as `new`. + const fn with_200k_tier( + mut self, + input: f64, + output: f64, + cache_creation: f64, + cache_read: f64, + ) -> Self { + self.threshold_tokens = Some(200_000); + self.input_cost_per_token_above_threshold = Some(input); + self.output_cost_per_token_above_threshold = Some(output); + self.cache_creation_input_cost_per_token_above_threshold = Some(cache_creation); + self.cache_read_input_cost_per_token_above_threshold = Some(cache_read); + self + } +} + +const GPT_5: CodexPricing = CodexPricing::new(1.25e-6, 1e-5, 1.25e-7); +const GPT_5_MINI: CodexPricing = CodexPricing::new(2.5e-7, 2e-6, 2.5e-8); +const GPT_5_2: CodexPricing = CodexPricing::new(1.75e-6, 1.4e-5, 1.75e-7); +// GPT-5.4 pricing (updated to match upstream 0.22). Like upstream, the +// whole request bills 2x input / 1.5x output above 272K input tokens. +const GPT_5_4: CodexPricing = + CodexPricing::new(2.5e-6, 1.5e-5, 2.5e-7).with_long_context(5e-6, 2.25e-5, 5e-7, None); +// GPT-5.4 Mini and Nano pricing (updated to match upstream 0.22) +const GPT_5_4_MINI: CodexPricing = CodexPricing::new(7.5e-7, 4.5e-6, 7.5e-8); +const GPT_5_4_NANO: CodexPricing = CodexPricing::new(2e-7, 1.25e-6, 2e-8); +const GPT_5_PRO_TIER: CodexPricing = CodexPricing::new(3e-5, 1.8e-4, 3e-5); +const GPT_CYBER: CodexPricing = CodexPricing::new(1.25e-5, 7.5e-5, 1.25e-6); + +/// Codex model pricing table +static CODEX_PRICING: LazyLock> = LazyLock::new(|| { + HashMap::from([ + ("gpt-5", GPT_5), + ("gpt-5-codex", GPT_5), + ("gpt-5-mini", GPT_5_MINI), + ("gpt-5-nano", CodexPricing::new(5e-8, 4e-7, 5e-9)), + ("gpt-5-pro", CodexPricing::new(1.5e-5, 1.2e-4, 1.5e-5)), + ("gpt-5.1", GPT_5), + ("gpt-5.1-codex", GPT_5), + ("gpt-5.1-codex-max", GPT_5), + ("gpt-5.1-codex-mini", GPT_5_MINI), + ("gpt-5.2", GPT_5_2), + ("gpt-5.2-codex", GPT_5_2), + ("gpt-5.2-pro", CodexPricing::new(2.1e-5, 1.68e-4, 2.1e-5)), + ("gpt-5.3-codex", GPT_5_2), + ( + "gpt-5.3-codex-spark", + CodexPricing::new(0.0, 0.0, 0.0).with_label("Research Preview"), + ), + ("gpt-5.4", GPT_5_4), + ("gpt-5.4-codex", GPT_5_4), + ("gpt-5.4-mini", GPT_5_4_MINI), + ("gpt-5.4-mini-codex", GPT_5_4_MINI), + ("gpt-5.4-nano", GPT_5_4_NANO), + ("gpt-5.4-nano-codex", GPT_5_4_NANO), + ("gpt-5.4-pro", GPT_5_PRO_TIER), + ( + "gpt-5.5", + CodexPricing::new(5e-6, 3e-5, 5e-7).with_long_context(1e-5, 4.5e-5, 1e-6, None), + ), + ("gpt-5.5-pro", GPT_5_PRO_TIER), + // GPT-5.6 Sol/Terra/Luna (OpenAI pricing page and model cards), in + // upstream `gpt56Pricing` order (input, cache read, cache write, output). + // Above 272K input tokens the whole request bills 2x input / 1.5x output; + // cache writes bill at 1.25x uncached input. Sol was repriced from $5/$30 + // to $4/$20 on 2026-08-21. Dated usage before a model's repricing keeps + // the rates in `codex_pricing::codex_historical_pricing`. + ( + "gpt-5.6-sol", + codex_pricing::gpt56_pricing((4e-6, 4e-7, 5e-6, 2e-5), (8e-6, 8e-7, 1e-5, 3e-5)), + ), + ( + "gpt-5.6-terra", + codex_pricing::gpt56_pricing((2e-6, 2e-7, 2.5e-6, 1.2e-5), (4e-6, 4e-7, 5e-6, 1.8e-5)), + ), + ( + "gpt-5.6-luna", + codex_pricing::gpt56_pricing((2e-7, 2e-8, 2.5e-7, 1.2e-6), (4e-7, 4e-8, 5e-7, 1.8e-6)), + ), + // Daybreak Cyber models (OpenAI pricing page). No long-context tier is + // published, and gpt-5.5-cyber lists no cache-write rate, so its writes + // bill at the input rate. + ("gpt-5.6-cyber", GPT_CYBER.with_cache_write(1.5625e-5)), + ("gpt-5.5-cyber", GPT_CYBER), + // GPT-6 Astra pricing (OpenAI model card and pricing table). + // Long-context rates apply to the whole request above 272K input tokens; + // cache writes bill at 1.25x uncached input in both tiers. + ( + "gpt-6-astra", + CodexPricing::new(1e-5, 5e-5, 1e-6) + .with_cache_write(1.25e-5) + .with_long_context(2e-5, 7.5e-5, 2e-6, Some(2.5e-5)), + ), + ]) +}); + +const CLAUDE_HAIKU_4_5: ClaudePricing = ClaudePricing::new(1e-6, 5e-6, 1.25e-6, 1e-7); +// Opus 4.5 through 4.8 share one price. +const CLAUDE_OPUS_4_5: ClaudePricing = ClaudePricing::new(5e-6, 2.5e-5, 6.25e-6, 5e-7); +// Sonnet 4 through 4.6 share one price, with tiered pricing at 200k tokens. +const CLAUDE_SONNET_4: ClaudePricing = + ClaudePricing::new(3e-6, 1.5e-5, 3.75e-6, 3e-7).with_200k_tier(6e-6, 2.25e-5, 7.5e-6, 6e-7); +const CLAUDE_OPUS_4: ClaudePricing = ClaudePricing::new(1.5e-5, 7.5e-5, 1.875e-5, 1.5e-6); + +/// Claude model pricing table +static CLAUDE_PRICING: LazyLock> = LazyLock::new(|| { + HashMap::from([ + ( + "claude-fable-5", + ClaudePricing::new(1e-5, 5e-5, 1.25e-5, 1e-6), + ), + ("claude-haiku-4-5", CLAUDE_HAIKU_4_5), + ("claude-haiku-4-5-20251001", CLAUDE_HAIKU_4_5), + ("claude-opus-4-6", CLAUDE_OPUS_4_5), + ("claude-opus-4-6-20260205", CLAUDE_OPUS_4_5), + ("claude-opus-4-7", CLAUDE_OPUS_4_5), + ("claude-opus-4-8", CLAUDE_OPUS_4_5), + ("claude-opus-4-5", CLAUDE_OPUS_4_5), + ("claude-opus-4-5-20251101", CLAUDE_OPUS_4_5), + ("claude-sonnet-4-5", CLAUDE_SONNET_4), + ("claude-sonnet-4-5-20250929", CLAUDE_SONNET_4), + ("claude-sonnet-4-6", CLAUDE_SONNET_4), + ("claude-opus-4-20250514", CLAUDE_OPUS_4), + ("claude-opus-4-1", CLAUDE_OPUS_4), + ("claude-sonnet-4-20250514", CLAUDE_SONNET_4), + ]) }); /// Cost usage pricing utilities @@ -793,24 +394,6 @@ impl CostUsagePricing { cached_input_tokens: u64, output_tokens: u64, pricing_date: NaiveDate, - ) -> Option { - Self::codex_cost_usd_at_date_with_pricing_snapshot( - model, - input_tokens, - cached_input_tokens, - output_tokens, - pricing_date, - None, - ) - } - - pub fn codex_cost_usd_at_date_with_pricing_snapshot( - model: &str, - input_tokens: u64, - cached_input_tokens: u64, - output_tokens: u64, - pricing_date: NaiveDate, - pricing_snapshot: Option<&models_dev_pricing::ModelsDevPricingSnapshot>, ) -> Option { Self::codex_cost_usd_at_date_with_cache_write_and_pricing_snapshot( model, @@ -819,7 +402,7 @@ impl CostUsagePricing { 0, output_tokens, pricing_date, - pricing_snapshot, + None, ) } diff --git a/rust/src/core/cost_pricing/codex.rs b/rust/src/core/cost_pricing/codex.rs index f5827acc46..2262980f29 100644 --- a/rust/src/core/cost_pricing/codex.rs +++ b/rust/src/core/cost_pricing/codex.rs @@ -1,7 +1,7 @@ use chrono::NaiveDate; use super::super::{codex_routed_pricing, models_dev_pricing}; -use super::{CODEX_PRICING, CodexLongContextRates, CodexPricing, CostUsagePricing}; +use super::{CODEX_PRICING, CodexPricing, CostUsagePricing}; pub(super) const CODEX_LONG_CONTEXT_THRESHOLD: u64 = 272_000; @@ -14,19 +14,14 @@ pub(super) type Gpt56Rates = (f64, f64, f64, f64); pub(super) const fn gpt56_pricing(standard: Gpt56Rates, long_context: Gpt56Rates) -> CodexPricing { let (input, cache_read, cache_write, output) = standard; let (long_input, long_cache_read, long_cache_write, long_output) = long_context; - CodexPricing { - input_cost_per_token: input, - output_cost_per_token: output, - cache_read_input_cost_per_token: cache_read, - cache_write_input_cost_per_token: Some(cache_write), - display_label: None, - long_context: Some(CodexLongContextRates { - input_cost_per_token: long_input, - output_cost_per_token: long_output, - cache_read_input_cost_per_token: long_cache_read, - cache_write_input_cost_per_token: Some(long_cache_write), - }), - } + CodexPricing::new(input, output, cache_read) + .with_cache_write(cache_write) + .with_long_context( + long_input, + long_output, + long_cache_read, + Some(long_cache_write), + ) } /// Upstream `codexHistoricalPricing`: the rates a model billed at before its From bfe654836efbc60a22104986ba11f9f3f154bb65 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:32:03 +0700 Subject: [PATCH 03/41] Route Codex JSONL parses through one entry point RCO-15: callers pass a CodexParseMode to JsonlScanner::parse_codex instead of picking one of seven forwarding wrappers; parse_codex_file stays as the from-zero convenience. Removes the two dead fork wrappers (RCO-02) and merges the two timestamp comparisons into codex_timestamp_cmp. --- rust/src/core/jsonl_scanner.rs | 2 +- rust/src/core/jsonl_scanner/codex.rs | 262 ++---------------- rust/src/core/jsonl_scanner/codex/parser.rs | 11 +- rust/src/core/jsonl_scanner/tests.rs | 32 ++- rust/src/cost_scanner/codex.rs | 73 ++--- rust/src/cost_scanner/codex/logical_target.rs | 12 +- 6 files changed, 94 insertions(+), 298 deletions(-) diff --git a/rust/src/core/jsonl_scanner.rs b/rust/src/core/jsonl_scanner.rs index 3b25ca2ff4..ce06b24fc9 100755 --- a/rust/src/core/jsonl_scanner.rs +++ b/rust/src/core/jsonl_scanner.rs @@ -613,12 +613,12 @@ impl CostUsageDayRange { /// JSONL Scanner for cost/usage logs pub struct JsonlScanner; pub(crate) mod codex; -pub(crate) use codex::CodexForkParseResume; pub(crate) use codex::priority::CodexPriorityOverlay; pub use codex::priority::{ CODEX_PRIORITY_COMPLETED_MODEL_RETENTION_LIMIT, CodexPriorityCursorAnchor, CodexPriorityTurnMetadata, CodexPriorityTurnsCursor, }; +pub(crate) use codex::{CodexForkParseResume, CodexParseMode}; mod save_skip; pub(crate) use codex::source_rows::{ read_source_rows, recover_rows, row_cache, row_cache_matches, row_cache_needs_recovery, diff --git a/rust/src/core/jsonl_scanner/codex.rs b/rust/src/core/jsonl_scanner/codex.rs index 942fdeee9c..d0374a14f9 100644 --- a/rust/src/core/jsonl_scanner/codex.rs +++ b/rust/src/core/jsonl_scanner/codex.rs @@ -9,7 +9,8 @@ use helpers::{ BoundedJsonlLine, CODEX_JSONL_MAX_LINE_BYTES, nonempty_json_string, parse_rfc3339_timestamp, read_bounded_jsonl_line, read_bounded_jsonl_line_until, session_meta_field, }; -use parser::{CodexParseMode, CodexParserState}; +pub(crate) use parser::CodexParseMode; +use parser::CodexParserState; /// Saved cursor and parser state of an unfinished parent-baseline fork parse. /// Restoring all of it accounts the remaining bytes exactly as one @@ -379,28 +380,10 @@ impl JsonlScanner { Some(format!("{:?}:{}", metadata.modified().ok(), metadata.len())) } - /// Compare RFC3339 timestamps using parsed instants. Malformed timestamps - /// are unsafe for fork-baseline reconciliation and therefore fail closed. - pub(crate) fn codex_timestamp_at_or_before(earlier: &str, later: &str) -> bool { - match ( - parse_rfc3339_timestamp(earlier), - parse_rfc3339_timestamp(later), - ) { - (Some(earlier), Some(later)) => earlier <= later, - _ => false, - } - } - - /// Strict counterpart of [`Self::codex_timestamp_at_or_before`]: `earlier` - /// is before `later`. Malformed timestamps fail closed. - pub(crate) fn codex_timestamp_before(earlier: &str, later: &str) -> bool { - match ( - parse_rfc3339_timestamp(earlier), - parse_rfc3339_timestamp(later), - ) { - (Some(earlier), Some(later)) => earlier < later, - _ => false, - } + /// Order two RFC3339 timestamps by parsed instant. `None` when either is + /// malformed, so fork-baseline reconciliation fails closed. + pub(crate) fn codex_timestamp_cmp(earlier: &str, later: &str) -> Option { + Some(parse_rfc3339_timestamp(earlier)?.cmp(&parse_rfc3339_timestamp(later)?)) } /// Parse a Codex JSONL file @@ -411,246 +394,33 @@ impl JsonlScanner { initial_model: Option, initial_totals: Option, ) -> std::io::Result { - Self::parse_codex_file_with_state( + Self::parse_codex( file_path, range, - start_offset, - initial_model, - initial_totals, - None, - None, - None, - ) - } - - /// Parse a Codex file while retaining the timestamp-order state of an - /// already decoded prefix. A known prefix only pays for the append - /// boundary and newly read token events; an unknown legacy prefix is - /// intentionally rejected by the caller and should be parsed from zero. - #[allow( - clippy::too_many_arguments, - reason = "resume state mirrors the persisted parser cache" - )] - pub fn parse_codex_file_with_state( - file_path: &Path, - range: &CostUsageDayRange, - start_offset: i64, - initial_model: Option, - initial_totals: Option, - previous_token_timestamp: Option, - token_timestamps_monotonic: Option, - cancel: Option<&AtomicBool>, - ) -> std::io::Result { - Self::parse_codex_file_with_state_bounded( - file_path, - range, - start_offset, - initial_model, - initial_totals, - previous_token_timestamp, - token_timestamps_monotonic, - cancel, - None, - ) - } - - /// Parse a Codex file with an optional cap on bytes newly consumed this pass. - /// The reader may finish the current bounded JSONL line before yielding. - #[allow( - clippy::too_many_arguments, - reason = "resume state mirrors the persisted parser cache" - )] - pub fn parse_codex_file_with_state_bounded( - file_path: &Path, - range: &CostUsageDayRange, - start_offset: i64, - initial_model: Option, - initial_totals: Option, - previous_token_timestamp: Option, - token_timestamps_monotonic: Option, - cancel: Option<&AtomicBool>, - max_bytes_to_read: Option, - ) -> std::io::Result { - Self::parse_codex_file_with_state_bounded_internal( - file_path, - range, - cancel, - None, - max_bytes_to_read, - CodexParseMode::Standard { - start_offset, - initial_model, - initial_totals, - previous_token_timestamp, - token_timestamps_monotonic, - }, - ) - } - - /// Parse a Codex file against a caller-owned frozen target. The target is - /// intentionally separate from the current physical EOF so an active - /// rollout cannot make a bounded catch-up pass chase its own growth. - #[allow( - clippy::too_many_arguments, - reason = "resume state mirrors the persisted parser cache" - )] - pub(crate) fn parse_codex_file_with_state_bounded_target( - file_path: &Path, - range: &CostUsageDayRange, - start_offset: i64, - initial_model: Option, - initial_totals: Option, - previous_token_timestamp: Option, - token_timestamps_monotonic: Option, - cancel: Option<&AtomicBool>, - scan_target_size: Option, - max_bytes_to_read: Option, - ) -> std::io::Result { - Self::parse_codex_file_with_state_bounded_internal( - file_path, - range, - cancel, - scan_target_size, - max_bytes_to_read, CodexParseMode::Standard { start_offset, initial_model, initial_totals, - previous_token_timestamp, - token_timestamps_monotonic, + previous_token_timestamp: None, + token_timestamps_monotonic: None, }, - ) - } - - /// Parse a forked Codex child from byte zero with a parent cumulative - /// baseline. This is intentionally separate from ordinary append-resume - /// parsing so existing non-fork semantics remain unchanged. - #[allow( - clippy::too_many_arguments, - reason = "fork parse state mirrors the persisted parser cache" - )] - pub(crate) fn parse_codex_file_with_state_bounded_fork( - file_path: &Path, - range: &CostUsageDayRange, - initial_totals: CodexTotals, - cancel: Option<&AtomicBool>, - max_bytes_to_read: Option, - ) -> std::io::Result { - Self::parse_codex_file_with_state_bounded_internal( - file_path, - range, - cancel, None, - max_bytes_to_read, - CodexParseMode::ParentBaseline { - baseline: initial_totals, - paginated_continuation: false, - remaining_inherited_totals: None, - }, - ) - } - - /// Fork equivalent of [`Self::parse_codex_file_with_state_bounded_target`]. - #[allow( - clippy::too_many_arguments, - reason = "fork parse state mirrors the persisted parser cache" - )] - pub(crate) fn parse_codex_file_with_state_bounded_fork_target( - file_path: &Path, - range: &CostUsageDayRange, - initial_totals: CodexTotals, - cancel: Option<&AtomicBool>, - scan_target_size: Option, - max_bytes_to_read: Option, - ) -> std::io::Result { - Self::parse_codex_file_with_state_bounded_fork_target_with_accounting( - file_path, - range, - initial_totals, - false, None, - cancel, - scan_target_size, - max_bytes_to_read, - ) - } - - pub(crate) fn parse_codex_file_with_inferred_fork_baseline( - file_path: &Path, - range: &CostUsageDayRange, - subagent_history_start_ordinal: Option, - cancel: Option<&AtomicBool>, - scan_target_size: Option, - max_bytes_to_read: Option, - ) -> std::io::Result { - Self::parse_codex_file_with_state_bounded_internal( - file_path, - range, - cancel, - scan_target_size, - max_bytes_to_read, - CodexParseMode::InferSubagent { - start_ordinal: subagent_history_start_ordinal, - }, - ) - } - - /// Fork equivalent with persisted paginated-continuation accounting. - #[allow( - clippy::too_many_arguments, - reason = "fork parse state mirrors the persisted parser cache" - )] - pub(crate) fn parse_codex_file_with_state_bounded_fork_target_with_accounting( - file_path: &Path, - range: &CostUsageDayRange, - initial_totals: CodexTotals, - paginated_continuation: bool, - remaining_inherited_totals: Option, - cancel: Option<&AtomicBool>, - scan_target_size: Option, - max_bytes_to_read: Option, - ) -> std::io::Result { - Self::parse_codex_file_with_state_bounded_internal( - file_path, - range, - cancel, - scan_target_size, - max_bytes_to_read, - CodexParseMode::ParentBaseline { - baseline: initial_totals, - paginated_continuation, - remaining_inherited_totals, - }, - ) - } - - /// Continue an unfinished fork parse at its saved cursor. Upstream 0.67.0 - /// resumes a resolved fork the same way instead of rereading its prefix. - pub(crate) fn parse_codex_fork_resume( - file_path: &Path, - range: &CostUsageDayRange, - resume: CodexForkParseResume, - cancel: Option<&AtomicBool>, - scan_target_size: Option, - max_bytes_to_read: Option, - ) -> std::io::Result { - Self::parse_codex_file_with_state_bounded_internal( - file_path, - range, - cancel, - scan_target_size, - max_bytes_to_read, - CodexParseMode::ResumeParentBaseline(resume), + None, ) } - fn parse_codex_file_with_state_bounded_internal( + /// Parse a Codex file in `mode`. `scan_target_size` freezes the end of the + /// pass below the physical EOF so an active rollout cannot make a bounded + /// catch-up pass chase its own growth. `max_bytes_to_read` caps the bytes + /// newly consumed; the reader may finish the current line before yielding. + pub(crate) fn parse_codex( file_path: &Path, range: &CostUsageDayRange, + mode: CodexParseMode, cancel: Option<&AtomicBool>, scan_target_size: Option, max_bytes_to_read: Option, - mode: CodexParseMode, ) -> std::io::Result { let file = File::open(file_path)?; // Session JSONL files are bounded by the cache budget; sizes fit i64. diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index 64a2fe3edc..948c7744e3 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -35,7 +35,11 @@ pub(super) struct CodexParserState { fork_parse_baseline: Option, } -pub(super) enum CodexParseMode { +pub(crate) enum CodexParseMode { + /// Resume from `start_offset` with the timestamp-order state of an already + /// decoded prefix. A known prefix only pays for the append boundary and + /// newly read token events; an unknown legacy prefix is intentionally + /// rejected by the caller and should be parsed from zero. Standard { start_offset: i64, initial_model: Option, @@ -43,12 +47,17 @@ pub(super) enum CodexParseMode { previous_token_timestamp: Option, token_timestamps_monotonic: Option, }, + /// Parse a forked child from byte zero with a parent cumulative baseline. + /// Kept separate from append-resume parsing so non-fork semantics remain + /// unchanged. ParentBaseline { baseline: CodexTotals, paginated_continuation: bool, remaining_inherited_totals: Option, }, /// Continue an unfinished `ParentBaseline` parse at its saved cursor. + /// Upstream 0.67.0 resumes a resolved fork the same way instead of + /// rereading its prefix. ResumeParentBaseline(CodexForkParseResume), InferSubagent { start_ordinal: Option, diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 1c3fadc082..88fa8641a9 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -715,14 +715,18 @@ fn codex_append_timestamp_state_is_output_equivalent_and_boundary_only() { ) .unwrap(); - let appended = JsonlScanner::parse_codex_file_with_state( + let appended = JsonlScanner::parse_codex( file.path(), &range, - prefix.parsed_bytes, - prefix.last_model.clone(), - prefix.last_totals.clone(), - prefix.last_token_timestamp.clone(), - prefix.token_timestamps_monotonic, + CodexParseMode::Standard { + start_offset: prefix.parsed_bytes, + initial_model: prefix.last_model.clone(), + initial_totals: prefix.last_totals.clone(), + previous_token_timestamp: prefix.last_token_timestamp.clone(), + token_timestamps_monotonic: prefix.token_timestamps_monotonic, + }, + None, + None, None, ) .expect("parse appended suffix"); @@ -768,14 +772,18 @@ fn codex_parse_publishes_only_the_committed_prefix_before_an_incomplete_tail() { assert!(partial.is_complete, "the logical prefix is complete"); writeln!(file, "{}", &complete_tail[split..]).unwrap(); - let resumed = JsonlScanner::parse_codex_file_with_state( + let resumed = JsonlScanner::parse_codex( file.path(), &range, - partial.parsed_bytes, - partial.last_model, - partial.last_totals, - partial.last_token_timestamp, - partial.token_timestamps_monotonic, + CodexParseMode::Standard { + start_offset: partial.parsed_bytes, + initial_model: partial.last_model, + initial_totals: partial.last_totals, + previous_token_timestamp: partial.last_token_timestamp, + token_timestamps_monotonic: partial.token_timestamps_monotonic, + }, + None, + None, None, ) .expect("resume completed tail"); diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 14ac8780f4..cef3621c56 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -1,5 +1,7 @@ use super::*; -use crate::core::{CodexForkAccountingState, CodexSessionLineage, CodexSessionMetadata}; +use crate::core::{ + CodexForkAccountingState, CodexParseMode, CodexSessionLineage, CodexSessionMetadata, +}; mod cache_days; mod flat_sources; @@ -671,14 +673,16 @@ impl CostScanner { && JsonlScanner::is_line_boundary_offset(path, start_offset) { let resumable_target_size = codex_resumable_scan_target_size(size, entry); - let parse_result = match JsonlScanner::parse_codex_file_with_state_bounded_target( + let parse_result = match JsonlScanner::parse_codex( path, range, - start_offset, - entry.last_model.clone(), - entry.last_totals.clone(), - entry.codex_last_token_timestamp.clone(), - entry.codex_token_timestamps_monotonic, + CodexParseMode::Standard { + start_offset, + initial_model: entry.last_model.clone(), + initial_totals: entry.last_totals.clone(), + previous_token_timestamp: entry.codex_last_token_timestamp.clone(), + token_timestamps_monotonic: entry.codex_token_timestamps_monotonic, + }, cancel, resumable_target_size, max_bytes_to_read, @@ -765,15 +769,18 @@ impl CostScanner { }) .flatten(); let parse_result = match match &accounting_mode { - CodexAccountingMode::Standard => JsonlScanner::parse_codex_file_with_state_bounded( + CodexAccountingMode::Standard => JsonlScanner::parse_codex( path, range, - 0, - None, - None, - None, - None, + CodexParseMode::Standard { + start_offset: 0, + initial_model: None, + initial_totals: None, + previous_token_timestamp: None, + token_timestamps_monotonic: None, + }, cancel, + None, max_bytes_to_read, ), CodexAccountingMode::Baseline { @@ -781,40 +788,42 @@ impl CostScanner { paginated_continuation, .. } => match fork_resume { - Some(resume) => JsonlScanner::parse_codex_fork_resume( + Some(resume) => JsonlScanner::parse_codex( path, range, - resume.parse, + CodexParseMode::ResumeParentBaseline(resume.parse), cancel, Some(resume.target_size), max_bytes_to_read, ), None => { - JsonlScanner::parse_codex_file_with_state_bounded_fork_target_with_accounting( + JsonlScanner::parse_codex( path, range, - baseline.clone(), - *paginated_continuation, - // A parse from byte zero replays the inherited counters - // itself. Only a resumed parse carries their used-up - // remainder, as upstream restores fork state only then. - None, + CodexParseMode::ParentBaseline { + baseline: baseline.clone(), + paginated_continuation: *paginated_continuation, + // A parse from byte zero replays the inherited counters + // itself. Only a resumed parse carries their used-up + // remainder, as upstream restores fork state only then. + remaining_inherited_totals: None, + }, cancel, parse_target_size, max_bytes_to_read, ) } }, - CodexAccountingMode::InferSubagent { start_ordinal } => { - JsonlScanner::parse_codex_file_with_inferred_fork_baseline( - path, - range, - *start_ordinal, - cancel, - parse_target_size, - max_bytes_to_read, - ) - } + CodexAccountingMode::InferSubagent { start_ordinal } => JsonlScanner::parse_codex( + path, + range, + CodexParseMode::InferSubagent { + start_ordinal: *start_ordinal, + }, + cancel, + parse_target_size, + max_bytes_to_read, + ), CodexAccountingMode::Unresolved => { unreachable!("unresolved forks return before parsing") } diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index c0a8ad5b3e..d9010b89c2 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -513,10 +513,8 @@ impl CodexLineagePlanner { let child_fork_timestamp = child_fork_timestamp?; if let Some(last_token_timestamp) = usage.codex_last_token_timestamp.as_deref() - && JsonlScanner::codex_timestamp_at_or_before( - last_token_timestamp, - child_fork_timestamp, - ) + && JsonlScanner::codex_timestamp_cmp(last_token_timestamp, child_fork_timestamp) + .is_some_and(std::cmp::Ordering::is_le) { return usage.last_totals.clone(); } @@ -525,11 +523,13 @@ impl CodexLineagePlanner { // its counters are still the origin it inherited when it forked. let (state, inherited) = fork_origin?; let forked_before_cutoff = usage.codex_fork_timestamp.as_deref().is_some_and(|forked| { - JsonlScanner::codex_timestamp_at_or_before(forked, child_fork_timestamp) + JsonlScanner::codex_timestamp_cmp(forked, child_fork_timestamp) + .is_some_and(std::cmp::Ordering::is_le) }); let first_own_token_after_cutoff = usage.codex_last_token_timestamp.is_none() || state.first_token_timestamp.as_deref().is_some_and(|first| { - JsonlScanner::codex_timestamp_before(child_fork_timestamp, first) + JsonlScanner::codex_timestamp_cmp(child_fork_timestamp, first) + .is_some_and(std::cmp::Ordering::is_lt) }); (forked_before_cutoff && first_own_token_after_cutoff).then(|| inherited.clone()) } From 18dd610ee0dc6920e03faa3f4d12b776c7a5a07e Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:49:54 +0700 Subject: [PATCH 04/41] Pin Claude root walk results across scan paths --- rust/src/cost_scanner/tests.rs | 3 + rust/src/cost_scanner/tests/claude_walk.rs | 179 +++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 rust/src/cost_scanner/tests/claude_walk.rs diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 70d118d239..5e0eb3fe94 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -3551,6 +3551,9 @@ mod claude_swap; #[path = "tests/claude_today.rs"] mod claude_today; #[cfg(test)] +#[path = "tests/claude_walk.rs"] +mod claude_walk; +#[cfg(test)] #[path = "tests/copied_prefix.rs"] mod copied_prefix; #[cfg(test)] diff --git a/rust/src/cost_scanner/tests/claude_walk.rs b/rust/src/cost_scanner/tests/claude_walk.rs new file mode 100644 index 0000000000..315dc0295e --- /dev/null +++ b/rust/src/cost_scanner/tests/claude_walk.rs @@ -0,0 +1,179 @@ +//! Every Claude history path (summary, chart snapshot, daily cost, daily +//! tokens) walks the same transcript roots. These fixtures pin what each path +//! reports for duplicates, incomplete requests, timestamp-less rows and +//! missing roots. + +use super::*; + +const CHILD_MARKER: &str = "CODEXBAR_CLAUDE_WALK_TEST_CHILD"; +const CHILD_DONE: &str = "claude walk fixtures verified"; + +fn completed(timestamp: Option<&str>, id: &str) -> String { + let timestamp = timestamp + .map(|timestamp| format!(r#""timestamp":"{timestamp}","#)) + .unwrap_or_default(); + format!( + r#"{{"type":"assistant",{timestamp}"requestId":"req_{id}","message":{{"id":"msg_{id}","model":"claude-sonnet-4-6","usage":{{"input_tokens":1000,"output_tokens":500}}}}}}"# + ) +} + +fn preliminary(timestamp: &str, id: &str) -> String { + format!( + r#"{{"type":"assistant","timestamp":"{timestamp}","requestId":"req_{id}","message":{{"id":"msg_{id}","model":"claude-sonnet-4-6","stop_reason":null,"usage":{{"input_tokens":1000}}}}}}"# + ) +} + +fn write_transcript(path: &Path, lines: &[String]) { + std::fs::write(path, format!("{}\n", lines.join("\n"))).unwrap(); +} + +fn summary_line(summary: &CostSummary) -> String { + format!( + "sessions={} input={} output={} cost={:.6} coverage={} known_zero={} incomplete={}", + summary.sessions_count, + summary.input_tokens, + summary.output_tokens, + summary.total_cost_usd, + summary.history_coverage_established, + summary.known_zero, + summary.incomplete_request_count, + ) +} + +/// Each history path's view of the fixtures. The daily series span two days, +/// so yesterday shows whether a complete scan zero-fills an empty day. +fn observe(today: &str) -> String { + let summary = CostScanner::new(1).scan_claude(); + let chart = CostScanner::new(1).scan_claude_chart_snapshot_with_cancel(None); + let (daily_cost, daily_incomplete) = get_daily_cost_and_incomplete_history("claude", 2); + let (daily_tokens, tokens_incomplete) = get_daily_token_history("claude", 2); + let daily_cost: Vec<_> = daily_cost + .iter() + .map(|(day, cost)| (day == today, cost.map(|cost| format!("{cost:.6}")))) + .collect(); + let daily_tokens: Vec<_> = daily_tokens + .iter() + .map(|(day, tokens)| (day == today, *tokens)) + .collect(); + format!( + "summary {}\nchart {}\nchart today={} cost={:?} incomplete={} quota={} quota_coverage={}\ndaily cost={daily_cost:?} incomplete={:?}\ndaily tokens={daily_tokens:?} incomplete={tokens_incomplete}\n", + summary_line(&summary), + summary_line(&chart.summary), + chart.today.tokens, + chart.today.cost_usd.map(|cost| format!("{cost:.6}")), + chart.daily_incomplete.len(), + chart.quota_history.records.len(), + chart.quota_history.history_coverage_established, + daily_incomplete + .iter() + .map(|(day, count)| (day == today, *count)) + .collect::>(), + ) +} + +fn child_body() { + use crate::cost_reporting_period::cost_bucket_zone; + let config_dir = PathBuf::from(std::env::var_os("CLAUDE_CONFIG_DIR").unwrap()); + let projects = config_dir.join("projects"); + let project = projects.join("fixture"); + std::fs::create_dir_all(&project).unwrap(); + let now = Utc::now(); + let today = super::super::today::day_key(cost_bucket_zone().date(now)); + let stamp = now.to_rfc3339(); + let ts = Some(stamp.as_str()); + + // A: req1 appears in three files and counts once; c.jsonl only repeats + // it, so it is not a session. + write_transcript( + &project.join("a.jsonl"), + &[completed(ts, "1"), completed(ts, "2")], + ); + write_transcript( + &project.join("b.jsonl"), + &[completed(ts, "1"), completed(ts, "3")], + ); + write_transcript(&project.join("c.jsonl"), &[completed(ts, "1")]); + assert_eq!( + observe(&today), + "\ +summary sessions=2 input=3000 output=1500 cost=0.031500 coverage=true known_zero=false incomplete=0 +chart sessions=2 input=3000 output=1500 cost=0.031500 coverage=true known_zero=false incomplete=0 +chart today=4500 cost=Some(\"0.031500\") incomplete=0 quota=3 quota_coverage=true +daily cost=[(false, Some(\"0.000000\")), (true, Some(\"0.031500\"))] incomplete=[] +daily tokens=[(false, 0), (true, 4500)] incomplete=false +" + ); + + // B: a preliminary row stays incomplete unless another file completes + // the same request. + write_transcript( + &project.join("d.jsonl"), + &[preliminary(&stamp, "4"), preliminary(&stamp, "2")], + ); + assert_eq!( + observe(&today), + "\ +summary sessions=2 input=3000 output=1500 cost=0.031500 coverage=false known_zero=false incomplete=1 +chart sessions=2 input=3000 output=1500 cost=0.031500 coverage=false known_zero=false incomplete=1 +chart today=4500 cost=Some(\"0.031500\") incomplete=1 quota=3 quota_coverage=false +daily cost=[(false, None), (true, Some(\"0.031500\"))] incomplete=[(true, 1)] +daily tokens=[(false, 0), (true, 4500)] incomplete=true +" + ); + + // C: rows without a timestamp still count but keep coverage unknown. + write_transcript(&project.join("d.jsonl"), &[]); + write_transcript( + &project.join("e.jsonl"), + &[completed(None, "5"), completed(None, "6")], + ); + assert_eq!( + observe(&today), + "\ +summary sessions=3 input=5000 output=2500 cost=0.052500 coverage=false known_zero=false incomplete=0 +chart sessions=3 input=5000 output=2500 cost=0.052500 coverage=false known_zero=false incomplete=0 +chart today=4500 cost=Some(\"0.031500\") incomplete=0 quota=3 quota_coverage=false +daily cost=[(false, None), (true, Some(\"0.031500\"))] incomplete=[] +daily tokens=[(false, 0), (true, 4500)] incomplete=true +" + ); + + // D: no transcript root at all. + std::fs::rename(&projects, config_dir.join("projects-off")).unwrap(); + assert_eq!( + observe(&today), + "\ +summary sessions=0 input=0 output=0 cost=0.000000 coverage=false known_zero=false incomplete=0 +chart sessions=0 input=0 output=0 cost=0.000000 coverage=false known_zero=false incomplete=0 +chart today=0 cost=None incomplete=0 quota=0 quota_coverage=false +daily cost=[(false, None), (true, None)] incomplete=[] +daily tokens=[(false, 0), (true, 0)] incomplete=true +" + ); + println!("{CHILD_DONE}"); +} + +#[test] +fn claude_root_walk_results_agree_across_scan_paths() { + if std::env::var_os(CHILD_MARKER).is_some() { + child_body(); + return; + } + let config_dir = tempfile::tempdir().unwrap(); + let test_thread = std::thread::current(); + let test_name = test_thread.name().expect("test harness names this thread"); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", test_name, "--nocapture", "--test-threads=1"]) + .env(CHILD_MARKER, "1") + .env("CLAUDE_CONFIG_DIR", config_dir.path()) + // Every home-relative root (claude-swap, Pi, OMP) resolves here. + .env("CODEXBAR_TEST_HOME", config_dir.path()) + .output() + .expect("spawn isolated exact-test child"); + assert!( + output.status.success() && String::from_utf8_lossy(&output.stdout).contains(CHILD_DONE), + "fixture child failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} From 4b356d6560b2457391bdd472abc4cba43a027d39 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:00:22 +0700 Subject: [PATCH 05/41] Walk Claude transcript roots through one record stream The summary, chart snapshot, daily cost and daily token paths each re-implemented the root walk, dedup set, pricing resolver and failure accounting. They now share walk_claude_records and keep only their own aggregation. Failed records are counted per record everywhere; the count is only read through is_complete, so results are unchanged (pinned by claude_root_walk_results_agree_across_scan_paths). --- rust/src/cost_scanner.rs | 302 +++++++++++---------------------- rust/src/cost_scanner/tests.rs | 33 ++++ 2 files changed, 134 insertions(+), 201 deletions(-) diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index 2c028b086a..2d271e18c3 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -550,6 +550,18 @@ impl ClaudeFileScanResult { } } +/// One de-duplicated pass over every Claude transcript root. +#[derive(Debug, Default)] +struct ClaudeRecordWalk { + roots_present: bool, + scan: ClaudeFileScanResult, + /// Keys of every counted record; they resolve incomplete requests. + completed_keys: HashSet, + incomplete: ClaudeIncompleteTracker, + /// Files that contributed at least one record. + files_with_usage: u32, +} + #[derive(Debug, Clone)] pub struct CostScanner { /// The window every scan resolves at its own start (never cached as dates). @@ -663,7 +675,6 @@ impl CostScanner { cancel: Option<&AtomicBool>, include_pi_sessions: bool, ) -> CostSummary { - let roots = self.claude_projects_roots(); let mut summary = CostSummary::default(); let now = Utc::now(); let window = self.claude_window(now); @@ -672,48 +683,13 @@ impl CostScanner { summary.period_start = Some(window.start); summary.period_end = Some(window.end); - // Walk through projects directory, de-duplicating usage records - // that appear across multiple files. - let mut claude_scan = ClaudeFileScanResult::default(); - let mut incomplete = ClaudeIncompleteTracker::default(); - let mut completed_keys = HashSet::new(); - if !roots.paths.is_empty() { - let mut seen = HashSet::new(); - let mut pricing = ClaudeScanPricingResolver::default(); - let traversal_read_failures = { - let mut handle_file = |path: &Path| { - let mut aggregation_complete = true; - let mut file_result = scan_claude_file_with_pricing( - path, - &cutoff, - &mut seen, - cancel, - &mut pricing, - &mut incomplete, - |record| { - aggregation_complete &= record.timestamp.is_some(); - aggregation_complete &= - add_claude_record_to_summary(&mut summary, record); - }, - ); - if !aggregation_complete { - file_result.aggregation_failures = - file_result.aggregation_failures.saturating_add(1); - } - if file_result.counted > 0 { - summary.sessions_count += 1; - } - claude_scan.absorb(file_result); - }; - self.walk_claude_roots(&roots.paths, &cutoff, cancel, &mut handle_file) - }; - claude_scan.read_failures = claude_scan - .read_failures - .saturating_add(roots.read_failures) - .saturating_add(traversal_read_failures); - completed_keys = seen; - } - incomplete.resolve(&completed_keys).apply_to(&mut summary); + let walk = self.walk_claude_records(&cutoff, cancel, |record| { + add_claude_record_to_summary(&mut summary, record) + }); + summary.sessions_count = walk.files_with_usage; + walk.incomplete + .resolve(&walk.completed_keys) + .apply_to(&mut summary); // OMP / pi-compatible anthropic rows, deduped across shared files. if include_pi_sessions { @@ -734,8 +710,8 @@ impl CostScanner { // than turning a partial zero into a known zero. finalize_claude_summary( &mut summary, - !roots.paths.is_empty(), - claude_scan, + walk.roots_present, + walk.scan, is_cancelled(cancel), ); @@ -748,7 +724,6 @@ impl CostScanner { &self, cancel: Option<&AtomicBool>, ) -> ClaudeChartSnapshot { - let roots = self.claude_projects_roots(); let now = Utc::now(); let window = self.claude_window(now); let cutoff = window.cutoff; @@ -776,56 +751,18 @@ impl CostScanner { } let mut quota_records = Vec::new(); - let mut scan_result = ClaudeFileScanResult::default(); - let mut incomplete = ClaudeIncompleteTracker::default(); - let mut completed_keys = HashSet::new(); - if !roots.paths.is_empty() { - let mut seen = HashSet::new(); - let mut pricing = ClaudeScanPricingResolver::default(); - let traversal_read_failures = - self.walk_claude_roots(&roots.paths, &cutoff, cancel, &mut |path| { - let mut file_has_usage = false; - let mut aggregation_complete = true; - let mut file_result = scan_claude_file_with_pricing( - path, - &cutoff, - &mut seen, - cancel, - &mut pricing, - &mut incomplete, - |record| { - file_has_usage = true; - aggregation_complete &= record.timestamp.is_some(); - aggregation_complete &= - add_claude_record_to_summary(&mut summary, record); - aggregation_complete &= add_claude_record_to_daily_costs( - &mut daily_cost, - &mut unknown_cost_dates, - record, - ); - aggregation_complete &= - add_claude_record_to_daily_tokens(&mut daily_tokens, record); - if let Some(quota_record) = quota_history_record_from_usage(record) { - quota_records.push(quota_record); - } - }, - ); - if !aggregation_complete { - file_result.aggregation_failures = - file_result.aggregation_failures.saturating_add(1); - } - if file_has_usage { - summary.sessions_count += 1; - } - scan_result.absorb(file_result); - }); - scan_result.read_failures = scan_result - .read_failures - .saturating_add(roots.read_failures) - .saturating_add(traversal_read_failures); - completed_keys = seen; - } - let incomplete_report = incomplete.resolve(&completed_keys); + let walk = self.walk_claude_records(&cutoff, cancel, |record| { + let summary_ok = add_claude_record_to_summary(&mut summary, record); + let cost_ok = + add_claude_record_to_daily_costs(&mut daily_cost, &mut unknown_cost_dates, record); + let tokens_ok = add_claude_record_to_daily_tokens(&mut daily_tokens, record); + if let Some(quota_record) = quota_history_record_from_usage(record) { + quota_records.push(quota_record); + } + summary_ok && cost_ok && tokens_ok + }); + summary.sessions_count = walk.files_with_usage; + let incomplete_report = walk.incomplete.resolve(&walk.completed_keys); incomplete_report.apply_to(&mut summary); crate::pi_session_cost::scan_pi_compatible_into( @@ -836,12 +773,11 @@ impl CostScanner { &mut HashSet::new(), ); - let complete = - !roots.paths.is_empty() && !is_cancelled(cancel) && scan_result.is_complete(); + let complete = walk.roots_present && !is_cancelled(cancel) && walk.scan.is_complete(); finalize_claude_summary( &mut summary, - !roots.paths.is_empty(), - scan_result, + walk.roots_present, + walk.scan, is_cancelled(cancel), ); if complete { @@ -953,6 +889,57 @@ impl CostScanner { ) } + /// Stream every in-window Claude usage record, de-duplicated across all + /// transcript roots, into `on_record`. A record `on_record` cannot + /// aggregate (it returns `false`), or one without a timestamp, leaves the + /// walk incomplete. + fn walk_claude_records( + &self, + cutoff: &DateTime, + cancel: Option<&AtomicBool>, + mut on_record: impl FnMut(&ClaudeUsageRecord) -> bool, + ) -> ClaudeRecordWalk { + let roots = self.claude_projects_roots(); + let mut walk = ClaudeRecordWalk { + roots_present: !roots.paths.is_empty(), + ..ClaudeRecordWalk::default() + }; + if !walk.roots_present { + return walk; + } + let mut pricing = ClaudeScanPricingResolver::default(); + let traversal_read_failures = + self.walk_claude_roots(&roots.paths, cutoff, cancel, &mut |path| { + let mut aggregation_failures = 0u32; + let mut file_result = scan_claude_file_with_pricing( + path, + cutoff, + &mut walk.completed_keys, + cancel, + &mut pricing, + &mut walk.incomplete, + |record| { + if !on_record(record) || record.timestamp.is_none() { + aggregation_failures = aggregation_failures.saturating_add(1); + } + }, + ); + file_result.aggregation_failures = file_result + .aggregation_failures + .saturating_add(aggregation_failures); + if file_result.counted > 0 { + walk.files_with_usage += 1; + } + walk.scan.absorb(file_result); + }); + walk.scan.read_failures = walk + .scan + .read_failures + .saturating_add(roots.read_failures) + .saturating_add(traversal_read_failures); + walk + } + /// Walk every root with one caller-owned `seen` set behind `on_file`, so a /// row copied between homes is counted once. Returns read failures. fn walk_claude_roots( @@ -1444,52 +1431,14 @@ pub fn get_daily_cost_and_incomplete_history(provider: &str, days: u32) -> Daily "claude" => { // Real per-day breakdown: walk the project logs once, // de-duplicating records across files. - let roots = scanner.claude_projects_roots(); - if !roots.paths.is_empty() { - let cutoff = Utc::now() - Duration::days(days as i64); - let mut seen = HashSet::new(); - let mut pricing = ClaudeScanPricingResolver::default(); - let mut claude_scan = ClaudeFileScanResult::default(); - let mut incomplete = ClaudeIncompleteTracker::default(); - let mut unknown_cost_dates = HashSet::new(); - let traversal_read_failures = { - let mut handle_file = |path: &Path| { - let mut aggregation_complete = true; - let mut file_result = scan_claude_file_with_pricing( - path, - &cutoff, - &mut seen, - None, - &mut pricing, - &mut incomplete, - |record| { - aggregation_complete &= record.timestamp.is_some(); - aggregation_complete &= add_claude_record_to_daily_costs( - &mut daily_costs, - &mut unknown_cost_dates, - record, - ); - }, - ); - if !aggregation_complete { - file_result.aggregation_failures = - file_result.aggregation_failures.saturating_add(1); - } - claude_scan.absorb(file_result); - }; - scanner.walk_claude_roots(&roots.paths, &cutoff, None, &mut handle_file) - }; - claude_scan.read_failures = claude_scan - .read_failures - .saturating_add(roots.read_failures) - .saturating_add(traversal_read_failures); - daily_incomplete = incomplete.resolve(&seen).daily_sorted(); - if claude_scan.is_complete() { - zero_fill_uninitialized_claude_daily_costs( - &mut daily_costs, - &unknown_cost_dates, - ); - } + let cutoff = Utc::now() - Duration::days(days as i64); + let mut unknown_cost_dates = HashSet::new(); + let walk = scanner.walk_claude_records(&cutoff, None, |record| { + add_claude_record_to_daily_costs(&mut daily_costs, &mut unknown_cost_dates, record) + }); + daily_incomplete = walk.incomplete.resolve(&walk.completed_keys).daily_sorted(); + if walk.roots_present && walk.scan.is_complete() { + zero_fill_uninitialized_claude_daily_costs(&mut daily_costs, &unknown_cost_dates); } } "opencodego" => { @@ -1571,30 +1520,12 @@ pub fn get_daily_token_history(provider: &str, days: u32) -> (Vec<(String, u64)> // Per-day token breakdown from the same de-duplicated record walk // as the cost chart. Only a complete valid walk establishes // authoritative coverage of the requested history window. - let roots = scanner.claude_projects_roots(); - if !roots.paths.is_empty() { - let cutoff = Utc::now() - Duration::days(days as i64); - let mut seen = HashSet::new(); - let mut pricing = ClaudeScanPricingResolver::default(); - let mut claude_scan = ClaudeFileScanResult::default(); - let traversal_read_failures = { - let mut handle_file = |path: &Path| { - let file_result = scan_claude_file_for_daily_tokens( - path, - &cutoff, - &mut seen, - &mut pricing, - &mut daily_tokens, - ); - claude_scan.absorb(file_result); - }; - scanner.walk_claude_roots(&roots.paths, &cutoff, None, &mut handle_file) - }; - claude_scan.read_failures = claude_scan - .read_failures - .saturating_add(roots.read_failures) - .saturating_add(traversal_read_failures); - mark_claude_daily_token_coverage(&mut covered_days, &daily_tokens, claude_scan); + let cutoff = Utc::now() - Duration::days(days as i64); + let walk = scanner.walk_claude_records(&cutoff, None, |record| { + add_claude_record_to_daily_tokens(&mut daily_tokens, record) + }); + if walk.roots_present { + mark_claude_daily_token_coverage(&mut covered_days, &daily_tokens, walk.scan); } } "pi" => { @@ -1656,37 +1587,6 @@ fn add_claude_record_to_daily_tokens( true } -fn scan_claude_file_for_daily_tokens( - path: &Path, - cutoff: &DateTime, - seen: &mut HashSet, - pricing: &mut ClaudeScanPricingResolver, - daily_tokens: &mut HashMap, -) -> ClaudeFileScanResult { - let mut aggregation_failures = 0u32; - // Token history ignores incomplete-request markers. - let mut incomplete = ClaudeIncompleteTracker::default(); - let mut result = scan_claude_file_with_pricing( - path, - cutoff, - seen, - None, - pricing, - &mut incomplete, - |record| { - if record.timestamp.is_none() - || !add_claude_record_to_daily_tokens(daily_tokens, record) - { - aggregation_failures = aggregation_failures.saturating_add(1); - } - }, - ); - result.aggregation_failures = result - .aggregation_failures - .saturating_add(aggregation_failures); - result -} - fn mark_claude_daily_token_coverage( covered_days: &mut HashSet, daily_tokens: &HashMap, diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 5e0eb3fe94..2a52d03b6f 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -560,6 +560,39 @@ fn malformed_claude_history_stays_unknown_while_valid_empty_history_is_known_zer assert!(!malformed_summary.known_zero); } +/// One file's daily token scan, counting each record that cannot be +/// aggregated as the token history path does. +fn scan_claude_file_for_daily_tokens( + path: &Path, + cutoff: &DateTime, + seen: &mut HashSet, + pricing: &mut ClaudeScanPricingResolver, + daily_tokens: &mut HashMap, +) -> ClaudeFileScanResult { + let mut aggregation_failures = 0u32; + // Token history ignores incomplete-request markers. + let mut incomplete = ClaudeIncompleteTracker::default(); + let mut result = scan_claude_file_with_pricing( + path, + cutoff, + seen, + None, + pricing, + &mut incomplete, + |record| { + if record.timestamp.is_none() + || !add_claude_record_to_daily_tokens(daily_tokens, record) + { + aggregation_failures = aggregation_failures.saturating_add(1); + } + }, + ); + result.aggregation_failures = result + .aggregation_failures + .saturating_add(aggregation_failures); + result +} + #[test] fn claude_daily_token_coverage_requires_a_complete_valid_scan() { let root = tempfile::tempdir().unwrap(); From c6f4c84b4165a5c812712d007041d1642cd61245 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:02:10 +0700 Subject: [PATCH 06/41] Delete unused Codex event structs from cost_scanner --- rust/src/cost_scanner.rs | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index 2d271e18c3..6990a011cb 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -220,31 +220,6 @@ fn system_time_to_unix_ms(modified: Option) -> i64 { millis } -/// JSONL event structures for Codex -#[allow( - dead_code, - reason = "JSONL event fields are deserialized for parsing but not all are read" -)] -#[derive(Debug, Deserialize)] -struct CodexEvent { - #[serde(rename = "type")] - event_type: Option, - event_msg: Option, -} - -#[allow( - dead_code, - reason = "event message fields are deserialized for parsing but not all are read" -)] -#[derive(Debug, Deserialize)] -struct CodexEventMsg { - #[serde(rename = "type")] - msg_type: Option, - input_tokens: Option, - cached_input_tokens: Option, - output_tokens: Option, -} - /// JSONL event structures for Claude transcripts. /// /// The flattened values retain otherwise-unknown metadata long enough to From 515a57db8b31da41112bcdfd0cd90bfee9100353 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:05:38 +0700 Subject: [PATCH 07/41] Delete unused Claude quota history scan wrappers --- rust/src/cost_scanner.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index 6990a011cb..8293f50435 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -777,25 +777,6 @@ impl CostScanner { } } - /// Scan Claude transcript rows for display-only quota-window history. - /// - /// The parser, preliminary-row filtering, provider filtering, and - /// request/session deduplication are shared with the ordinary Claude cost - /// summary path. Missing timestamps or incomplete file reads keep the - /// coverage flag false; callers must not present those rows as an exact - /// historical total. - pub fn scan_claude_quota_history(&self) -> ClaudeQuotaHistoryScan { - self.scan_claude_quota_history_with_cancel(None) - } - - /// Cancellable form of [`Self::scan_claude_quota_history`]. - pub fn scan_claude_quota_history_with_cancel( - &self, - cancel: Option<&AtomicBool>, - ) -> ClaudeQuotaHistoryScan { - self.scan_claude_chart_snapshot_with_cancel(cancel) - .quota_history - } /// Scan OpenCode Go local SQLite usage (upstream #2649 per-model cost breakdown). /// /// Reads the local `opencode.db` and maps rows onto the shared `CostSummary` From a4ddd2ff40b3990a8cb7542a58f9719ff44839d5 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:10:23 +0700 Subject: [PATCH 08/41] Share clamped file-length and epoch-millis helpers in cost_scanner --- rust/src/cost_scanner.rs | 29 +++++++------------ rust/src/cost_scanner/codex.rs | 6 +--- rust/src/cost_scanner/codex/logical_target.rs | 6 ++-- rust/src/cost_scanner/codex/reconciliation.rs | 3 +- rust/src/cost_scanner/codex/scan.rs | 9 +----- 5 files changed, 15 insertions(+), 38 deletions(-) diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index 8293f50435..879a8fff94 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -195,29 +195,20 @@ fn is_cancelled(cancel: Option<&AtomicBool>) -> bool { } fn unix_now_ms() -> i64 { - // Duration is clamped to i64::MAX before casting, so the value fits i64. - #[allow( - clippy::cast_possible_truncation, - reason = "clamped to i64::MAX before casting" - )] - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis().min(i64::MAX as u128) as i64) - .unwrap_or(0); - millis + system_time_to_unix_ms(Some(SystemTime::now())) } +/// Milliseconds since the Unix epoch, clamped to `i64::MAX`; 0 when unknown. fn system_time_to_unix_ms(modified: Option) -> i64 { - // Duration is clamped to i64::MAX before casting, so the value fits i64. - #[allow( - clippy::cast_possible_truncation, - reason = "clamped to i64::MAX before casting" - )] - let millis = modified + modified .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) - .map(|d| d.as_millis().min(i64::MAX as u128) as i64) - .unwrap_or(0); - millis + .map(|d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +/// File length as the cache stores it, clamped to `i64::MAX`. +fn file_len_i64(metadata: &fs::Metadata) -> i64 { + i64::try_from(metadata.len()).unwrap_or(i64::MAX) } /// JSONL event structures for Claude transcripts. diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index cef3621c56..44762d8871 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -437,11 +437,7 @@ impl CostScanner { Ok(metadata) => metadata, Err(_) => return CodexFileScanOutcome::default(), }; - #[allow( - clippy::cast_possible_wrap, - reason = "file sizes are clamped to i64::MAX" - )] - let size = metadata.len().min(i64::MAX as u64) as i64; + let size = file_len_i64(&metadata); let mtime_ms = system_time_to_unix_ms(metadata.modified().ok()); let path_key = path.to_string_lossy().to_string(); let file_identity = JsonlScanner::codex_file_identity(path, &metadata); diff --git a/rust/src/cost_scanner/codex/logical_target.rs b/rust/src/cost_scanner/codex/logical_target.rs index d9010b89c2..da166e11db 100644 --- a/rust/src/cost_scanner/codex/logical_target.rs +++ b/rust/src/cost_scanner/codex/logical_target.rs @@ -502,8 +502,7 @@ impl CodexLineagePlanner { if expected_identity != &actual_identity { return None; } - #[allow(clippy::cast_possible_wrap, reason = "session file sizes fit i64")] - let size = metadata.len().min(i64::MAX as u64) as i64; + let size = file_len_i64(&metadata); if usage.mtime_unix_ms != system_time_to_unix_ms(metadata.modified().ok()) || usage.size != size || usage.parsed_bytes.unwrap_or(0) < size @@ -617,8 +616,7 @@ pub(super) fn cached_codex_file_is_complete_for_range( usage.codex_file_identity.as_deref(), JsonlScanner::codex_file_identity(Path::new(path_key), &metadata).as_deref(), ); - #[allow(clippy::cast_possible_wrap, reason = "session file sizes fit i64")] - let size = metadata.len().min(i64::MAX as u64) as i64; + let size = file_len_i64(&metadata); identity_matches && usage.mtime_unix_ms == system_time_to_unix_ms(metadata.modified().ok()) && usage.size == size diff --git a/rust/src/cost_scanner/codex/reconciliation.rs b/rust/src/cost_scanner/codex/reconciliation.rs index 505f8509fd..e21fa14baa 100644 --- a/rust/src/cost_scanner/codex/reconciliation.rs +++ b/rust/src/cost_scanner/codex/reconciliation.rs @@ -105,8 +105,7 @@ fn codex_pending_path_affects_current_window( let Ok(metadata) = fs::metadata(path_key) else { return true; }; - #[allow(clippy::cast_possible_wrap, reason = "file sizes are clamped to i64")] - let observed_size = metadata.len().min(i64::MAX as u64) as i64; + let observed_size = file_len_i64(&metadata); if codex_logical_target_has_unconsumed_tail(observed_size, usage) { return true; } diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index ba9882a0ea..c93068252d 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -407,14 +407,7 @@ pub(super) fn scan_codex_detailed_with_cache( pending_next.retain(|pending| pending != &key); let observed_size = fs::metadata(&candidate.path) .ok() - .map(|metadata| { - #[allow( - clippy::cast_possible_wrap, - reason = "file sizes are clamped to i64::MAX" - )] - let size = metadata.len().min(i64::MAX as u64) as i64; - size - }) + .map(|metadata| file_len_i64(&metadata)) .unwrap_or(0); let has_unconsumed_tail = cache .files From d3fea1b8285bafbb2a838e3bba18eeb747f5435d Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:19:56 +0700 Subject: [PATCH 09/41] Mark unpriced models through one ModelPricingCompleteness method --- rust/src/codex_costs.rs | 26 ++------------------------ rust/src/cost_scanner.rs | 16 ++++++++++++++++ rust/src/pi_session_cost.rs | 17 ++++------------- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/rust/src/codex_costs.rs b/rust/src/codex_costs.rs index e71bbe074e..6a778330af 100644 --- a/rust/src/codex_costs.rs +++ b/rust/src/codex_costs.rs @@ -347,18 +347,7 @@ fn add_codex_tokens_to_summary( tokens, ); // Mark the breakdown as partial so the dashboard labels it (F18). - match &mut summary.model_pricing_completeness { - ModelPricingCompleteness::Complete => { - summary.model_pricing_completeness = ModelPricingCompleteness::Partial { - unpriced_models: vec![model_key.clone()], - }; - } - ModelPricingCompleteness::Partial { unpriced_models } => { - if !unpriced_models.contains(&model_key) { - unpriced_models.push(model_key.clone()); - } - } - } + summary.model_pricing_completeness.mark_unpriced(&model_key); return Some(0.0); } @@ -375,18 +364,7 @@ fn add_codex_tokens_to_summary( }); if uses_fallback_pricing { summary.unknown_models.insert(model_key.clone()); - match &mut summary.model_pricing_completeness { - ModelPricingCompleteness::Complete => { - summary.model_pricing_completeness = ModelPricingCompleteness::Partial { - unpriced_models: vec![model_key.clone()], - }; - } - ModelPricingCompleteness::Partial { unpriced_models } => { - if !unpriced_models.contains(&model_key) { - unpriced_models.push(model_key.clone()); - } - } - } + summary.model_pricing_completeness.mark_unpriced(&model_key); } add_summary_tokens(summary, tokens); diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index 879a8fff94..a3b2063a45 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -79,6 +79,22 @@ impl ModelPricingCompleteness { pub fn is_partial(&self) -> bool { matches!(self, Self::Partial { .. }) } + + /// Record `model` as unpriced, which makes the breakdown partial. + pub(crate) fn mark_unpriced(&mut self, model: &str) { + match self { + Self::Complete => { + *self = Self::Partial { + unpriced_models: vec![model.to_string()], + }; + } + Self::Partial { unpriced_models } => { + if !unpriced_models.iter().any(|known| known == model) { + unpriced_models.push(model.to_string()); + } + } + } + } } /// Cost summary from scanning local logs diff --git a/rust/src/pi_session_cost.rs b/rust/src/pi_session_cost.rs index 3df51b2e9c..c817661699 100644 --- a/rust/src/pi_session_cost.rs +++ b/rust/src/pi_session_cost.rs @@ -17,7 +17,7 @@ use crate::agent_sessions::pi_family::roots::{ omp_profile_selector, pi_settings_session_directory, }; use crate::core::CostUsagePricing; -use crate::cost_scanner::{CostSummary, ModelPricingCompleteness}; +use crate::cost_scanner::CostSummary; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PiMappedProvider { @@ -272,18 +272,9 @@ struct PiEntry { fn apply_entry(summary: &mut CostSummary, entry: &PiEntry) { if !entry.pricing_known { summary.unknown_models.insert(entry.model.clone()); - match &mut summary.model_pricing_completeness { - ModelPricingCompleteness::Complete => { - summary.model_pricing_completeness = ModelPricingCompleteness::Partial { - unpriced_models: vec![entry.model.clone()], - }; - } - ModelPricingCompleteness::Partial { unpriced_models } => { - if !unpriced_models.contains(&entry.model) { - unpriced_models.push(entry.model.clone()); - } - } - } + summary + .model_pricing_completeness + .mark_unpriced(&entry.model); } summary.input_tokens += entry.input; summary.output_tokens += entry.output; From 0bd79f98993c68c7ce7d2c8ba8d69430f99adc7a Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:26:24 +0700 Subject: [PATCH 10/41] Move test-only Codex cost helpers into codex_costs tests (RCO-19) --- rust/src/codex_costs.rs | 548 +--------------------------------- rust/src/codex_costs/tests.rs | 515 ++++++++++++++++++++++++++++++++ rust/src/cost_scanner.rs | 2 +- 3 files changed, 517 insertions(+), 548 deletions(-) create mode 100644 rust/src/codex_costs/tests.rs diff --git a/rust/src/codex_costs.rs b/rust/src/codex_costs.rs index 6a778330af..00ce44b702 100644 --- a/rust/src/codex_costs.rs +++ b/rust/src/codex_costs.rs @@ -17,7 +17,6 @@ pub(crate) use summary_contract::{ use chrono::{Duration, NaiveDate, Utc}; use std::collections::HashSet; -use std::path::Path; use crate::core::{ CodexUsageRecord, CostUsageCache, CostUsageDayRange, CostUsagePricing, JsonlScanner, @@ -204,36 +203,6 @@ pub(crate) fn add_codex_days_map_to_summary( (total_cost, has_tokens) } -#[cfg_attr( - not(test), - allow( - dead_code, - reason = "test-only helper path: only scan_codex_file_cost (test) calls this outside tests" - ) -)] -pub(crate) fn scan_codex_file_cost_for_range(path: &Path, range: &CostUsageDayRange) -> f64 { - let parse_result = match JsonlScanner::parse_codex_file(path, range, 0, None, None) { - Ok(result) => result, - Err(_) => return 0.0, - }; - - codex_records_cost( - &parse_result - .records - .iter() - .map(|(record, _)| record.clone()) - .collect::>(), - range, - ) -} - -#[cfg(test)] -pub(crate) fn scan_codex_file_cost(path: &Path) -> f64 { - let today = chrono::Local::now().date_naive(); - let range = CostUsageDayRange::new(codex_period_start(today, 30), today); - scan_codex_file_cost_for_range(path, &range) -} - #[derive(Clone, Copy)] struct CodexTokenCounts { input: u64, @@ -389,39 +358,6 @@ fn add_codex_tokens_to_summary( Some(cost) } -#[cfg_attr( - not(test), - allow( - dead_code, - reason = "only reached from scan_codex_file_cost_for_range, which is test-only in non-test builds" - ) -)] -fn codex_records_cost(records: &[CodexUsageRecord], range: &CostUsageDayRange) -> f64 { - let mut total_cost = 0.0; - - for record in records.iter().filter(|record| { - CostUsageDayRange::is_in_range(&record.day_key, &range.since_key, &range.until_key) - }) { - if CostUsagePricing::is_codex_unattributed_model(&record.model) - || !CostUsagePricing::counts_toward_codex_subscription(&record.model) - { - continue; - } - let tokens = CodexTokenCounts::from_values(record.input, record.cached, record.output); - if !tokens.is_empty() { - total_cost += codex_cost_usd_for_day( - &record.model, - tokens.input, - tokens.cached, - tokens.output, - CostUsageDayRange::parse_day_key(&record.day_key), - ); - } - } - - total_cost -} - fn codex_speed_bucket(model: &str) -> &'static str { let normalized = model.to_ascii_lowercase(); if normalized.contains("fast") @@ -435,28 +371,6 @@ fn codex_speed_bucket(model: &str) -> &'static str { } } -#[cfg_attr( - not(test), - allow(dead_code, reason = "only reached from test paths in non-test builds") -)] -fn codex_cost_usd(model: &str, input: u64, cached: u64, output: u64) -> f64 { - codex_cost_usd_for_day(model, input, cached, output, None) -} - -fn codex_cost_usd_for_day( - model: &str, - input: u64, - cached: u64, - output: u64, - pricing_day: Option, -) -> f64 { - if CostUsagePricing::is_codex_unattributed_model(model) { - return 0.0; - } - CostUsagePricing::codex_day_aggregate_cost_usd(model, input, cached, output, pricing_day) - .unwrap_or_else(|| codex_cost_usd_fallback(model, input, cached, output)) -} - fn codex_cost_usd_fallback(model: &str, input: u64, cached: u64, output: u64) -> f64 { let (input_price, cached_price, output_price) = match model.to_lowercase().as_str() { m if m.contains("gpt-4o-mini") => (0.15, 0.075, 0.60), @@ -478,464 +392,4 @@ fn codex_cost_usd_fallback(model: &str, input: u64, cached: u64, output: u64) -> } #[cfg(test)] -mod tests { - use super::*; - use crate::core::CodexUsageRecord; - use chrono::DateTime; - - #[test] - fn test_codex_pricing() { - // Test GPT-4o pricing: $2.50/1M input, $10/1M output - let cost = codex_cost_usd("gpt-4o", 1_000_000, 0, 1_000_000); - assert!((cost - 12.50).abs() < 0.01); - } - - #[test] - fn token_breakdown_addition_saturates_without_wrapping() { - let counts = ModelTokenCounts { - input_tokens: u64::MAX, - output_tokens: 1, - cached_tokens: u64::MAX, - reasoning_tokens: Some(7), - }; - assert_eq!(counts.total(), u64::MAX); - - let mut merged = ModelTokenCounts { - input_tokens: u64::MAX, - output_tokens: u64::MAX, - cached_tokens: u64::MAX, - reasoning_tokens: Some(7), - }; - add_tokens( - &mut merged, - CodexTokenCounts { - input: 1, - cached: 1, - output: 1, - reasoning: Some(1), - }, - ); - assert_eq!(merged.input_tokens, u64::MAX); - assert_eq!(merged.output_tokens, u64::MAX); - assert_eq!(merged.cached_tokens, u64::MAX); - } - - #[test] - fn known_reasoning_is_exposed_without_changing_cost() { - let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(target, target); - let make_record = |reasoning| CodexUsageRecord { - day_key: "2026-05-31".to_string(), - timestamp: None, - model: "gpt-5.6-sol".to_string(), - input: 100, - cached: 0, - output: 20, - reasoning, - turn_id: None, - }; - - let mut known_summary = CostSummary::default(); - let (known_cost, known_has_tokens) = - add_codex_records_to_summary(&mut known_summary, &[(make_record(Some(7)), 0)], &range); - let mut unknown_summary = CostSummary::default(); - let (unknown_cost, unknown_has_tokens) = - add_codex_records_to_summary(&mut unknown_summary, &[(make_record(None), 0)], &range); - - assert!(known_has_tokens && unknown_has_tokens); - assert_eq!(known_summary.output_tokens, 20); - assert_eq!(known_summary.reasoning_tokens, Some(7)); - assert_eq!( - known_summary.by_model_tokens["gpt-5.6-sol"].reasoning_tokens, - Some(7) - ); - assert_eq!(known_cost, unknown_cost); - } - - #[test] - fn reasoning_unknown_is_sticky_for_summary_and_model() { - let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(target, target); - let make_record = |reasoning| CodexUsageRecord { - day_key: "2026-05-31".to_string(), - timestamp: None, - model: "gpt-5.6-sol".to_string(), - input: 1, - cached: 0, - output: 20, - reasoning, - turn_id: None, - }; - let records = vec![ - (make_record(Some(7)), 0), - (make_record(None), 0), - (make_record(Some(3)), 0), - ]; - let mut summary = CostSummary::default(); - - add_codex_records_to_summary(&mut summary, &records, &range); - - assert_eq!(summary.reasoning_tokens, None); - assert_eq!( - summary.by_model_tokens["gpt-5.6-sol"].reasoning_tokens, - None - ); - } - - #[test] - fn packed_reasoning_slot_distinguishes_known_from_unknown() { - let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let mut known = CostSummary::default(); - add_codex_packed_tokens_to_summary( - &mut known, - "gpt-5.6-sol", - &[100, 0, 20, 7], - Some(target), - ); - assert_eq!(known.reasoning_tokens, Some(7)); - - let mut unknown = CostSummary::default(); - add_codex_packed_tokens_to_summary( - &mut unknown, - "gpt-5.6-sol", - &[100, 0, 20], - Some(target), - ); - assert_eq!(unknown.reasoning_tokens, None); - } - - #[test] - fn test_codex_pricing_uses_gpt55_standard_short_context_rates() { - let cost = codex_cost_usd("gpt-5.5", 200_000, 80_000, 100_000); - - // GPT-5.5 standard short-context pricing: - // 120k non-cached input at $5/M, 80k cached input at $0.50/M, - // and 100k output at $30/M. - assert!((cost - 3.64).abs() < 1e-9); - } - - #[test] - fn test_codex_pricing_bills_whole_gpt55_request_at_long_context_rates() { - let cost = codex_cost_usd("gpt-5.5", 1_000_000, 400_000, 1_000_000); - - // Above 272K input the whole request bills at $10/M input, - // $1/M cached input and $45/M output. - assert!((cost - 51.40).abs() < 1e-9); - } - - #[test] - fn codex_summary_prices_gpt56_usage_records_individually() { - let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(target, target); - let records = vec![ - ( - CodexUsageRecord { - day_key: "2026-05-31".to_string(), - timestamp: None, - model: "gpt-5.6-sol".to_string(), - input: 200_000, - cached: 0, - output: 0, - reasoning: None, - turn_id: None, - }, - 0, - ), - ( - CodexUsageRecord { - day_key: "2026-05-31".to_string(), - timestamp: None, - model: "gpt-5.6-sol".to_string(), - input: 200_000, - cached: 0, - output: 0, - reasoning: None, - turn_id: None, - }, - 0, - ), - ( - CodexUsageRecord { - day_key: "2026-05-30".to_string(), - timestamp: None, - model: "gpt-5.6-sol".to_string(), - input: 200_000, - cached: 0, - output: 0, - reasoning: None, - turn_id: None, - }, - 0, - ), - ]; - let mut summary = CostSummary::default(); - - let (cost, has_tokens) = add_codex_records_to_summary(&mut summary, &records, &range); - - assert!(has_tokens); - assert_eq!(summary.input_tokens, 400_000); - assert!((cost - 2.0).abs() < f64::EPSILON); - } - - #[test] - fn cached_day_fold_preserves_gpt56_historical_pricing() { - let before = NaiveDate::from_ymd_opt(2026, 7, 29).unwrap(); - let after = NaiveDate::from_ymd_opt(2026, 7, 30).unwrap(); - let range = CostUsageDayRange::new(before, after); - let days = std::collections::HashMap::from([ - ( - "2026-07-29".to_string(), - std::collections::HashMap::from([("gpt-5.6-terra".to_string(), vec![100, 10, 5])]), - ), - ( - "2026-07-30".to_string(), - std::collections::HashMap::from([("gpt-5.6-terra".to_string(), vec![100, 10, 5])]), - ), - ]); - let mut summary = CostSummary::default(); - let (cost, has_tokens) = add_codex_days_map_to_summary(&mut summary, &days, &range); - - let before_expected = 90.0 * 2.5e-6 + 10.0 * 2.5e-7 + 5.0 * 1.5e-5; - let after_expected = 90.0 * 2e-6 + 10.0 * 2e-7 + 5.0 * 1.2e-5; - assert!(has_tokens); - assert!((cost - (before_expected + after_expected)).abs() < 1e-12); - } - - #[test] - fn routed_models_do_not_count_toward_native_codex_summary() { - let target = NaiveDate::from_ymd_opt(2026, 8, 19).unwrap(); - let range = CostUsageDayRange::new(target, target); - let records = vec![ - ( - CodexUsageRecord { - day_key: "2026-08-19".to_string(), - timestamp: None, - model: "gpt-5.6-sol".to_string(), - input: 100, - cached: 0, - output: 5, - reasoning: None, - turn_id: None, - }, - 0, - ), - ( - CodexUsageRecord { - day_key: "2026-08-19".to_string(), - timestamp: None, - model: "deepseek/deepseek-chat".to_string(), - input: 1_000_000, - cached: 0, - output: 1_000_000, - reasoning: None, - turn_id: None, - }, - 0, - ), - ]; - let mut summary = CostSummary::default(); - let (cost, has_tokens) = add_codex_records_to_summary(&mut summary, &records, &range); - assert!(has_tokens); - assert_eq!(summary.input_tokens, 100); - assert_eq!(summary.output_tokens, 5); - assert!(!summary.by_model.contains_key("deepseek/deepseek-chat")); - assert!( - cost < 0.01, - "routed DeepSeek cost leaked into native Codex: {cost}" - ); - } - - #[test] - fn routed_models_are_not_persisted_in_codex_day_token_cache() { - let records = vec![( - CodexUsageRecord { - day_key: "2026-08-19".to_string(), - timestamp: None, - model: "opencode/gpt-5".to_string(), - input: 10, - cached: 0, - output: 1, - reasoning: None, - turn_id: None, - }, - 0, - )]; - let mut days = std::collections::HashMap::new(); - merge_codex_records_into_days(&mut days, &records); - assert!(days.is_empty()); - } - - #[test] - fn model_less_codex_usage_is_visible_but_unpriced() { - let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(target, target); - let records = vec![( - CodexUsageRecord { - day_key: "2026-05-31".to_string(), - timestamp: None, - model: CostUsagePricing::CODEX_UNATTRIBUTED_MODEL.to_string(), - input: 55_000_000, - cached: 0, - output: 0, - reasoning: None, - turn_id: None, - }, - 0, - )]; - let mut summary = CostSummary::default(); - - let (cost, has_tokens) = add_codex_records_to_summary(&mut summary, &records, &range); - - assert!(has_tokens); - assert_eq!(cost, 0.0); - assert_eq!(summary.input_tokens, 55_000_000); - assert_eq!( - summary - .by_model - .get(CostUsagePricing::CODEX_UNATTRIBUTED_MODEL) - .copied(), - Some(0.0) - ); - assert!(summary.unknown_models.is_empty()); - } - - #[test] - fn records_unknown_codex_model_while_using_fallback_cost() { - let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(target, target); - let records = vec![( - CodexUsageRecord { - day_key: "2026-05-31".to_string(), - timestamp: None, - model: "gpt-mystery".to_string(), - input: 1_000_000, - cached: 0, - output: 1_000_000, - reasoning: None, - turn_id: None, - }, - 0, - )]; - let mut summary = CostSummary::default(); - - let (cost, has_tokens) = add_codex_records_to_summary(&mut summary, &records, &range); - - assert!(has_tokens); - assert!(cost > 0.0); - assert!(summary.unknown_models.contains("gpt-mystery")); - } - - #[test] - fn test_codex_speed_bucket() { - assert_eq!(codex_speed_bucket("gpt-5.5-fast"), "fast"); - assert_eq!(codex_speed_bucket("gpt-5.3-codex-spark"), "fast"); - assert_eq!(codex_speed_bucket("gpt-5-codex"), "standard"); - } - - #[test] - fn summary_wire_is_versioned_and_path_free() { - let complete = CostSummary { - total_cost_usd: 0.25, - input_tokens: 100, - output_tokens: 50, - cached_tokens: 20, - sessions_count: 1, - by_model: std::collections::HashMap::from([("fixture-model".to_string(), 0.25)]), - by_model_tokens: std::collections::HashMap::from([( - "fixture-model".to_string(), - ModelTokenCounts { - input_tokens: 100, - output_tokens: 50, - cached_tokens: 20, - reasoning_tokens: None, - }, - )]), - history_coverage_established: true, - ..CostSummary::default() - }; - let summary = CodexCostSummary::from_summaries_at( - &complete, - &complete, - 30, - DateTime::from_timestamp(1_700_000_000, 0).unwrap(), - "UTC", - ); - - summary.validate(30).unwrap(); - let wire = serde_json::to_string(&[summary]).unwrap(); - assert!(wire.contains("schemaVersion")); - assert!(wire.contains("costUSD")); - assert!(wire.contains("bucketTimeZone")); - assert!(!wire.contains("fixture-model")); - assert!(!wire.contains("sessions")); - assert!(!wire.contains("project")); - } - - #[test] - fn incomplete_scan_keeps_remote_totals_unknown() { - let partial = CostSummary { - total_cost_usd: 9.0, - input_tokens: 1_000, - output_tokens: 200, - history_coverage_established: false, - ..CostSummary::default() - }; - let summary = CodexCostSummary::from_summaries_at( - &partial, - &partial, - 30, - DateTime::from_timestamp(1_700_000_000, 0).unwrap(), - "UTC", - ); - - assert_eq!(summary.history.total_tokens, None); - assert_eq!(summary.history.cost_usd, None); - assert_eq!(summary.today.total_tokens, None); - assert_eq!(summary.today.cost_usd, None); - summary.validate(30).unwrap(); - } - - #[test] - fn remote_summary_decoder_rejects_wrong_version_and_oversized_output() { - let source = CostSummary { - history_coverage_established: true, - known_zero: true, - ..CostSummary::default() - }; - let mut summary = CodexCostSummary::from_summaries_at( - &source, - &source, - 30, - DateTime::from_timestamp(1_700_000_000, 0).unwrap(), - "UTC", - ); - summary.schema_version = 2; - let wire = serde_json::to_string(&[summary]).unwrap(); - assert!(decode_remote_codex_summary(&wire, 30).is_err()); - assert!( - decode_remote_codex_summary(&"x".repeat(MAX_REMOTE_CODEX_COST_BYTES + 1), 30).is_err() - ); - } - - #[test] - fn remote_summary_decoder_rejects_numeric_totals_with_incomplete_coverage() { - let source = CostSummary { - history_coverage_established: true, - known_zero: true, - ..Default::default() - }; - let mut summary = CodexCostSummary::from_summaries_at( - &source, - &source, - 30, - DateTime::from_timestamp(1_700_000_000, 0).unwrap(), - "UTC", - ); - summary.history_coverage_is_established = false; - summary.history.total_tokens = Some(42); - summary.history.cost_usd = Some(1.25); - - let wire = serde_json::to_string(&[summary]).unwrap(); - assert!(decode_remote_codex_summary(&wire, 30).is_err()); - } -} +pub(crate) mod tests; diff --git a/rust/src/codex_costs/tests.rs b/rust/src/codex_costs/tests.rs new file mode 100644 index 0000000000..385fa2ebca --- /dev/null +++ b/rust/src/codex_costs/tests.rs @@ -0,0 +1,515 @@ +use super::*; +use crate::core::CodexUsageRecord; +use chrono::DateTime; +use std::path::Path; + +/// Cost of one Codex file over the last 30 days, through the record path. +pub(crate) fn scan_codex_file_cost(path: &Path) -> f64 { + let today = chrono::Local::now().date_naive(); + let range = CostUsageDayRange::new(codex_period_start(today, 30), today); + scan_codex_file_cost_for_range(path, &range) +} + +fn scan_codex_file_cost_for_range(path: &Path, range: &CostUsageDayRange) -> f64 { + let parse_result = match JsonlScanner::parse_codex_file(path, range, 0, None, None) { + Ok(result) => result, + Err(_) => return 0.0, + }; + + codex_records_cost( + &parse_result + .records + .iter() + .map(|(record, _)| record.clone()) + .collect::>(), + range, + ) +} + +fn codex_records_cost(records: &[CodexUsageRecord], range: &CostUsageDayRange) -> f64 { + let mut total_cost = 0.0; + + for record in records.iter().filter(|record| { + CostUsageDayRange::is_in_range(&record.day_key, &range.since_key, &range.until_key) + }) { + if CostUsagePricing::is_codex_unattributed_model(&record.model) + || !CostUsagePricing::counts_toward_codex_subscription(&record.model) + { + continue; + } + let tokens = CodexTokenCounts::from_values(record.input, record.cached, record.output); + if !tokens.is_empty() { + total_cost += codex_cost_usd_for_day( + &record.model, + tokens.input, + tokens.cached, + tokens.output, + CostUsageDayRange::parse_day_key(&record.day_key), + ); + } + } + + total_cost +} + +fn codex_cost_usd(model: &str, input: u64, cached: u64, output: u64) -> f64 { + codex_cost_usd_for_day(model, input, cached, output, None) +} + +fn codex_cost_usd_for_day( + model: &str, + input: u64, + cached: u64, + output: u64, + pricing_day: Option, +) -> f64 { + if CostUsagePricing::is_codex_unattributed_model(model) { + return 0.0; + } + CostUsagePricing::codex_day_aggregate_cost_usd(model, input, cached, output, pricing_day) + .unwrap_or_else(|| codex_cost_usd_fallback(model, input, cached, output)) +} + +#[test] +fn test_codex_pricing() { + // Test GPT-4o pricing: $2.50/1M input, $10/1M output + let cost = codex_cost_usd("gpt-4o", 1_000_000, 0, 1_000_000); + assert!((cost - 12.50).abs() < 0.01); +} + +#[test] +fn token_breakdown_addition_saturates_without_wrapping() { + let counts = ModelTokenCounts { + input_tokens: u64::MAX, + output_tokens: 1, + cached_tokens: u64::MAX, + reasoning_tokens: Some(7), + }; + assert_eq!(counts.total(), u64::MAX); + + let mut merged = ModelTokenCounts { + input_tokens: u64::MAX, + output_tokens: u64::MAX, + cached_tokens: u64::MAX, + reasoning_tokens: Some(7), + }; + add_tokens( + &mut merged, + CodexTokenCounts { + input: 1, + cached: 1, + output: 1, + reasoning: Some(1), + }, + ); + assert_eq!(merged.input_tokens, u64::MAX); + assert_eq!(merged.output_tokens, u64::MAX); + assert_eq!(merged.cached_tokens, u64::MAX); +} + +#[test] +fn known_reasoning_is_exposed_without_changing_cost() { + let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(target, target); + let make_record = |reasoning| CodexUsageRecord { + day_key: "2026-05-31".to_string(), + timestamp: None, + model: "gpt-5.6-sol".to_string(), + input: 100, + cached: 0, + output: 20, + reasoning, + turn_id: None, + }; + + let mut known_summary = CostSummary::default(); + let (known_cost, known_has_tokens) = + add_codex_records_to_summary(&mut known_summary, &[(make_record(Some(7)), 0)], &range); + let mut unknown_summary = CostSummary::default(); + let (unknown_cost, unknown_has_tokens) = + add_codex_records_to_summary(&mut unknown_summary, &[(make_record(None), 0)], &range); + + assert!(known_has_tokens && unknown_has_tokens); + assert_eq!(known_summary.output_tokens, 20); + assert_eq!(known_summary.reasoning_tokens, Some(7)); + assert_eq!( + known_summary.by_model_tokens["gpt-5.6-sol"].reasoning_tokens, + Some(7) + ); + assert_eq!(known_cost, unknown_cost); +} + +#[test] +fn reasoning_unknown_is_sticky_for_summary_and_model() { + let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(target, target); + let make_record = |reasoning| CodexUsageRecord { + day_key: "2026-05-31".to_string(), + timestamp: None, + model: "gpt-5.6-sol".to_string(), + input: 1, + cached: 0, + output: 20, + reasoning, + turn_id: None, + }; + let records = vec![ + (make_record(Some(7)), 0), + (make_record(None), 0), + (make_record(Some(3)), 0), + ]; + let mut summary = CostSummary::default(); + + add_codex_records_to_summary(&mut summary, &records, &range); + + assert_eq!(summary.reasoning_tokens, None); + assert_eq!( + summary.by_model_tokens["gpt-5.6-sol"].reasoning_tokens, + None + ); +} + +#[test] +fn packed_reasoning_slot_distinguishes_known_from_unknown() { + let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let mut known = CostSummary::default(); + add_codex_packed_tokens_to_summary(&mut known, "gpt-5.6-sol", &[100, 0, 20, 7], Some(target)); + assert_eq!(known.reasoning_tokens, Some(7)); + + let mut unknown = CostSummary::default(); + add_codex_packed_tokens_to_summary(&mut unknown, "gpt-5.6-sol", &[100, 0, 20], Some(target)); + assert_eq!(unknown.reasoning_tokens, None); +} + +#[test] +fn test_codex_pricing_uses_gpt55_standard_short_context_rates() { + let cost = codex_cost_usd("gpt-5.5", 200_000, 80_000, 100_000); + + // GPT-5.5 standard short-context pricing: + // 120k non-cached input at $5/M, 80k cached input at $0.50/M, + // and 100k output at $30/M. + assert!((cost - 3.64).abs() < 1e-9); +} + +#[test] +fn test_codex_pricing_bills_whole_gpt55_request_at_long_context_rates() { + let cost = codex_cost_usd("gpt-5.5", 1_000_000, 400_000, 1_000_000); + + // Above 272K input the whole request bills at $10/M input, + // $1/M cached input and $45/M output. + assert!((cost - 51.40).abs() < 1e-9); +} + +#[test] +fn codex_summary_prices_gpt56_usage_records_individually() { + let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(target, target); + let records = vec![ + ( + CodexUsageRecord { + day_key: "2026-05-31".to_string(), + timestamp: None, + model: "gpt-5.6-sol".to_string(), + input: 200_000, + cached: 0, + output: 0, + reasoning: None, + turn_id: None, + }, + 0, + ), + ( + CodexUsageRecord { + day_key: "2026-05-31".to_string(), + timestamp: None, + model: "gpt-5.6-sol".to_string(), + input: 200_000, + cached: 0, + output: 0, + reasoning: None, + turn_id: None, + }, + 0, + ), + ( + CodexUsageRecord { + day_key: "2026-05-30".to_string(), + timestamp: None, + model: "gpt-5.6-sol".to_string(), + input: 200_000, + cached: 0, + output: 0, + reasoning: None, + turn_id: None, + }, + 0, + ), + ]; + let mut summary = CostSummary::default(); + + let (cost, has_tokens) = add_codex_records_to_summary(&mut summary, &records, &range); + + assert!(has_tokens); + assert_eq!(summary.input_tokens, 400_000); + assert!((cost - 2.0).abs() < f64::EPSILON); +} + +#[test] +fn cached_day_fold_preserves_gpt56_historical_pricing() { + let before = NaiveDate::from_ymd_opt(2026, 7, 29).unwrap(); + let after = NaiveDate::from_ymd_opt(2026, 7, 30).unwrap(); + let range = CostUsageDayRange::new(before, after); + let days = std::collections::HashMap::from([ + ( + "2026-07-29".to_string(), + std::collections::HashMap::from([("gpt-5.6-terra".to_string(), vec![100, 10, 5])]), + ), + ( + "2026-07-30".to_string(), + std::collections::HashMap::from([("gpt-5.6-terra".to_string(), vec![100, 10, 5])]), + ), + ]); + let mut summary = CostSummary::default(); + let (cost, has_tokens) = add_codex_days_map_to_summary(&mut summary, &days, &range); + + let before_expected = 90.0 * 2.5e-6 + 10.0 * 2.5e-7 + 5.0 * 1.5e-5; + let after_expected = 90.0 * 2e-6 + 10.0 * 2e-7 + 5.0 * 1.2e-5; + assert!(has_tokens); + assert!((cost - (before_expected + after_expected)).abs() < 1e-12); +} + +#[test] +fn routed_models_do_not_count_toward_native_codex_summary() { + let target = NaiveDate::from_ymd_opt(2026, 8, 19).unwrap(); + let range = CostUsageDayRange::new(target, target); + let records = vec![ + ( + CodexUsageRecord { + day_key: "2026-08-19".to_string(), + timestamp: None, + model: "gpt-5.6-sol".to_string(), + input: 100, + cached: 0, + output: 5, + reasoning: None, + turn_id: None, + }, + 0, + ), + ( + CodexUsageRecord { + day_key: "2026-08-19".to_string(), + timestamp: None, + model: "deepseek/deepseek-chat".to_string(), + input: 1_000_000, + cached: 0, + output: 1_000_000, + reasoning: None, + turn_id: None, + }, + 0, + ), + ]; + let mut summary = CostSummary::default(); + let (cost, has_tokens) = add_codex_records_to_summary(&mut summary, &records, &range); + assert!(has_tokens); + assert_eq!(summary.input_tokens, 100); + assert_eq!(summary.output_tokens, 5); + assert!(!summary.by_model.contains_key("deepseek/deepseek-chat")); + assert!( + cost < 0.01, + "routed DeepSeek cost leaked into native Codex: {cost}" + ); +} + +#[test] +fn routed_models_are_not_persisted_in_codex_day_token_cache() { + let records = vec![( + CodexUsageRecord { + day_key: "2026-08-19".to_string(), + timestamp: None, + model: "opencode/gpt-5".to_string(), + input: 10, + cached: 0, + output: 1, + reasoning: None, + turn_id: None, + }, + 0, + )]; + let mut days = std::collections::HashMap::new(); + merge_codex_records_into_days(&mut days, &records); + assert!(days.is_empty()); +} + +#[test] +fn model_less_codex_usage_is_visible_but_unpriced() { + let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(target, target); + let records = vec![( + CodexUsageRecord { + day_key: "2026-05-31".to_string(), + timestamp: None, + model: CostUsagePricing::CODEX_UNATTRIBUTED_MODEL.to_string(), + input: 55_000_000, + cached: 0, + output: 0, + reasoning: None, + turn_id: None, + }, + 0, + )]; + let mut summary = CostSummary::default(); + + let (cost, has_tokens) = add_codex_records_to_summary(&mut summary, &records, &range); + + assert!(has_tokens); + assert_eq!(cost, 0.0); + assert_eq!(summary.input_tokens, 55_000_000); + assert_eq!( + summary + .by_model + .get(CostUsagePricing::CODEX_UNATTRIBUTED_MODEL) + .copied(), + Some(0.0) + ); + assert!(summary.unknown_models.is_empty()); +} + +#[test] +fn records_unknown_codex_model_while_using_fallback_cost() { + let target = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(target, target); + let records = vec![( + CodexUsageRecord { + day_key: "2026-05-31".to_string(), + timestamp: None, + model: "gpt-mystery".to_string(), + input: 1_000_000, + cached: 0, + output: 1_000_000, + reasoning: None, + turn_id: None, + }, + 0, + )]; + let mut summary = CostSummary::default(); + + let (cost, has_tokens) = add_codex_records_to_summary(&mut summary, &records, &range); + + assert!(has_tokens); + assert!(cost > 0.0); + assert!(summary.unknown_models.contains("gpt-mystery")); +} + +#[test] +fn test_codex_speed_bucket() { + assert_eq!(codex_speed_bucket("gpt-5.5-fast"), "fast"); + assert_eq!(codex_speed_bucket("gpt-5.3-codex-spark"), "fast"); + assert_eq!(codex_speed_bucket("gpt-5-codex"), "standard"); +} + +#[test] +fn summary_wire_is_versioned_and_path_free() { + let complete = CostSummary { + total_cost_usd: 0.25, + input_tokens: 100, + output_tokens: 50, + cached_tokens: 20, + sessions_count: 1, + by_model: std::collections::HashMap::from([("fixture-model".to_string(), 0.25)]), + by_model_tokens: std::collections::HashMap::from([( + "fixture-model".to_string(), + ModelTokenCounts { + input_tokens: 100, + output_tokens: 50, + cached_tokens: 20, + reasoning_tokens: None, + }, + )]), + history_coverage_established: true, + ..CostSummary::default() + }; + let summary = CodexCostSummary::from_summaries_at( + &complete, + &complete, + 30, + DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + "UTC", + ); + + summary.validate(30).unwrap(); + let wire = serde_json::to_string(&[summary]).unwrap(); + assert!(wire.contains("schemaVersion")); + assert!(wire.contains("costUSD")); + assert!(wire.contains("bucketTimeZone")); + assert!(!wire.contains("fixture-model")); + assert!(!wire.contains("sessions")); + assert!(!wire.contains("project")); +} + +#[test] +fn incomplete_scan_keeps_remote_totals_unknown() { + let partial = CostSummary { + total_cost_usd: 9.0, + input_tokens: 1_000, + output_tokens: 200, + history_coverage_established: false, + ..CostSummary::default() + }; + let summary = CodexCostSummary::from_summaries_at( + &partial, + &partial, + 30, + DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + "UTC", + ); + + assert_eq!(summary.history.total_tokens, None); + assert_eq!(summary.history.cost_usd, None); + assert_eq!(summary.today.total_tokens, None); + assert_eq!(summary.today.cost_usd, None); + summary.validate(30).unwrap(); +} + +#[test] +fn remote_summary_decoder_rejects_wrong_version_and_oversized_output() { + let source = CostSummary { + history_coverage_established: true, + known_zero: true, + ..CostSummary::default() + }; + let mut summary = CodexCostSummary::from_summaries_at( + &source, + &source, + 30, + DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + "UTC", + ); + summary.schema_version = 2; + let wire = serde_json::to_string(&[summary]).unwrap(); + assert!(decode_remote_codex_summary(&wire, 30).is_err()); + assert!(decode_remote_codex_summary(&"x".repeat(MAX_REMOTE_CODEX_COST_BYTES + 1), 30).is_err()); +} + +#[test] +fn remote_summary_decoder_rejects_numeric_totals_with_incomplete_coverage() { + let source = CostSummary { + history_coverage_established: true, + known_zero: true, + ..Default::default() + }; + let mut summary = CodexCostSummary::from_summaries_at( + &source, + &source, + 30, + DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + "UTC", + ); + summary.history_coverage_is_established = false; + summary.history.total_tokens = Some(42); + summary.history.cost_usd = Some(1.25); + + let wire = serde_json::to_string(&[summary]).unwrap(); + assert!(decode_remote_codex_summary(&wire, 30).is_err()); +} diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index a3b2063a45..72299b655c 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -19,7 +19,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; #[cfg(test)] -use crate::codex_costs::scan_codex_file_cost; +use crate::codex_costs::tests::scan_codex_file_cost; use crate::codex_costs::{ add_codex_days_map_to_summary, add_codex_records_to_summary, codex_scan_dates, merge_codex_records_into_days, From 7b43bfbee856417ec07e58e6a39c6b730a6023a4 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:27:48 +0700 Subject: [PATCH 11/41] Fold identical Codex model-key branches (RCO-19) --- rust/src/codex_costs.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rust/src/codex_costs.rs b/rust/src/codex_costs.rs index 00ce44b702..1992d32a8c 100644 --- a/rust/src/codex_costs.rs +++ b/rust/src/codex_costs.rs @@ -290,13 +290,11 @@ fn add_codex_tokens_to_summary( } let is_routing_unpriced = is_unpriced_codex_routing_model(model); + // Routing models (e.g. "codex-auto-review") keep their own name, so the + // breakdown shows a deliberately-unpriced row distinct from the model-less + // "unknown" sentinel. let model_key = if CostUsagePricing::is_codex_unattributed_model(model) { CostUsagePricing::CODEX_UNATTRIBUTED_MODEL.to_string() - } else if is_routing_unpriced { - // Preserve the original routing model name (e.g. "codex-auto-review") so - // the breakdown shows it as a deliberately-unpriced row, distinct from the - // model-less "unknown" sentinel. - model.to_string() } else { model.to_string() }; From a5c040efd1b79cc5a9b8f158812eebfd9a9935fc Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:32:09 +0700 Subject: [PATCH 12/41] Pin spend activity histogram cells --- rust/src/spend_contract/opencodex/tests.rs | 37 +++++++++ rust/src/spend_contract/tests.rs | 2 + rust/src/spend_contract/tests/activity.rs | 91 ++++++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 rust/src/spend_contract/tests/activity.rs diff --git a/rust/src/spend_contract/opencodex/tests.rs b/rust/src/spend_contract/opencodex/tests.rs index d35bafddeb..e5f07784f0 100644 --- a/rust/src/spend_contract/opencodex/tests.rs +++ b/rust/src/spend_contract/opencodex/tests.rs @@ -513,3 +513,40 @@ fn nonnegative_u64_accepts_json_numbers_and_bounded_floats() { ); assert_eq!(nonnegative_u64(None), None); } + +#[test] +fn aggregate_counts_one_activity_conversation_per_kept_row() { + use super::super::tests::activity::{cells, counted}; + let now = DateTime::parse_from_rfc3339("2026-08-19T12:00:00Z") + .unwrap() + .with_timezone(&Utc); + let at = |timestamp: &str| { + DateTime::parse_from_rfc3339(timestamp) + .unwrap() + .with_timezone(&Utc) + }; + let row = |request_id: &str, timestamp: DateTime| OpenCodexEntry { + request_id: request_id.into(), + timestamp, + ..entry("openai", "gpt-5") + }; + let first = at("2026-08-18T10:00:00Z"); + let second = at("2026-08-18T10:10:00Z"); + let other_day = at("2026-08-17T03:00:00Z"); + let source = aggregate( + vec![ + row("a", first), + row("b", second), + row("c", other_day), + row("old", at("2026-08-01T10:00:00Z")), + ], + now, + 7, + &CustomPricing::default(), + ) + .expect("source"); + assert_eq!( + cells(&source.hourly_activity), + counted(&[first, second, other_day]) + ); +} diff --git a/rust/src/spend_contract/tests.rs b/rust/src/spend_contract/tests.rs index 2c3be37665..29c348fe75 100644 --- a/rust/src/spend_contract/tests.rs +++ b/rust/src/spend_contract/tests.rs @@ -1,5 +1,7 @@ use super::*; +pub(super) mod activity; + #[test] fn local_history_total_requires_complete_scan_and_pricing() { let priced = LocalCostEstimate { diff --git a/rust/src/spend_contract/tests/activity.rs b/rust/src/spend_contract/tests/activity.rs new file mode 100644 index 0000000000..158b68dd53 --- /dev/null +++ b/rust/src/spend_contract/tests/activity.rs @@ -0,0 +1,91 @@ +//! Weekday/hour activity cells: built from sessions in the local zone, and +//! merged across sources. + +use super::*; +use crate::codex_workspaces::{CostEstimate, UsageTotals}; +use chrono::DateTime; + +fn at(timestamp: &str) -> DateTime { + DateTime::parse_from_rfc3339(timestamp) + .unwrap() + .with_timezone(&Utc) +} + +fn local_cell(timestamp: DateTime) -> (u8, u8) { + let local = timestamp.with_timezone(&Local); + ( + u8::try_from(local.weekday().num_days_from_monday()).unwrap(), + u8::try_from(local.hour()).unwrap(), + ) +} + +pub(in crate::spend_contract) fn cells(activity: &[SpendActivityCell]) -> Vec<(u8, u8, u32)> { + activity + .iter() + .map(|cell| (cell.weekday, cell.hour, cell.conversations)) + .collect() +} + +/// Expected cells for one conversation per timestamp, in key order. +pub(in crate::spend_contract) fn counted(timestamps: &[DateTime]) -> Vec<(u8, u8, u32)> { + let mut expected: BTreeMap<(u8, u8), u32> = BTreeMap::new(); + for timestamp in timestamps { + *expected.entry(local_cell(*timestamp)).or_default() += 1; + } + expected + .into_iter() + .map(|((weekday, hour), conversations)| (weekday, hour, conversations)) + .collect() +} + +fn session( + started_at: Option>, + latest_activity: Option>, +) -> SessionUsage { + SessionUsage { + id: "s".into(), + project_id: "p".into(), + display_title: "s".into(), + cwd: None, + started_at, + latest_activity, + totals: UsageTotals::from_parts(1, 0, 0), + cost_estimate: CostEstimate::default(), + top_model: None, + } +} + +#[test] +fn session_activity_counts_latest_activity_or_start_per_local_cell() { + let monday = at("2026-08-17T10:15:00Z"); + let tuesday = at("2026-08-18T23:45:00Z"); + let activity = activity_from_sessions(&[ + session(Some(tuesday), Some(monday)), + session(Some(tuesday), None), + session(None, None), + session(None, Some(monday)), + ]); + assert_eq!(cells(&activity), counted(&[monday, monday, tuesday])); +} + +#[test] +fn merged_activity_sums_cells_and_keeps_the_last_duplicate_on_the_left() { + let cell = |weekday, hour, conversations| SpendActivityCell { + weekday, + hour, + conversations, + }; + let merged = merge_activity( + vec![cell(1, 2, 5), cell(0, 9, 1), cell(1, 2, 3)], + &[ + cell(1, 2, 4), + cell(6, 23, u32::MAX), + cell(6, 23, 1), + cell(0, 9, 2), + ], + ); + assert_eq!( + cells(&merged), + vec![(0, 9, 3), (1, 2, 7), (6, 23, u32::MAX)] + ); +} From 7b7d1c6cb87adce16c1399071688e69d7ad3cbd8 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:38:01 +0700 Subject: [PATCH 13/41] Count spend activity cells through one ActivityHistogram (RCO-20) --- rust/src/spend_contract.rs | 77 +++++++++++++++------------- rust/src/spend_contract/opencodex.rs | 32 +++--------- 2 files changed, 48 insertions(+), 61 deletions(-) diff --git a/rust/src/spend_contract.rs b/rust/src/spend_contract.rs index 1fbf885442..92be73ca94 100644 --- a/rust/src/spend_contract.rs +++ b/rust/src/spend_contract.rs @@ -12,7 +12,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs; use std::path::PathBuf; -use chrono::{Datelike, Local, Timelike, Utc}; +use chrono::{DateTime, Datelike, Local, Timelike, Utc}; use serde::{Deserialize, Serialize}; use crate::codex_workspaces::{CodexWorkspacesIndex, ProjectUsage, SessionUsage, SourceStatus}; @@ -816,14 +816,13 @@ fn daily_points(provider_id: &str, days: u32) -> Vec { .collect() } -fn activity_from_sessions(sessions: &[SessionUsage]) -> Vec { - let mut cells: BTreeMap<(u8, u8), u32> = BTreeMap::new(); - for session in sessions { - let Some(timestamp) = session.latest_activity.or(session.started_at) else { - continue; - }; +/// Conversation counts per local (weekday, hour) cell. +#[derive(Default)] +struct ActivityHistogram(BTreeMap<(u8, u8), u32>); + +impl ActivityHistogram { + fn add_local(&mut self, timestamp: DateTime) { let local = timestamp.with_timezone(&Local); - // Weekday (0-6) and hour (0-23) both fit u8. #[allow( clippy::cast_possible_truncation, reason = "weekday (0-6) and hour (0-23) fit u8" @@ -832,17 +831,34 @@ fn activity_from_sessions(sessions: &[SessionUsage]) -> Vec { local.weekday().num_days_from_monday() as u8, local.hour() as u8, ); - let next = cells.get(&key).copied().unwrap_or(0).saturating_add(1); - cells.insert(key, next); + self.add(key, 1); + } + + fn add(&mut self, key: (u8, u8), conversations: u32) { + let count = self.0.entry(key).or_insert(0); + *count = count.saturating_add(conversations); + } + + fn into_cells(self) -> Vec { + self.0 + .into_iter() + .map(|((weekday, hour), conversations)| SpendActivityCell { + weekday, + hour, + conversations, + }) + .collect() } - cells - .into_iter() - .map(|((weekday, hour), conversations)| SpendActivityCell { - weekday, - hour, - conversations, - }) - .collect() +} + +fn activity_from_sessions(sessions: &[SessionUsage]) -> Vec { + let mut activity = ActivityHistogram::default(); + for session in sessions { + if let Some(timestamp) = session.latest_activity.or(session.started_at) { + activity.add_local(timestamp); + } + } + activity.into_cells() } fn sum_optional_cost(left: Option, right: Option) -> Option { let valid = |value: f64| value.is_finite() && value >= 0.0; @@ -993,25 +1009,16 @@ fn merge_activity( left: Vec, right: &[SpendActivityCell], ) -> Vec { - let mut cells: BTreeMap<(u8, u8), u32> = left - .into_iter() - .map(|cell| ((cell.weekday, cell.hour), cell.conversations)) - .collect(); + // Collected, not added: a duplicate cell on the left keeps the last count. + let mut activity = ActivityHistogram( + left.into_iter() + .map(|cell| ((cell.weekday, cell.hour), cell.conversations)) + .collect(), + ); for cell in right { - let current = cells.get(&(cell.weekday, cell.hour)).copied().unwrap_or(0); - cells.insert( - (cell.weekday, cell.hour), - current.saturating_add(cell.conversations), - ); + activity.add((cell.weekday, cell.hour), cell.conversations); } - cells - .into_iter() - .map(|((weekday, hour), conversations)| SpendActivityCell { - weekday, - hour, - conversations, - }) - .collect() + activity.into_cells() } #[cfg(test)] diff --git a/rust/src/spend_contract/opencodex.rs b/rust/src/spend_contract/opencodex.rs index b7cb680d1a..42e6d993cd 100644 --- a/rust/src/spend_contract/opencodex.rs +++ b/rust/src/spend_contract/opencodex.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::PathBuf; -use chrono::{DateTime, Datelike, Duration, Local, NaiveDate, TimeZone, Timelike, Utc}; +use chrono::{DateTime, Duration, NaiveDate, TimeZone, Utc}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -10,8 +10,8 @@ use crate::core::{ }; use super::{ - CostCoverageCounts, CostProvenance, CustomPricing, CustomRates, ImportedSpendSource, - SpendActivityCell, SpendDailyPoint, SpendModelRow, SpendTokenMix, + ActivityHistogram, CostCoverageCounts, CostProvenance, CustomPricing, CustomRates, + ImportedSpendSource, SpendDailyPoint, SpendModelRow, SpendTokenMix, }; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -200,7 +200,7 @@ fn aggregate_with_pricing( let mut token_mix = SpendTokenMix::default(); let mut token_total: Option = None; let mut coverage = CostCoverageCounts::default(); - let mut activity: BTreeMap<(u8, u8), u32> = BTreeMap::new(); + let mut activity = ActivityHistogram::default(); let mut models: HashMap = HashMap::new(); let mut daily: BTreeMap = BTreeMap::new(); let mut known_cost = 0.0; @@ -254,20 +254,7 @@ fn aggregate_with_pricing( } } - let local = entry.timestamp.with_timezone(&Local); - // Weekday (0-6) and hour (0-23) both fit u8. - #[allow( - clippy::cast_possible_truncation, - reason = "weekday (0-6) and hour (0-23) fit u8" - )] - let key = ( - local.weekday().num_days_from_monday() as u8, - local.hour() as u8, - ); - activity.insert( - key, - activity.get(&key).copied().unwrap_or(0).saturating_add(1), - ); + activity.add_local(entry.timestamp); let day = daily .entry(zone.date(entry.timestamp).format("%Y-%m-%d").to_string()) @@ -363,14 +350,7 @@ fn aggregate_with_pricing( total_tokens: acc.saw_tokens.then_some(acc.total_tokens), }) .collect(), - hourly_activity: activity - .into_iter() - .map(|((weekday, hour), conversations)| SpendActivityCell { - weekday, - hour, - conversations, - }) - .collect(), + hourly_activity: activity.into_cells(), }) } From 9a82896910d82b17af8293e27753d8a0dff8c453 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:39:20 +0700 Subject: [PATCH 14/41] Share one add_optional across spend sources (RCO-20) --- rust/src/spend_contract/opencodex.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/rust/src/spend_contract/opencodex.rs b/rust/src/spend_contract/opencodex.rs index 42e6d993cd..bf89c1a33c 100644 --- a/rust/src/spend_contract/opencodex.rs +++ b/rust/src/spend_contract/opencodex.rs @@ -11,7 +11,7 @@ use crate::core::{ use super::{ ActivityHistogram, CostCoverageCounts, CostProvenance, CustomPricing, CustomRates, - ImportedSpendSource, SpendDailyPoint, SpendModelRow, SpendTokenMix, + ImportedSpendSource, SpendDailyPoint, SpendModelRow, SpendTokenMix, add_optional, }; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -657,14 +657,5 @@ fn nonnegative_u64(value: Option<&Value>) -> Option { parsed } -fn add_optional(left: Option, right: Option) -> Option { - match (left, right) { - (Some(left), Some(right)) => left.checked_add(right), - (Some(left), None) => Some(left), - (None, Some(right)) => Some(right), - (None, None) => None, - } -} - #[cfg(test)] mod tests; From 680e2a6a13a0cb74d8533f15a3666c0aef18cf37 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:43:53 +0700 Subject: [PATCH 15/41] Merge spend token classes in one loop (RCO-20) --- rust/src/spend_contract.rs | 46 +++++++++++++------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/rust/src/spend_contract.rs b/rust/src/spend_contract.rs index 92be73ca94..f0fb0c7741 100644 --- a/rust/src/spend_contract.rs +++ b/rust/src/spend_contract.rs @@ -894,37 +894,21 @@ fn merge_coverage( } fn merge_token_mix(mut left: SpendTokenMix, right: &SpendTokenMix) -> SpendTokenMix { - left.overflowed_classes |= right.overflowed_classes; - left.input_tokens = merge_token_class( - left.input_tokens, - right.input_tokens, - &mut left.overflowed_classes, - 1 << 0, - ); - left.output_tokens = merge_token_class( - left.output_tokens, - right.output_tokens, - &mut left.overflowed_classes, - 1 << 1, - ); - left.cache_read_tokens = merge_token_class( - left.cache_read_tokens, - right.cache_read_tokens, - &mut left.overflowed_classes, - 1 << 2, - ); - left.cache_creation_tokens = merge_token_class( - left.cache_creation_tokens, - right.cache_creation_tokens, - &mut left.overflowed_classes, - 1 << 3, - ); - left.reasoning_tokens = merge_token_class( - left.reasoning_tokens, - right.reasoning_tokens, - &mut left.overflowed_classes, - 1 << 4, - ); + let mut overflowed_classes = left.overflowed_classes | right.overflowed_classes; + for (bit, merged, incoming) in [ + (0, &mut left.input_tokens, right.input_tokens), + (1, &mut left.output_tokens, right.output_tokens), + (2, &mut left.cache_read_tokens, right.cache_read_tokens), + ( + 3, + &mut left.cache_creation_tokens, + right.cache_creation_tokens, + ), + (4, &mut left.reasoning_tokens, right.reasoning_tokens), + ] { + *merged = merge_token_class(*merged, incoming, &mut overflowed_classes, 1 << bit); + } + left.overflowed_classes = overflowed_classes; left } From 0c3049e13c7d35c9668b2f6bfe0a4094c4193f10 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:48:19 +0700 Subject: [PATCH 16/41] Share daily history slot, sort and Codex day helpers (RCO-24) --- rust/src/cost_scanner.rs | 105 ++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 58 deletions(-) diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index 72299b655c..a04e3a1751 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -714,8 +714,6 @@ impl CostScanner { period_end: Some(window.end), ..CostSummary::default() }; - let mut daily_cost = HashMap::new(); - let mut daily_tokens = HashMap::new(); // The chart never shows more than a year of daily slots, so an // all-available window does not allocate one per day since 1970. let slot_days = match self.period { @@ -724,13 +722,9 @@ impl CostScanner { window.days.min(MAX_ROLLING_DAYS) } }; + let mut daily_cost = day_slots(window.end, slot_days, None); + let mut daily_tokens = day_slots(window.end, slot_days, 0); let mut unknown_cost_dates = HashSet::new(); - for days_ago in 0..slot_days { - let date = window.end - Duration::days(days_ago as i64); - let key = today::day_key(date); - daily_cost.insert(key.clone(), None); - daily_tokens.insert(key, 0); - } let mut quota_records = Vec::new(); let walk = self.walk_claude_records(&cutoff, cancel, |record| { @@ -767,15 +761,11 @@ impl CostScanner { } let today = TodayUsage::from_buckets(window.end, &daily_cost, &daily_tokens); - let mut daily_cost = daily_cost.into_iter().collect::>(); - daily_cost.sort_by(|left, right| left.0.cmp(&right.0)); - let mut daily_tokens = daily_tokens.into_iter().collect::>(); - daily_tokens.sort_by(|left, right| left.0.cmp(&right.0)); ClaudeChartSnapshot { today, summary, - daily_cost, - daily_tokens, + daily_cost: sorted_days(daily_cost), + daily_tokens: sorted_days(daily_tokens), daily_incomplete: incomplete_report.daily_sorted(), quota_history: ClaudeQuotaHistoryScan { records: quota_records, @@ -1326,6 +1316,35 @@ pub fn has_cost_usage_sources() -> bool { .any(|dir| dir.exists()) } +/// One `init` slot per day key, from `end` back over `days` days. +fn day_slots(end: NaiveDate, days: u32, init: T) -> HashMap { + (0..days) + .map(|days_ago| { + let date = end - Duration::days(i64::from(days_ago)); + (today::day_key(date), init.clone()) + }) + .collect() +} + +fn sorted_days(days: HashMap) -> Vec<(String, T)> { + let mut days: Vec<_> = days.into_iter().collect(); + days.sort_by(|left, right| left.0.cmp(&right.0)); + days +} + +/// One packed Codex cache day summarized on its own, with its cost. +fn codex_day_summary( + day_key: &str, + models: &HashMap>, +) -> Option<(CostSummary, f64)> { + let day = CostUsageDayRange::parse_day_key(day_key)?; + let one_day = HashMap::from([(day_key.to_string(), models.clone())]); + let mut summary = CostSummary::default(); + let (cost, _) = + add_codex_days_map_to_summary(&mut summary, &one_day, &CostUsageDayRange::new(day, day)); + Some((summary, cost)) +} + /// Get daily cost history for the last N days /// Returns calendar-preserving daily costs sorted by date. `None` means the day /// is unscanned or contains unpriced Codex usage; `Some(0)` is a known zero. @@ -1343,17 +1362,11 @@ pub fn get_daily_cost_and_incomplete_history(provider: &str, days: u32) -> Daily let mut daily_incomplete = Vec::new(); let scanner = CostScanner::new(days); let today = cost_bucket_zone().date(Utc::now()); - let mut daily_costs: HashMap> = HashMap::new(); - - // Initialize all days with 0 - for days_ago in 0..days { - let date = today - Duration::days(days_ago as i64); - let date_str = today::day_key(date); - daily_costs.insert( - date_str, - (provider != "codex" && provider != "claude" && provider != "pi").then_some(0.0), - ); - } + let mut daily_costs = day_slots( + today, + days, + (provider != "codex" && provider != "claude" && provider != "pi").then_some(0.0), + ); match provider { "codex" => { @@ -1380,15 +1393,10 @@ pub fn get_daily_cost_and_incomplete_history(provider: &str, days: u32) -> Daily let Some(slot) = daily_costs.get_mut(day_key) else { continue; }; - let Some(day) = CostUsageDayRange::parse_day_key(day_key) else { + let Some((day, cost)) = codex_day_summary(day_key, models) else { continue; }; - let day_range = CostUsageDayRange::new(day, day); - let mut one_day = HashMap::new(); - one_day.insert(day_key.clone(), models.clone()); - let mut scratch = CostSummary::default(); - let (cost, _) = add_codex_days_map_to_summary(&mut scratch, &one_day, &day_range); - *slot = (!scratch.model_pricing_completeness.is_partial()).then_some(cost); + *slot = (!day.model_pricing_completeness.is_partial()).then_some(cost); } } "claude" => { @@ -1431,10 +1439,7 @@ pub fn get_daily_cost_and_incomplete_history(provider: &str, days: u32) -> Daily _ => {} } - // Convert to sorted vector - let mut result: Vec<(String, Option)> = daily_costs.into_iter().collect(); - result.sort_by(|a, b| a.0.cmp(&b.0)); - (result, daily_incomplete) + (sorted_days(daily_costs), daily_incomplete) } /// Daily token totals (provider-aware, see [`cache_is_separate_from_input`]) @@ -1445,16 +1450,9 @@ pub fn get_daily_cost_and_incomplete_history(provider: &str, days: u32) -> Daily pub fn get_daily_token_history(provider: &str, days: u32) -> (Vec<(String, u64)>, bool) { let scanner = CostScanner::new(days); let today = cost_bucket_zone().date(Utc::now()); - let mut daily_tokens: HashMap = HashMap::new(); + let mut daily_tokens = day_slots(today, days, 0u64); let mut covered_days: HashSet = HashSet::new(); - // Initialize all days with 0 - for days_ago in 0..days { - let date = today - Duration::days(days_ago as i64); - let date_str = today::day_key(date); - daily_tokens.insert(date_str, 0); - } - match provider { "codex" => { // Warm/refresh the disk cache, then read exact local token totals @@ -1462,20 +1460,13 @@ pub fn get_daily_token_history(provider: &str, days: u32) -> (Vec<(String, u64)> // uses. let (_summary, _stats, cache) = scanner.scan_codex_detailed_with_cache(None); for (day_key, models) in &cache.days { - if !daily_tokens.contains_key(day_key) { + let Some(slot) = daily_tokens.get_mut(day_key) else { continue; - } - let Some(day) = CostUsageDayRange::parse_day_key(day_key) else { + }; + let Some((day, _)) = codex_day_summary(day_key, models) else { continue; }; - let day_range = CostUsageDayRange::new(day, day); - let mut one_day = HashMap::new(); - one_day.insert(day_key.clone(), models.clone()); - let mut scratch = CostSummary::default(); - add_codex_days_map_to_summary(&mut scratch, &one_day, &day_range); - if let Some(slot) = daily_tokens.get_mut(day_key) { - *slot = scratch.total_tokens_for_provider("codex"); - } + *slot = day.total_tokens_for_provider("codex"); covered_days.insert(day_key.clone()); } } @@ -1505,9 +1496,7 @@ pub fn get_daily_token_history(provider: &str, days: u32) -> (Vec<(String, u64)> _ => {} } - // Convert to sorted vector - let mut result: Vec<(String, u64)> = daily_tokens.into_iter().collect(); - result.sort_by(|a, b| a.0.cmp(&b.0)); + let result = sorted_days(daily_tokens); let incomplete = if matches!(provider, "claude" | "pi") { // A complete filesystem scan covers the requested window even when From 1cdb06b6df8101eae9503c0b9a707ae929bac8ec Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:50:22 +0700 Subject: [PATCH 17/41] Count in-range Codex cache sessions through one helper (RCO-25) --- rust/src/codex_costs.rs | 23 ++++++++++++++++++----- rust/src/cost_scanner/codex/scan.rs | 21 +++------------------ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/rust/src/codex_costs.rs b/rust/src/codex_costs.rs index 1992d32a8c..1351c173c1 100644 --- a/rust/src/codex_costs.rs +++ b/rust/src/codex_costs.rs @@ -55,17 +55,30 @@ fn codex_today_summary(history: &CostSummary, cache: &CostUsageCache) -> CostSum }; let (cost, _) = add_codex_days_map_to_summary(&mut summary, &cache.days, &range); summary.total_cost_usd = cost; - summary.sessions_count = cache - .files - .values() - .filter(|usage| usage.days.contains_key(&range.until_key)) - .count() + summary.sessions_count = codex_cache_sessions_in_range(cache, &range) .try_into() .unwrap_or(u32::MAX); summary.known_zero = summary.history_coverage_established && summary.sessions_count == 0; summary } +/// Cached Codex files with usage on at least one day of `range`. +pub(crate) fn codex_cache_sessions_in_range( + cache: &CostUsageCache, + range: &CostUsageDayRange, +) -> usize { + cache + .files + .values() + .filter(|usage| { + usage + .days + .keys() + .any(|day| CostUsageDayRange::is_in_range(day, &range.since_key, &range.until_key)) + }) + .count() +} + fn coverage_from_summary(summary: &CostSummary) -> CostCoverageCounts { let mut model_names = HashSet::new(); model_names.extend(summary.by_model.keys().cloned()); diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index c93068252d..d560d99f90 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -1,5 +1,6 @@ use super::cache_days::days_from_codex_source_rows; use super::*; +use crate::codex_costs::codex_cache_sessions_in_range; use crate::core::{ CodexSourceRowCache, CodexSourceUsageRow, read_source_rows, recover_rows, row_cache, row_cache_matches, row_cache_needs_recovery, @@ -245,15 +246,7 @@ pub(super) fn scan_codex_detailed_with_cache( summary.total_cost_usd += cost; // Session count is a display field; the cache holds far fewer files than u32::MAX. #[allow(clippy::cast_possible_truncation, reason = "cache file counts fit u32")] - let sessions_count = cache - .files - .values() - .filter(|usage| { - usage.days.keys().any(|day| { - CostUsageDayRange::is_in_range(day, &range.since_key, &range.until_key) - }) - }) - .count() as u32; + let sessions_count = codex_cache_sessions_in_range(&cache, &range) as u32; summary.sessions_count = sessions_count; // Pi-compatible sessions are outside the Codex JSONL cache. @@ -548,15 +541,7 @@ pub(super) fn scan_codex_detailed_with_cache( rebuilt.total_cost_usd += native_cost; #[allow(clippy::cast_possible_truncation, reason = "cache file counts fit u32")] { - rebuilt.sessions_count = summary_cache - .files - .values() - .filter(|usage| { - usage.days.keys().any(|day| { - CostUsageDayRange::is_in_range(day, &range.since_key, &range.until_key) - }) - }) - .count() as u32; + rebuilt.sessions_count = codex_cache_sessions_in_range(&summary_cache, &range) as u32; } let cancelled_with_missing_cache_rows = is_cancelled(cancel) && cache.files.keys().any(|path| !Path::new(path).exists()); From 0f01140a8e295c370e223d9f3bc1b4642f430fd2 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:53:24 +0700 Subject: [PATCH 18/41] Reuse the Codex sessions dirs within one detailed scan (RCO-25) --- rust/src/cost_scanner/codex/scan.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/src/cost_scanner/codex/scan.rs b/rust/src/cost_scanner/codex/scan.rs index d560d99f90..6871affcfd 100644 --- a/rust/src/cost_scanner/codex/scan.rs +++ b/rust/src/cost_scanner/codex/scan.rs @@ -196,7 +196,6 @@ pub(super) fn scan_codex_detailed_with_cache( let cache_root = scanner.cache_root.as_deref(); let mut cache = JsonlScanner::load_cache(ProviderId::Codex, cache_root); - let sessions_dirs = scanner.get_codex_sessions_dirs(); let priority_metadata_key = codex_priority_metadata_key(scanner); let pending_scan = CodexPendingScanContext::new( &cache, From 29a45edaf9e53361e0998b27ebbcc2835e637b8e Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:57:47 +0700 Subject: [PATCH 19/41] Share Codex file billing, skip and unresolved-fork helpers (RCO-18) --- rust/src/cost_scanner/codex.rs | 215 ++++++++++++++------------------- 1 file changed, 90 insertions(+), 125 deletions(-) diff --git a/rust/src/cost_scanner/codex.rs b/rust/src/cost_scanner/codex.rs index 44762d8871..35037fc8a6 100644 --- a/rust/src/cost_scanner/codex.rs +++ b/rust/src/cost_scanner/codex.rs @@ -475,17 +475,7 @@ impl CostScanner { && cache_entry_is_fresh(entry) && identity_matches_cached(entry) { - let (session_cost, has_tokens) = - add_codex_days_map_to_summary(summary, &entry.days, range); - if has_tokens { - summary.total_cost_usd += session_cost; - summary.sessions_count += 1; - } - stats.files_skipped = stats.files_skipped.saturating_add(1); - return CodexFileScanOutcome { - bytes_read: 0, - is_complete: true, - }; + return skip_fresh_codex_file(summary, stats, entry, range); } let session_metadata = if let Some(prepared) = prepared_candidate { @@ -561,6 +551,40 @@ impl CostScanner { state.history_base_thread_id != session_metadata.history_base_thread_id })) }); + let mark_unresolved_fork = |cache: &mut CostUsageCache, + stats: &mut CostScanStats, + path_key: String, + bytes_read: i64| { + // Priority evidence lives in `codex_fork_rows`; a fork whose parent + // baseline is unresolved or ambiguous must not keep stale rows. + cache.codex_fork_rows.remove(&path_key); + cache.files.insert( + path_key, + CostUsageFileUsage { + mtime_unix_ms: mtime_ms, + size, + codex_file_identity: file_identity.clone(), + days: HashMap::new(), + parsed_bytes: Some(0), + codex_scan_target_size: None, + last_model: None, + last_totals: None, + codex_token_timestamps_monotonic: None, + codex_last_token_timestamp: None, + codex_session_id: codex_session_id.clone(), + codex_forked_from_id: codex_forked_from_id.clone(), + codex_fork_accounting_state: None, + codex_lineage, + codex_fork_timestamp: codex_fork_timestamp.clone(), + codex_unresolved_fork_parent: true, + }, + ); + stats.files_parsed = stats.files_parsed.saturating_add(1); + CodexFileScanOutcome { + bytes_read, + is_complete: false, + } + }; let is_fork = codex_lineage.uses_parent_baseline(); let cached_fork_state_matches = cached_fork_accounting_state.as_ref().is_some_and(|state| { @@ -597,35 +621,7 @@ impl CostScanner { ); if accounting_mode.is_unresolved() { - // Priority evidence lives in `codex_fork_rows`; a fork that could - // not resolve its parent baseline must not keep stale rows. - cache.codex_fork_rows.remove(&path_key); - cache.files.insert( - path_key, - CostUsageFileUsage { - mtime_unix_ms: mtime_ms, - size, - codex_file_identity: file_identity.clone(), - days: HashMap::new(), - parsed_bytes: Some(0), - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id, - codex_forked_from_id, - codex_fork_accounting_state: None, - codex_lineage, - codex_fork_timestamp, - codex_unresolved_fork_parent: true, - }, - ); - stats.files_parsed = stats.files_parsed.saturating_add(1); - return CodexFileScanOutcome { - bytes_read: 0, - is_complete: false, - }; + return mark_unresolved_fork(cache, stats, path_key, 0); } if let Some(entry) = &cached @@ -634,17 +630,7 @@ impl CostScanner { && !cached_identity_changed && !accounting_mode.requires_cached_reparse() { - let (session_cost, has_tokens) = - add_codex_days_map_to_summary(summary, &entry.days, range); - if has_tokens { - summary.total_cost_usd += session_cost; - summary.sessions_count += 1; - } - stats.files_skipped = stats.files_skipped.saturating_add(1); - return CodexFileScanOutcome { - bytes_read: 0, - is_complete: true, - }; + return skip_fresh_codex_file(summary, stats, entry, range); } stats.codex_history_read_paths.push(path_key.clone()); @@ -700,12 +686,8 @@ impl CostScanner { } else { cache.codex_fork_rows.remove(&path_key); } - let (session_cost, has_tokens) = - add_codex_days_map_to_summary(summary, &days, range); - if has_tokens { - summary.total_cost_usd += session_cost; - summary.sessions_count += 1; - } + let billed = add_codex_days_map_to_summary(summary, &days, range); + bill_codex_session(summary, billed); let outcome = CodexFileScanOutcome { bytes_read: parse_result.bytes_read, is_complete: parse_result.is_complete, @@ -764,10 +746,8 @@ impl CostScanner { .and_then(|entry| codex_resumable_scan_target_size(size, entry)) }) .flatten(); - let parse_result = match match &accounting_mode { - CodexAccountingMode::Standard => JsonlScanner::parse_codex( - path, - range, + let (parse_mode, target_size) = match &accounting_mode { + CodexAccountingMode::Standard => ( CodexParseMode::Standard { start_offset: 0, initial_model: None, @@ -775,55 +755,47 @@ impl CostScanner { previous_token_timestamp: None, token_timestamps_monotonic: None, }, - cancel, None, - max_bytes_to_read, ), CodexAccountingMode::Baseline { baseline, paginated_continuation, .. } => match fork_resume { - Some(resume) => JsonlScanner::parse_codex( - path, - range, + Some(resume) => ( CodexParseMode::ResumeParentBaseline(resume.parse), - cancel, Some(resume.target_size), - max_bytes_to_read, ), - None => { - JsonlScanner::parse_codex( - path, - range, - CodexParseMode::ParentBaseline { - baseline: baseline.clone(), - paginated_continuation: *paginated_continuation, - // A parse from byte zero replays the inherited counters - // itself. Only a resumed parse carries their used-up - // remainder, as upstream restores fork state only then. - remaining_inherited_totals: None, - }, - cancel, - parse_target_size, - max_bytes_to_read, - ) - } + None => ( + CodexParseMode::ParentBaseline { + baseline: baseline.clone(), + paginated_continuation: *paginated_continuation, + // A parse from byte zero replays the inherited counters + // itself. Only a resumed parse carries their used-up + // remainder, as upstream restores fork state only then. + remaining_inherited_totals: None, + }, + parse_target_size, + ), }, - CodexAccountingMode::InferSubagent { start_ordinal } => JsonlScanner::parse_codex( - path, - range, + CodexAccountingMode::InferSubagent { start_ordinal } => ( CodexParseMode::InferSubagent { start_ordinal: *start_ordinal, }, - cancel, parse_target_size, - max_bytes_to_read, ), CodexAccountingMode::Unresolved => { unreachable!("unresolved forks return before parsing") } - } { + }; + let parse_result = match JsonlScanner::parse_codex( + path, + range, + parse_mode, + cancel, + target_size, + max_bytes_to_read, + ) { Ok(result) => result, Err(_) => return CodexFileScanOutcome::default(), }; @@ -834,35 +806,7 @@ impl CostScanner { || (accounting_mode.infers_subagent_baseline() && !parse_result.fork_baseline_locally_resolved) { - // Priority evidence lives in `codex_fork_rows`; an ambiguous fork - // baseline must not keep stale rows. - cache.codex_fork_rows.remove(&path_key); - cache.files.insert( - path_key, - CostUsageFileUsage { - mtime_unix_ms: mtime_ms, - size, - codex_file_identity: file_identity.clone(), - days: HashMap::new(), - parsed_bytes: Some(0), - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id, - codex_forked_from_id, - codex_fork_accounting_state: None, - codex_lineage, - codex_fork_timestamp, - codex_unresolved_fork_parent: true, - }, - ); - stats.files_parsed = stats.files_parsed.saturating_add(1); - return CodexFileScanOutcome { - bytes_read: parse_result.bytes_read, - is_complete: false, - }; + return mark_unresolved_fork(cache, stats, path_key, parse_result.bytes_read); } // A resumed parse returns only the suffix; its prefix is the cached day // map, which is billed as the non-fork resume path bills it. @@ -882,15 +826,12 @@ impl CostScanner { } else { cache.codex_fork_rows.remove(&path_key); } - let (session_cost, has_tokens) = if resumed { + let billed = if resumed { add_codex_days_map_to_summary(summary, &days, range) } else { add_codex_records_to_summary(summary, &parse_result.records, range) }; - if has_tokens { - summary.total_cost_usd += session_cost; - summary.sessions_count += 1; - } + bill_codex_session(summary, billed); let outcome = CodexFileScanOutcome { bytes_read: parse_result.bytes_read, is_complete: parse_result.is_complete, @@ -943,6 +884,30 @@ impl CostScanner { } } +/// Count one file's billed (cost, has_tokens) as a session when it has tokens. +fn bill_codex_session(summary: &mut CostSummary, (cost, has_tokens): (f64, bool)) { + if has_tokens { + summary.total_cost_usd += cost; + summary.sessions_count += 1; + } +} + +/// Bill a fresh cached file from its day map without reading it. +fn skip_fresh_codex_file( + summary: &mut CostSummary, + stats: &mut CostScanStats, + entry: &CostUsageFileUsage, + range: &CostUsageDayRange, +) -> CodexFileScanOutcome { + let billed = add_codex_days_map_to_summary(summary, &entry.days, range); + bill_codex_session(summary, billed); + stats.files_skipped = stats.files_skipped.saturating_add(1); + CodexFileScanOutcome { + bytes_read: 0, + is_complete: true, + } +} + /// Request rows for a fork-shaped file, keeping exactly the records that /// [`merge_codex_records_into_days`] folds into its day totals. fn codex_fork_rows_from_records( From 8743d288dbca42579fae0f46d42216bd32a6c665 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:02:11 +0700 Subject: [PATCH 20/41] Delete the unused duplicate Codex JSONL line bound --- rust/src/core/jsonl_scanner.rs | 3 --- rust/src/core/jsonl_scanner/codex/helpers.rs | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/rust/src/core/jsonl_scanner.rs b/rust/src/core/jsonl_scanner.rs index ce06b24fc9..7fd8998f58 100755 --- a/rust/src/core/jsonl_scanner.rs +++ b/rust/src/core/jsonl_scanner.rs @@ -76,9 +76,6 @@ where deserializer.deserialize_map(NonemptyObjectVisitor) } -/// Maximum retained Codex JSONL line size (upstream session-metadata bound). -const CODEX_JSONL_MAX_LINE_BYTES: usize = 256 * 1024; - /// Default scanner-side refresh debounce (upstream CostUsageScanner). pub const DEFAULT_COST_SCAN_REFRESH_MIN_INTERVAL_SECS: u64 = 60; /// Default number of dirty Codex rollouts inspected in one refresh. diff --git a/rust/src/core/jsonl_scanner/codex/helpers.rs b/rust/src/core/jsonl_scanner/codex/helpers.rs index ae61240cc5..eb2aa04a94 100644 --- a/rust/src/core/jsonl_scanner/codex/helpers.rs +++ b/rust/src/core/jsonl_scanner/codex/helpers.rs @@ -4,6 +4,7 @@ use serde::Deserialize; use serde_json::Value; use std::io::BufRead; +/// Maximum retained Codex JSONL line size (upstream session-metadata bound). pub(super) const CODEX_JSONL_MAX_LINE_BYTES: usize = 256 * 1024; #[derive(Debug, Deserialize)] From d3d99bec8331805210b2ec437bd29508dae18cf6 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:02:39 +0700 Subject: [PATCH 21/41] Delete the unused fast Codex last-usage delta --- rust/src/core/jsonl_scanner/codex/helpers.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/rust/src/core/jsonl_scanner/codex/helpers.rs b/rust/src/core/jsonl_scanner/codex/helpers.rs index eb2aa04a94..24be202fdc 100644 --- a/rust/src/core/jsonl_scanner/codex/helpers.rs +++ b/rust/src/core/jsonl_scanner/codex/helpers.rs @@ -680,13 +680,3 @@ pub(super) fn last_usage_delta(last: &Value) -> (i64, i64, i64, Option) { totals.reasoning, ) } - -pub(super) fn fast_last_usage_delta(last: CodexFastTotals) -> (i64, i64, i64, Option) { - let totals = codex_totals_from_fast(last); - ( - totals.input.max(0), - totals.cached.max(0), - totals.output.max(0), - totals.reasoning, - ) -} From 760d6cfd4f282f73fbf06e13a75b805a0fb69f82 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:06:25 +0700 Subject: [PATCH 22/41] Move test-only Codex JSONL helpers into the scanner tests (RCO-03) --- rust/src/core/jsonl_scanner/codex.rs | 5 ++-- rust/src/core/jsonl_scanner/codex/helpers.rs | 14 --------- rust/src/core/jsonl_scanner/codex/parser.rs | 14 --------- rust/src/core/jsonl_scanner/tests.rs | 30 ++++++++++++++++++++ 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/rust/src/core/jsonl_scanner/codex.rs b/rust/src/core/jsonl_scanner/codex.rs index d0374a14f9..78310fb7d3 100644 --- a/rust/src/core/jsonl_scanner/codex.rs +++ b/rust/src/core/jsonl_scanner/codex.rs @@ -89,9 +89,8 @@ pub(crate) fn codex_cache_stamp_schema_version(cache: &mut CostUsageCache) { #[cfg(test)] use helpers::{ - CodexFastPayload, CodexFastTotals, bare_usage_totals, codex_timestamp_day_key, - codex_totals_from_fast, fast_totals_from_payload, is_candidate_codex_line, last_usage_delta, - parse_codex_timestamp, read_token_totals, + CodexFastPayload, CodexFastTotals, bare_usage_totals, codex_totals_from_fast, + fast_totals_from_payload, is_candidate_codex_line, parse_codex_timestamp, read_token_totals, }; impl JsonlScanner { diff --git a/rust/src/core/jsonl_scanner/codex/helpers.rs b/rust/src/core/jsonl_scanner/codex/helpers.rs index 24be202fdc..cb957dc85f 100644 --- a/rust/src/core/jsonl_scanner/codex/helpers.rs +++ b/rust/src/core/jsonl_scanner/codex/helpers.rs @@ -371,10 +371,6 @@ pub(super) fn event_payload_type(obj: &Value) -> Option<&str> { .and_then(Value::as_str) } -pub(super) fn codex_timestamp_day_key(timestamp: &str) -> Option { - parse_codex_timestamp(timestamp).map(|parsed| parsed.day_key()) -} - #[derive(Debug, Clone)] pub(super) struct ParsedCodexTimestamp { pub(super) parsed: Option>, @@ -670,13 +666,3 @@ fn optional_token_i64(value: &Value, key: &str) -> Option { pub(super) fn clamp_reasoning(reasoning: Option, output: i64) -> Option { reasoning.map(|tokens| tokens.max(0).min(output.max(0))) } - -pub(super) fn last_usage_delta(last: &Value) -> (i64, i64, i64, Option) { - let totals = read_token_totals(last); - ( - totals.input.max(0), - totals.cached.max(0), - totals.output.max(0), - totals.reasoning, - ) -} diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index 948c7744e3..06da915bf4 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -224,16 +224,6 @@ fn totals_delta(last: &CodexTotals, total: &CodexTotals) -> CodexTotals { } impl CodexParserState { - pub(super) fn new(initial_model: Option, initial_totals: Option) -> Self { - Self::from_mode(CodexParseMode::Standard { - start_offset: 0, - initial_model, - initial_totals, - previous_token_timestamp: None, - token_timestamps_monotonic: None, - }) - } - pub(super) fn from_mode(mode: CodexParseMode) -> Self { let fork_parse_baseline = match &mode { CodexParseMode::ParentBaseline { baseline, .. } => Some(baseline.clone()), @@ -372,10 +362,6 @@ impl CodexParserState { .is_some_and(|inference| inference.locally_confirmed) } - pub(super) fn process_line(&mut self, line: &str, range: &CostUsageDayRange) { - self.process_line_with_source_offset(line, range, 0); - } - pub(super) fn process_line_with_source_offset( &mut self, line: &str, diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 88fa8641a9..f63e6d5fe5 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -2,6 +2,36 @@ use super::*; use chrono::TimeZone; use std::io::Write; +impl CodexParserState { + fn new(initial_model: Option, initial_totals: Option) -> Self { + Self::from_mode(CodexParseMode::Standard { + start_offset: 0, + initial_model, + initial_totals, + previous_token_timestamp: None, + token_timestamps_monotonic: None, + }) + } + + fn process_line(&mut self, line: &str, range: &CostUsageDayRange) { + self.process_line_with_source_offset(line, range, 0); + } +} + +fn codex_timestamp_day_key(timestamp: &str) -> Option { + parse_codex_timestamp(timestamp).map(|parsed| parsed.day_key()) +} + +fn last_usage_delta(last: &Value) -> (i64, i64, i64, Option) { + let totals = read_token_totals(last); + ( + totals.input.max(0), + totals.cached.max(0), + totals.output.max(0), + totals.reasoning, + ) +} + #[test] fn test_day_range() { let since = NaiveDate::from_ymd_opt(2026, 1, 15).unwrap(); From ee454890be406d01ad207573bf917596be8e8576 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:06:26 +0700 Subject: [PATCH 23/41] Drop the blanket dead_code allow on the JSONL scanner --- rust/src/core/jsonl_scanner.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rust/src/core/jsonl_scanner.rs b/rust/src/core/jsonl_scanner.rs index 7fd8998f58..9a3b65db5d 100755 --- a/rust/src/core/jsonl_scanner.rs +++ b/rust/src/core/jsonl_scanner.rs @@ -3,11 +3,6 @@ //! Incremental log file parsing for Codex and Claude session logs. //! Supports file-level caching to avoid re-parsing unchanged files. -#![allow( - dead_code, - reason = "scanner types are deserialized from JSONL for parsing but not all are read" -)] - use crate::core::{CostUsagePricing, ProviderId}; use chrono::{DateTime, NaiveDate, Utc}; From 6b16b42132be2c89b8228934341c0380253279e6 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:09:44 +0700 Subject: [PATCH 24/41] Move the test-only Codex display label lookup into pricing tests (RCO-03) --- rust/src/core/cost_pricing.rs | 8 -------- rust/src/core/cost_pricing_tests.rs | 10 ++++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/rust/src/core/cost_pricing.rs b/rust/src/core/cost_pricing.rs index bde8af627e..b9293f5b6e 100755 --- a/rust/src/core/cost_pricing.rs +++ b/rust/src/core/cost_pricing.rs @@ -329,14 +329,6 @@ impl CostUsagePricing { codex_routed_pricing::counts_toward_codex_subscription(model) } - /// Get the display label for a Codex model (e.g. "Research Preview") - pub fn codex_display_label(model: &str) -> Option<&'static str> { - let key = Self::normalize_codex_model(model); - CODEX_PRICING - .get(key.as_str()) - .and_then(|p| p.display_label) - } - /// Strip Fast/priority suffix to find the base model for pricing lookup. /// /// Fast-tier models ("gpt-5.5-fast", "gpt-5.6-sol-priority") price as the diff --git a/rust/src/core/cost_pricing_tests.rs b/rust/src/core/cost_pricing_tests.rs index f8168aeec4..74807dde36 100644 --- a/rust/src/core/cost_pricing_tests.rs +++ b/rust/src/core/cost_pricing_tests.rs @@ -1,6 +1,16 @@ use super::codex_routed_pricing; use super::*; +impl CostUsagePricing { + /// Get the display label for a Codex model (e.g. "Research Preview") + fn codex_display_label(model: &str) -> Option<&'static str> { + let key = Self::normalize_codex_model(model); + CODEX_PRICING + .get(key.as_str()) + .and_then(|p| p.display_label) + } +} + #[test] fn test_normalize_codex_model() { assert_eq!(CostUsagePricing::normalize_codex_model("gpt-5"), "gpt-5"); From 279e208005a086106fc790a712a052069054b405 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:11:13 +0700 Subject: [PATCH 25/41] Move the test-only model name formatter into pricing tests (RCO-03) --- rust/src/core/cost_pricing.rs | 48 ----------------------------- rust/src/core/cost_pricing_tests.rs | 48 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/rust/src/core/cost_pricing.rs b/rust/src/core/cost_pricing.rs index b9293f5b6e..f02f6e2fad 100755 --- a/rust/src/core/cost_pricing.rs +++ b/rust/src/core/cost_pricing.rs @@ -471,54 +471,6 @@ impl CostUsagePricing { output_tokens, ) } - - /// Format model name for display (e.g., "claude-3.5-sonnet" → "Sonnet 3.5") - pub fn format_model_name(model: &str) -> String { - let lower = model.to_lowercase(); - - // GPT models: format as "GPT-{version}[ Mini| Nano]" - if lower.contains("gpt-") { - let version = regex_lite::Regex::new(r"gpt-(\d+(?:\.\d+)?)") - .ok() - .and_then(|re| re.captures(&lower)) - .and_then(|c| c.get(1)) - .map(|m| m.as_str().to_string()); - - let suffix = if lower.contains("nano") { - " Nano" - } else if lower.contains("mini") { - " Mini" - } else { - "" - }; - - return match version { - Some(v) => format!("GPT-{}{}", v, suffix), - None => model.to_string(), - }; - } - - // Claude models: extract version and family - let version = regex_lite::Regex::new(r"(\d+(?:\.\d+)?)") - .ok() - .and_then(|re| re.find(&lower)) - .map(|m| m.as_str().to_string()); - - let family = if lower.contains("opus") { - "Opus" - } else if lower.contains("sonnet") { - "Sonnet" - } else if lower.contains("haiku") { - "Haiku" - } else { - return model.to_string(); - }; - - match version { - Some(v) => format!("{} {}", family, v), - None => family.to_string(), - } - } } #[cfg(test)] diff --git a/rust/src/core/cost_pricing_tests.rs b/rust/src/core/cost_pricing_tests.rs index 74807dde36..df11e7809c 100644 --- a/rust/src/core/cost_pricing_tests.rs +++ b/rust/src/core/cost_pricing_tests.rs @@ -9,6 +9,54 @@ impl CostUsagePricing { .get(key.as_str()) .and_then(|p| p.display_label) } + + /// Format model name for display (e.g., "claude-3.5-sonnet" → "Sonnet 3.5") + fn format_model_name(model: &str) -> String { + let lower = model.to_lowercase(); + + // GPT models: format as "GPT-{version}[ Mini| Nano]" + if lower.contains("gpt-") { + let version = regex_lite::Regex::new(r"gpt-(\d+(?:\.\d+)?)") + .ok() + .and_then(|re| re.captures(&lower)) + .and_then(|c| c.get(1)) + .map(|m| m.as_str().to_string()); + + let suffix = if lower.contains("nano") { + " Nano" + } else if lower.contains("mini") { + " Mini" + } else { + "" + }; + + return match version { + Some(v) => format!("GPT-{}{}", v, suffix), + None => model.to_string(), + }; + } + + // Claude models: extract version and family + let version = regex_lite::Regex::new(r"(\d+(?:\.\d+)?)") + .ok() + .and_then(|re| re.find(&lower)) + .map(|m| m.as_str().to_string()); + + let family = if lower.contains("opus") { + "Opus" + } else if lower.contains("sonnet") { + "Sonnet" + } else if lower.contains("haiku") { + "Haiku" + } else { + return model.to_string(); + }; + + match version { + Some(v) => format!("{} {}", family, v), + None => family.to_string(), + } + } } #[test] From 1c957c82ffda3b5b826b744af56b1b3798c41e61 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:12:19 +0700 Subject: [PATCH 26/41] Move the test-only snapshot Codex cost wrapper into pricing tests (RCO-03) --- rust/src/core/cost_pricing/codex.rs | 17 ----------------- rust/src/core/cost_pricing_tests.rs | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/rust/src/core/cost_pricing/codex.rs b/rust/src/core/cost_pricing/codex.rs index 2262980f29..dd9b377c62 100644 --- a/rust/src/core/cost_pricing/codex.rs +++ b/rust/src/core/cost_pricing/codex.rs @@ -269,23 +269,6 @@ impl CostUsagePricing { ) } - pub fn codex_cost_usd_with_pricing_snapshot( - model: &str, - input_tokens: u64, - cached_input_tokens: u64, - output_tokens: u64, - pricing_snapshot: Option<&models_dev_pricing::ModelsDevPricingSnapshot>, - ) -> Option { - Self::codex_cost_usd_with_cache_write_and_pricing_snapshot( - model, - input_tokens, - cached_input_tokens, - 0, - output_tokens, - pricing_snapshot, - ) - } - pub(super) fn codex_cost_usd_with_cache_write_and_pricing_snapshot( model: &str, input_tokens: u64, diff --git a/rust/src/core/cost_pricing_tests.rs b/rust/src/core/cost_pricing_tests.rs index df11e7809c..028e90c27a 100644 --- a/rust/src/core/cost_pricing_tests.rs +++ b/rust/src/core/cost_pricing_tests.rs @@ -57,6 +57,23 @@ impl CostUsagePricing { None => family.to_string(), } } + + fn codex_cost_usd_with_pricing_snapshot( + model: &str, + input_tokens: u64, + cached_input_tokens: u64, + output_tokens: u64, + pricing_snapshot: Option<&models_dev_pricing::ModelsDevPricingSnapshot>, + ) -> Option { + Self::codex_cost_usd_with_cache_write_and_pricing_snapshot( + model, + input_tokens, + cached_input_tokens, + 0, + output_tokens, + pricing_snapshot, + ) + } } #[test] From c4bb4921665a5ae0ca0c5006baa163a3a039c1c5 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:15:56 +0700 Subject: [PATCH 27/41] Move the test-only Claude pricing oracle into scanner tests (RCO-03) --- rust/src/cost_scanner.rs | 2 - rust/src/cost_scanner/claude_pricing.rs | 51 ------------------------- rust/src/cost_scanner/tests.rs | 51 +++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 53 deletions(-) diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index a04e3a1751..4a8c2212bd 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -47,8 +47,6 @@ mod window; pub use claude_incomplete::ClaudeIncompleteReport; use claude_incomplete::ClaudeIncompleteTracker; use claude_pricing::ClaudeScanPricingResolver; -#[cfg(test)] -use claude_pricing::{ClaudePricing, FALLBACK_CLAUDE_MODEL}; use claude_usage::{ ClaudeUsageDedupKey, claude_usage_dedup_key, session_id_from_entries, should_count_claude_record, diff --git a/rust/src/cost_scanner/claude_pricing.rs b/rust/src/cost_scanner/claude_pricing.rs index 5d3d251036..a0c17953f6 100644 --- a/rust/src/cost_scanner/claude_pricing.rs +++ b/rust/src/cost_scanner/claude_pricing.rs @@ -3,57 +3,6 @@ use std::collections::HashMap; pub(super) const FALLBACK_CLAUDE_MODEL: &str = "claude-sonnet-4-6"; -#[cfg(test)] -pub(super) struct ClaudePricing; - -#[cfg(test)] -impl ClaudePricing { - pub(super) fn cost_usd_with_cache_ttl( - model: &str, - input: u64, - cache_create: u64, - cache_create_1h: u64, - cache_read: u64, - output: u64, - ) -> f64 { - let cache_create_1h = cache_create_1h.min(cache_create); - let cache_create_5m = cache_create.saturating_sub(cache_create_1h); - - // Standard buckets (input, cache-read, 5-minute cache-write, output), - // including any long-context tiering, come from the canonical table. - // Unknown/retired models fall back to Sonnet pricing. - #[allow( - clippy::cast_possible_truncation, - reason = "clamped to i32::MAX before casting" - )] - let clamp = |v: u64| v.min(i32::MAX as u64) as i32; - let base = CostUsagePricing::claude_cost_usd( - model, - clamp(input), - clamp(cache_read), - clamp(cache_create_5m), - clamp(output), - ) - .or_else(|| { - CostUsagePricing::claude_cost_usd( - FALLBACK_CLAUDE_MODEL, - clamp(input), - clamp(cache_read), - clamp(cache_create_5m), - clamp(output), - ) - }) - .unwrap_or(0.0); - - // Scanner-specific: one-hour cache writes bill at 2x the input rate. - let input_rate = CostUsagePricing::claude_input_cost_per_token(model) - .or_else(|| CostUsagePricing::claude_input_cost_per_token(FALLBACK_CLAUDE_MODEL)) - .unwrap_or(0.0); - - base + (cache_create_1h as f64) * input_rate * 2.0 - } -} - /// Per-scan Claude pricing memo. /// /// Claude logs commonly repeat the same model across many files and records. Keep model diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 2a52d03b6f..1bc8e8a3b6 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -2,8 +2,59 @@ use super::*; use crate::codex_costs::codex_period_start; use crate::core::{CodexSessionLineage, CostUsagePricing}; use chrono::{FixedOffset, Local, NaiveTime, TimeZone}; +use claude_pricing::FALLBACK_CLAUDE_MODEL; use std::io::Write; +/// Scanner Claude pricing without the per-scan memo, as an oracle. +struct ClaudePricing; + +impl ClaudePricing { + fn cost_usd_with_cache_ttl( + model: &str, + input: u64, + cache_create: u64, + cache_create_1h: u64, + cache_read: u64, + output: u64, + ) -> f64 { + let cache_create_1h = cache_create_1h.min(cache_create); + let cache_create_5m = cache_create.saturating_sub(cache_create_1h); + + // Standard buckets (input, cache-read, 5-minute cache-write, output), + // including any long-context tiering, come from the canonical table. + // Unknown/retired models fall back to Sonnet pricing. + #[allow( + clippy::cast_possible_truncation, + reason = "clamped to i32::MAX before casting" + )] + let clamp = |v: u64| v.min(i32::MAX as u64) as i32; + let base = CostUsagePricing::claude_cost_usd( + model, + clamp(input), + clamp(cache_read), + clamp(cache_create_5m), + clamp(output), + ) + .or_else(|| { + CostUsagePricing::claude_cost_usd( + FALLBACK_CLAUDE_MODEL, + clamp(input), + clamp(cache_read), + clamp(cache_create_5m), + clamp(output), + ) + }) + .unwrap_or(0.0); + + // Scanner-specific: one-hour cache writes bill at 2x the input rate. + let input_rate = CostUsagePricing::claude_input_cost_per_token(model) + .or_else(|| CostUsagePricing::claude_input_cost_per_token(FALLBACK_CLAUDE_MODEL)) + .unwrap_or(0.0); + + base + (cache_create_1h as f64) * input_rate * 2.0 + } +} + #[test] fn test_unknown_model_falls_back_to_sonnet() { // Unknown/retired Claude IDs fall back to Sonnet 4.6 base pricing From bf3191694da2fcb44aa281652882960719956c85 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:17:41 +0700 Subject: [PATCH 28/41] Move test-only Claude record readers into scanner tests (RCO-03) --- rust/src/cost_scanner.rs | 36 ---------------------------------- rust/src/cost_scanner/tests.rs | 31 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index 4a8c2212bd..ec6af23d9e 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -963,36 +963,6 @@ impl CostScanner { } } -/// Stream the de-duplicated, in-window usage records from one transcript -/// file into `on_record`. Both the summary scan and the daily-history scan -/// consume this single reader, so Claude log semantics live in one place. -/// Returns the number of records consumed, so callers can tell whether the -/// file contributed anything. -#[cfg(test)] -fn for_each_claude_usage_record( - path: &Path, - cutoff: &DateTime, - seen: &mut HashSet, - cancel: Option<&AtomicBool>, - on_record: F, -) -> usize -where - F: FnMut(&ClaudeUsageRecord), -{ - let mut pricing = ClaudeScanPricingResolver::default(); - let mut incomplete = ClaudeIncompleteTracker::default(); - scan_claude_file_with_pricing( - path, - cutoff, - seen, - cancel, - &mut pricing, - &mut incomplete, - on_record, - ) - .counted -} - fn scan_claude_file_with_pricing( path: &Path, cutoff: &DateTime, @@ -1116,12 +1086,6 @@ fn finalize_claude_summary( && summary.total_cost_usd == 0.0; } -#[cfg(test)] -fn claude_usage_record_from_event(event: &ClaudeEvent) -> Option { - let mut pricing = ClaudeScanPricingResolver::default(); - claude_usage_record_from_event_with_pricing(event, &mut pricing) -} - fn claude_usage_record_from_event_with_pricing( event: &ClaudeEvent, pricing: &mut ClaudeScanPricingResolver, diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 1bc8e8a3b6..ce6c29f0ca 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -55,6 +55,37 @@ impl ClaudePricing { } } +/// Stream the de-duplicated, in-window usage records from one transcript +/// file into `on_record`, returning how many it consumed. +fn for_each_claude_usage_record( + path: &Path, + cutoff: &DateTime, + seen: &mut HashSet, + cancel: Option<&AtomicBool>, + on_record: F, +) -> usize +where + F: FnMut(&ClaudeUsageRecord), +{ + let mut pricing = ClaudeScanPricingResolver::default(); + let mut incomplete = ClaudeIncompleteTracker::default(); + scan_claude_file_with_pricing( + path, + cutoff, + seen, + cancel, + &mut pricing, + &mut incomplete, + on_record, + ) + .counted +} + +fn claude_usage_record_from_event(event: &ClaudeEvent) -> Option { + let mut pricing = ClaudeScanPricingResolver::default(); + claude_usage_record_from_event_with_pricing(event, &mut pricing) +} + #[test] fn test_unknown_model_falls_back_to_sonnet() { // Unknown/retired Claude IDs fall back to Sonnet 4.6 base pricing From 4966ab5d753e6e04bb8a9a393f0257fab94c65f4 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:19:37 +0700 Subject: [PATCH 29/41] Move the test-only pi entry filter into its tests (RCO-03) --- rust/src/pi_session_cost.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/src/pi_session_cost.rs b/rust/src/pi_session_cost.rs index c817661699..d02c041fe0 100644 --- a/rust/src/pi_session_cost.rs +++ b/rust/src/pi_session_cost.rs @@ -467,12 +467,6 @@ fn map_provider(raw: &str) -> Option { None } -#[cfg(test)] -fn parse_pi_assistant_entry(value: &Value, target: PiMappedProvider) -> Option { - let entry = parse_pi_assistant_entry_any(value)?; - (entry.provider == target).then_some(entry) -} - fn parse_pi_assistant_entry_any(value: &Value) -> Option { // Accept either flat or nested { message: {...} } pi-compatible rows. let message = value.get("message").unwrap_or(value); @@ -664,6 +658,11 @@ mod tests { use std::io::Write; use tempfile::tempdir; + fn parse_pi_assistant_entry(value: &Value, target: PiMappedProvider) -> Option { + let entry = parse_pi_assistant_entry_any(value)?; + (entry.provider == target).then_some(entry) + } + #[test] fn maps_openai_codex_and_anthropic_providers() { assert_eq!( From fe7c39df340b89bdc7cbf1534328b2815bd630ec Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:27:37 +0700 Subject: [PATCH 30/41] Share one CostUsageFileUsage test fixture (TST-06) --- rust/src/core/jsonl_scanner.rs | 3 + rust/src/core/jsonl_scanner/tests.rs | 126 ++++-------------- rust/src/core/jsonl_scanner/tests/fixtures.rs | 30 +++++ .../src/core/jsonl_scanner/tests/save_skip.rs | 23 +--- .../codex/priority_trace/tests/scanner.rs | 23 +--- rust/src/cost_scanner/tests.rs | 40 ++---- 6 files changed, 77 insertions(+), 168 deletions(-) create mode 100644 rust/src/core/jsonl_scanner/tests/fixtures.rs diff --git a/rust/src/core/jsonl_scanner.rs b/rust/src/core/jsonl_scanner.rs index 9a3b65db5d..8e55c9e619 100755 --- a/rust/src/core/jsonl_scanner.rs +++ b/rust/src/core/jsonl_scanner.rs @@ -612,6 +612,9 @@ pub use codex::priority::{ }; pub(crate) use codex::{CodexForkParseResume, CodexParseMode}; mod save_skip; +#[cfg(test)] +#[path = "jsonl_scanner/tests/fixtures.rs"] +pub(crate) mod test_fixtures; pub(crate) use codex::source_rows::{ read_source_rows, recover_rows, row_cache, row_cache_matches, row_cache_needs_recovery, row_priced_model, rows_from_records, diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index f63e6d5fe5..11f8d38bb0 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::core::test_fixtures::test_file_usage; use chrono::TimeZone; use std::io::Write; @@ -1346,46 +1347,23 @@ fn catch_up_snapshot_preserves_established_codex_cost_and_tokens() { cache.files.insert( "session.jsonl".to_string(), CostUsageFileUsage { - mtime_unix_ms: 0, - size: 100, - codex_file_identity: None, - days: HashMap::from([( - "2026-08-20".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![1_000, 250, 100])]), - )]), parsed_bytes: Some(100), - codex_scan_target_size: None, last_model: Some("gpt-5.6-sol".to_string()), - last_totals: None, codex_token_timestamps_monotonic: Some(true), - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, + ..test_file_usage( + 100, + HashMap::from([( + "2026-08-20".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![1_000, 250, 100])]), + )]), + ) }, ); cache.files.insert( "empty.jsonl".to_string(), CostUsageFileUsage { - mtime_unix_ms: 0, - size: 10, - codex_file_identity: None, - days: HashMap::new(), parsed_bytes: Some(10), - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, + ..test_file_usage(10, HashMap::new()) }, ); cache.days.insert( @@ -1416,25 +1394,16 @@ fn catch_up_snapshot_preserves_established_codex_cost_and_tokens() { #[test] fn ranged_catch_up_snapshot_excludes_historical_days_and_keeps_measurement_time() { let usage = |day: &str| CostUsageFileUsage { - mtime_unix_ms: 0, - size: 100, - codex_file_identity: None, - days: HashMap::from([( - day.to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![100, 25, 10, 4])]), - )]), parsed_bytes: Some(100), - codex_scan_target_size: None, last_model: Some("gpt-5.6-sol".to_string()), - last_totals: None, codex_token_timestamps_monotonic: Some(true), - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, + ..test_file_usage( + 100, + HashMap::from([( + day.to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![100, 25, 10, 4])]), + )]), + ) }; let mut cache = CostUsageCache { last_scan_unix_ms: 1, @@ -1604,27 +1573,13 @@ fn save_cache_persists_small_codex_artifact() { scan_until_key: Some("2026-01-31".to_string()), files: HashMap::from([( "a.jsonl".to_string(), - CostUsageFileUsage { - mtime_unix_ms: 0, - size: 100, - codex_file_identity: None, - days: HashMap::from([( + test_file_usage( + 100, + HashMap::from([( "2026-01-10".to_string(), HashMap::from([("gpt-5.6-sol".to_string(), vec![10, 0, 1])]), )]), - parsed_bytes: None, - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, - }, + ), )]), ..Default::default() }; @@ -1678,24 +1633,7 @@ fn save_cache_refuses_non_bounded_provider_oversize() { let mut cache = CostUsageCache::default(); cache.files.insert( "claude.jsonl".to_string(), - CostUsageFileUsage { - mtime_unix_ms: 0, - size: 100, - codex_file_identity: None, - days: HashMap::new(), - parsed_bytes: None, - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, - }, + test_file_usage(100, HashMap::new()), ); JsonlScanner::save_cache(ProviderId::Claude, &mut cache, Some(&cache_root)); @@ -1714,27 +1652,13 @@ fn save_cache_refusal_removes_preexisting_destination_artifact() { let mut cache = CostUsageCache::default(); cache.files.insert( "big.jsonl".to_string(), - CostUsageFileUsage { - mtime_unix_ms: 0, - size: 100, - codex_file_identity: None, - days: HashMap::from([( + test_file_usage( + 100, + HashMap::from([( "2026-01-10".to_string(), HashMap::from([("gpt-5.6-sol".to_string(), vec![10, 0, 1])]), )]), - parsed_bytes: None, - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, - }, + ), ); // Precreate a "stale" destination artifact so the refusal must remove diff --git a/rust/src/core/jsonl_scanner/tests/fixtures.rs b/rust/src/core/jsonl_scanner/tests/fixtures.rs new file mode 100644 index 0000000000..56fecd2cf3 --- /dev/null +++ b/rust/src/core/jsonl_scanner/tests/fixtures.rs @@ -0,0 +1,30 @@ +//! Cache-entry fixtures shared by the JSONL and cost scanner tests. + +use super::{CodexSessionLineage, CostUsageFileUsage}; +use std::collections::HashMap; + +/// A root-session cache entry with no parse state; tests override the +/// fields they exercise with struct update syntax. +pub(crate) fn test_file_usage( + size: i64, + days: HashMap>>, +) -> CostUsageFileUsage { + CostUsageFileUsage { + mtime_unix_ms: 0, + size, + codex_file_identity: None, + days, + parsed_bytes: None, + codex_scan_target_size: None, + last_model: None, + last_totals: None, + codex_token_timestamps_monotonic: None, + codex_last_token_timestamp: None, + codex_session_id: None, + codex_forked_from_id: None, + codex_fork_accounting_state: None, + codex_lineage: CodexSessionLineage::Root, + codex_fork_timestamp: None, + codex_unresolved_fork_parent: false, + } +} diff --git a/rust/src/core/jsonl_scanner/tests/save_skip.rs b/rust/src/core/jsonl_scanner/tests/save_skip.rs index 01fc5ab3bf..408b73b26f 100644 --- a/rust/src/core/jsonl_scanner/tests/save_skip.rs +++ b/rust/src/core/jsonl_scanner/tests/save_skip.rs @@ -1,27 +1,14 @@ use super::*; +use crate::core::test_fixtures::test_file_usage; fn file_usage(day: &str, model: &str, counts: Vec) -> CostUsageFileUsage { - CostUsageFileUsage { - mtime_unix_ms: 0, - size: 100, - codex_file_identity: None, - days: HashMap::from([( + test_file_usage( + 100, + HashMap::from([( day.to_string(), HashMap::from([(model.to_string(), counts)]), )]), - parsed_bytes: None, - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, - } + ) } fn seeded_cache() -> CostUsageCache { diff --git a/rust/src/cost_scanner/codex/priority_trace/tests/scanner.rs b/rust/src/cost_scanner/codex/priority_trace/tests/scanner.rs index 802637975c..5b1fcfb7e4 100644 --- a/rust/src/cost_scanner/codex/priority_trace/tests/scanner.rs +++ b/rust/src/cost_scanner/codex/priority_trace/tests/scanner.rs @@ -7,10 +7,11 @@ //! just before the current time and "time passing" ages the persisted cache. use super::*; +use crate::core::test_fixtures::test_file_usage; use crate::core::{ - CodexPriorityTurnMetadata, CodexPriorityTurnsCursor, CodexSessionLineage, - CodexSourcePricingEvidence, CodexSourceRowCache, CodexSourceUsageRow, CostUsageCache, - CostUsageCacheBudget, CostUsageFileUsage, JsonlScanner, ProviderId, + CodexPriorityTurnMetadata, CodexPriorityTurnsCursor, CodexSourcePricingEvidence, + CodexSourceRowCache, CodexSourceUsageRow, CostUsageCache, CostUsageCacheBudget, + CostUsageFileUsage, JsonlScanner, ProviderId, }; use crate::cost_scanner::codex::ambient_codex_trace_database_path; use crate::cost_scanner::codex::scan::{codex_priority_metadata_appeared, save_codex_cache}; @@ -679,22 +680,8 @@ fn vanished_database_keeps_its_evidence_and_metadata_key() { fn file_usage(days: DayModels) -> CostUsageFileUsage { CostUsageFileUsage { - mtime_unix_ms: 0, - size: 10, - codex_file_identity: None, - days, parsed_bytes: Some(10), - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, + ..test_file_usage(10, days) } } diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index ce6c29f0ca..959994ed16 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -1,5 +1,6 @@ use super::*; use crate::codex_costs::codex_period_start; +use crate::core::test_fixtures::test_file_usage; use crate::core::{CodexSessionLineage, CostUsagePricing}; use chrono::{FixedOffset, Local, NaiveTime, TimeZone}; use claude_pricing::FALLBACK_CLAUDE_MODEL; @@ -1241,25 +1242,14 @@ fn write_codex_session_fixture_with_inputs( fn cached_usage_with_packed(day: &str, model: &str, packed: Vec) -> CostUsageFileUsage { CostUsageFileUsage { - mtime_unix_ms: 0, - size: 1, - codex_file_identity: None, - days: HashMap::from([( - day.to_string(), - HashMap::from([(model.to_string(), packed)]), - )]), parsed_bytes: Some(1), - codex_scan_target_size: None, - last_model: None, - last_totals: None, - codex_token_timestamps_monotonic: None, - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, + ..test_file_usage( + 1, + HashMap::from([( + day.to_string(), + HashMap::from([(model.to_string(), packed)]), + )]), + ) } } @@ -2175,22 +2165,10 @@ fn cancelled_fresh_cache_hit_is_not_authoritative() { files: HashMap::from([( "cached.jsonl".to_string(), CostUsageFileUsage { - mtime_unix_ms: 0, - size: 100, - codex_file_identity: None, - days: usage.clone(), parsed_bytes: Some(100), - codex_scan_target_size: None, last_model: Some("gpt-5.6-sol".to_string()), - last_totals: None, codex_token_timestamps_monotonic: Some(true), - codex_last_token_timestamp: None, - codex_session_id: None, - codex_forked_from_id: None, - codex_fork_accounting_state: None, - codex_lineage: CodexSessionLineage::Root, - codex_fork_timestamp: None, - codex_unresolved_fork_parent: false, + ..test_file_usage(100, usage.clone()) }, )]), days: usage, From 2a40cdbf844260579d42b3e7488562236dbe4df7 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:32:22 +0700 Subject: [PATCH 31/41] Table-drive the single-assert pricing tests (TST-05) --- rust/src/core/cost_pricing_tests.rs | 220 ++++++++-------------------- rust/src/cost_scanner/tests.rs | 118 +++++++-------- 2 files changed, 119 insertions(+), 219 deletions(-) diff --git a/rust/src/core/cost_pricing_tests.rs b/rust/src/core/cost_pricing_tests.rs index 028e90c27a..46d48d28ec 100644 --- a/rust/src/core/cost_pricing_tests.rs +++ b/rust/src/core/cost_pricing_tests.rs @@ -77,48 +77,35 @@ impl CostUsagePricing { } #[test] -fn test_normalize_codex_model() { - assert_eq!(CostUsagePricing::normalize_codex_model("gpt-5"), "gpt-5"); - assert_eq!( - CostUsagePricing::normalize_codex_model("openai/gpt-5"), - "gpt-5" - ); - assert_eq!( - CostUsagePricing::normalize_codex_model("gpt-5-codex"), - "gpt-5" - ); - assert_eq!( - CostUsagePricing::normalize_codex_model(""), - CostUsagePricing::CODEX_UNATTRIBUTED_MODEL - ); - assert_eq!( - CostUsagePricing::normalize_codex_model("unknown"), - CostUsagePricing::CODEX_UNATTRIBUTED_MODEL - ); - assert_eq!( - CostUsagePricing::normalize_codex_model("gpt-reserve"), - "gpt-5.6-luna" - ); - assert_eq!( - CostUsagePricing::normalize_codex_model(" GPT-RESERVE "), - "gpt-5.6-luna" - ); - assert_eq!( - CostUsagePricing::normalize_codex_model("openai/gpt-reserve"), - "gpt-5.6-luna" - ); - assert_eq!( - CostUsagePricing::normalize_codex_model("OPENAI/GPT-RESERVE"), - "gpt-5.6-luna" - ); - assert_eq!( - CostUsagePricing::normalize_codex_model("gpt-reserve-preview"), - "gpt-reserve-preview" - ); - assert_eq!( - CostUsagePricing::normalize_codex_model("my-gpt-reserve"), - "my-gpt-reserve" - ); +fn normalizes_codex_model_ids() { + let unattributed = CostUsagePricing::CODEX_UNATTRIBUTED_MODEL; + for (model, expected) in [ + ("gpt-5", "gpt-5"), + ("openai/gpt-5", "gpt-5"), + ("gpt-5-codex", "gpt-5"), + ("", unattributed), + ("unknown", unattributed), + ("gpt-reserve", "gpt-5.6-luna"), + (" GPT-RESERVE ", "gpt-5.6-luna"), + ("openai/gpt-reserve", "gpt-5.6-luna"), + ("OPENAI/GPT-RESERVE", "gpt-5.6-luna"), + ("gpt-reserve-preview", "gpt-reserve-preview"), + ("my-gpt-reserve", "my-gpt-reserve"), + ("gpt-5.4-mini-codex", "gpt-5.4-mini"), + ("openai/gpt-5.5-2026-04-23", "gpt-5.5"), + ("gpt-5.5-pro-2026-04-23", "gpt-5.5-pro"), + ("gpt-5.6", "gpt-5.6-sol"), + ("openai/gpt-5.6", "gpt-5.6-sol"), + ("gpt-5.6-codex", "gpt-5.6-sol"), + ("gpt-5.6-2099-01-01", "gpt-5.6-sol"), + ("openai/gpt-5.6-codex-2099-01-01", "gpt-5.6-sol"), + ] { + assert_eq!( + CostUsagePricing::normalize_codex_model(model), + expected, + "{model:?}" + ); + } } #[test] @@ -144,28 +131,39 @@ fn test_normalize_claude_model() { } #[test] -fn test_codex_cost() { - let cost = CostUsagePricing::codex_cost_usd("gpt-5", 1000, 0, 500).unwrap(); - assert!((cost - 0.00625).abs() < 1e-10); -} - -#[test] -fn test_claude_cost() { - assert!( - CostUsagePricing::claude_cost_usd("claude-haiku-4-5-20251001", 1000, 0, 0, 500).is_some() - ); -} - -#[test] -fn test_opus_4_8_cost() { - let cost = CostUsagePricing::claude_cost_usd("claude-opus-4-8", 1_000, 0, 0, 500).unwrap(); - assert!((cost - 0.0175).abs() < 1e-10); +fn prices_codex_models() { + // (model, input, cached input, output, expected USD). + for (model, input, cached, output, expected) in [ + ("gpt-5", 1000, 0, 500, 0.00625), + ("gpt-5.4-mini", 1000, 0, 500, 0.003), + ("gpt-5.4-nano", 1000, 0, 500, 0.000825), + ("gpt-5-pro", 1000, 0, 500, 0.075), + ("gpt-5.5", 1000, 500, 500, 0.01775), + ("gpt-5.6-sol", 1_000, 400, 1_000, 0.02256), + ("gpt-5.6-terra", 1_000, 400, 1_000, 0.01328), + ("gpt-5.6-luna", 1_000, 400, 1_000, 0.001328), + ] { + let cost = CostUsagePricing::codex_cost_usd(model, input, cached, output).unwrap(); + assert!((cost - expected).abs() < 1e-10, "{model}"); + } } #[test] -fn test_fable_5_cost() { - let cost = CostUsagePricing::claude_cost_usd("claude-fable-5", 1_000, 0, 0, 500).unwrap(); - assert!((cost - 0.035).abs() < 1e-10); +fn prices_claude_models() { + for (model, expected) in [("claude-opus-4-8", 0.0175), ("claude-fable-5", 0.035)] { + let cost = CostUsagePricing::claude_cost_usd(model, 1_000, 0, 0, 500).unwrap(); + assert!((cost - expected).abs() < 1e-10, "{model}"); + } + for model in [ + "claude-haiku-4-5-20251001", + "claude-opus-4-7", + "claude-sonnet-4-6", + ] { + assert!( + CostUsagePricing::claude_cost_usd(model, 1000, 0, 0, 500).is_some(), + "{model}" + ); + } } #[test] @@ -185,85 +183,14 @@ fn test_claude_input_cost_per_token() { } #[test] -fn test_format_model_name() { - assert_eq!( - CostUsagePricing::format_model_name("claude-3.5-sonnet"), - "Sonnet 3.5" - ); - assert_eq!( - CostUsagePricing::format_model_name("claude-opus-4"), - "Opus 4" - ); - assert_eq!(CostUsagePricing::format_model_name("gpt-5"), "GPT-5"); -} - -#[test] -fn test_gpt54_mini_cost() { - let cost = CostUsagePricing::codex_cost_usd("gpt-5.4-mini", 1000, 0, 500).unwrap(); - assert!((cost - 0.003).abs() < 1e-10); -} - -#[test] -fn test_gpt54_nano_cost() { - let cost = CostUsagePricing::codex_cost_usd("gpt-5.4-nano", 1000, 0, 500).unwrap(); - assert!((cost - 0.000825).abs() < 1e-10); -} - -#[test] -fn test_normalize_gpt54_codex() { - assert_eq!( - CostUsagePricing::normalize_codex_model("gpt-5.4-mini-codex"), - "gpt-5.4-mini" - ); -} - -#[test] -fn test_gpt55_pricing() { - assert_eq!( - CostUsagePricing::normalize_codex_model("openai/gpt-5.5-2026-04-23"), - "gpt-5.5" - ); - assert_eq!( - CostUsagePricing::normalize_codex_model("gpt-5.5-pro-2026-04-23"), - "gpt-5.5-pro" - ); - let cost = CostUsagePricing::codex_cost_usd("gpt-5.5", 1000, 500, 500).unwrap(); - assert!((cost - 0.01775).abs() < 1e-10); -} - -#[test] -fn test_format_gpt54_mini() { - assert_eq!( - CostUsagePricing::format_model_name("gpt-5.4-mini"), - "GPT-5.4 Mini" - ); -} - -#[test] -fn test_opus_4_7_cost() { - assert!(CostUsagePricing::claude_cost_usd("claude-opus-4-7", 1000, 0, 0, 500).is_some()); -} - -#[test] -fn test_sonnet_4_6_cost() { - assert!(CostUsagePricing::claude_cost_usd("claude-sonnet-4-6", 1000, 0, 0, 500).is_some()); -} - -#[test] -fn test_gpt5_pro_cost() { - let cost = CostUsagePricing::codex_cost_usd("gpt-5-pro", 1000, 0, 500).unwrap(); - assert!((cost - 0.075).abs() < 1e-10); -} - -#[test] -fn test_gpt56_standard_pricing() { +fn formats_model_names() { for (model, expected) in [ - ("gpt-5.6-sol", 0.02256), - ("gpt-5.6-terra", 0.01328), - ("gpt-5.6-luna", 0.001328), + ("claude-3.5-sonnet", "Sonnet 3.5"), + ("claude-opus-4", "Opus 4"), + ("gpt-5", "GPT-5"), + ("gpt-5.4-mini", "GPT-5.4 Mini"), ] { - let cost = CostUsagePricing::codex_cost_usd(model, 1_000, 400, 1_000); - assert!((cost.unwrap() - expected).abs() < 1e-10, "{model}"); + assert_eq!(CostUsagePricing::format_model_name(model), expected); } } @@ -307,23 +234,6 @@ fn test_gpt56_context_threshold_is_exclusive() { } } -#[test] -fn test_normalize_gpt56_aliases() { - for model in [ - "gpt-5.6", - "openai/gpt-5.6", - "gpt-5.6-codex", - "gpt-5.6-2099-01-01", - "openai/gpt-5.6-codex-2099-01-01", - ] { - assert_eq!( - CostUsagePricing::normalize_codex_model(model), - "gpt-5.6-sol", - "{model}" - ); - } -} - #[test] fn test_codex_display_label() { assert_eq!( diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 959994ed16..24150eb5df 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -201,81 +201,71 @@ fn claude_scan_pricing_resolver_bounds_normalization_memo() { } #[test] -fn test_claude_fable_5_pricing() { - let cost = ClaudePricing::cost_usd_with_cache_ttl("claude-fable-5", 100, 10, 0, 20, 5); - let expected = (100.0 / 1_000_000.0) * 10.00 - + (10.0 / 1_000_000.0) * 12.50 - + (20.0 / 1_000_000.0) * 1.00 - + (5.0 / 1_000_000.0) * 50.00; - assert!((cost - expected).abs() < f64::EPSILON); -} - -#[test] -fn test_claude_one_hour_cache_write_pricing() { - let cost = ClaudePricing::cost_usd_with_cache_ttl("claude-fable-5", 100, 30, 20, 20, 5); - let expected = (100.0 / 1_000_000.0) * 10.00 - + (10.0 / 1_000_000.0) * 12.50 - + (20.0 / 1_000_000.0) * 20.00 - + (20.0 / 1_000_000.0) * 1.00 - + (5.0 / 1_000_000.0) * 50.00; - assert!((cost - expected).abs() < f64::EPSILON); -} - -#[test] -fn test_claude_sonnet_46_honors_200k_tier() { - // Delegating to the canonical table means the scanner now honors the - // 200k long-context tier: 200k @ $3/M + 40k @ $6/M = 0.60 + 0.24 = 0.84 - // (the scanner's old inline table applied a flat $3/M = 0.72). - let cost = ClaudePricing::cost_usd_with_cache_ttl("claude-sonnet-4-6", 240_000, 0, 0, 0, 0); - assert!((cost - 0.84).abs() < 0.001); -} - -#[test] -fn test_current_gen_opus_uses_5_25_pricing() { - // Opus 4.5/4.6/4.7/4.8 bill at $5/1M input + $25/1M output = $30 total. - // Delegation regression guard: opus-4-8 in particular must resolve - // through the canonical table (it was missing there before this fix). - for model in [ - "claude-opus-4-5", - "claude-opus-4-6", - "claude-opus-4-7", - "claude-opus-4-8", +fn claude_fable_5_prices_five_minute_and_one_hour_cache_writes() { + // (input, cache write, 1h share of it, cache read, output, expected USD). + for (input, cache_create, cache_create_1h, cache_read, output, expected) in [ + ( + 100, + 10, + 0, + 20, + 5, + (100.0 / 1_000_000.0) * 10.00 + + (10.0 / 1_000_000.0) * 12.50 + + (20.0 / 1_000_000.0) * 1.00 + + (5.0 / 1_000_000.0) * 50.00, + ), + ( + 100, + 30, + 20, + 20, + 5, + (100.0 / 1_000_000.0) * 10.00 + + (10.0 / 1_000_000.0) * 12.50 + + (20.0 / 1_000_000.0) * 20.00 + + (20.0 / 1_000_000.0) * 1.00 + + (5.0 / 1_000_000.0) * 50.00, + ), ] { - let cost = ClaudePricing::cost_usd_with_cache_ttl(model, 1_000_000, 0, 0, 0, 1_000_000); - assert!( - (cost - 30.00).abs() < 0.001, - "{model} should bill $30 ($5 in + $25 out), got {cost}" + let cost = ClaudePricing::cost_usd_with_cache_ttl( + "claude-fable-5", + input, + cache_create, + cache_create_1h, + cache_read, + output, ); + assert!((cost - expected).abs() < f64::EPSILON, "{cache_create_1h}"); } } #[test] -fn test_legacy_opus_keeps_legacy_pricing() { - // Legacy Opus 4.0 / 4.1 remain at $15/1M input + $75/1M output = $90 in - // the canonical table. (Retired IDs absent from the table — e.g. Opus 3 - // `claude-3-opus-...` — fall back to Sonnet instead; they are outside - // any realistic 30-day scan window.) - for model in ["claude-opus-4-20250514", "claude-opus-4-1"] { - let cost = ClaudePricing::cost_usd_with_cache_ttl(model, 1_000_000, 0, 0, 0, 1_000_000); +fn claude_scan_pricing_follows_the_canonical_table() { + // Sonnet 4.6 honors the 200k tier: 200k @ $3/M + 40k @ $6/M = 0.84 (the + // scanner's old inline table applied a flat $3/M = 0.72). Opus 4.5-4.8 + // bill $5 in + $25 out; opus-4-8 must resolve through the canonical table. + // Legacy Opus 4.0/4.1 stay at $15 + $75 (retired IDs absent from the + // table, e.g. `claude-3-opus-...`, fall back to Sonnet). Haiku 4.5 bills + // $1 + $5, not the Haiku 3 rate. + for (model, input, output, expected) in [ + ("claude-sonnet-4-6", 240_000, 0, 0.84), + ("claude-opus-4-5", 1_000_000, 1_000_000, 30.00), + ("claude-opus-4-6", 1_000_000, 1_000_000, 30.00), + ("claude-opus-4-7", 1_000_000, 1_000_000, 30.00), + ("claude-opus-4-8", 1_000_000, 1_000_000, 30.00), + ("claude-opus-4-20250514", 1_000_000, 1_000_000, 90.00), + ("claude-opus-4-1", 1_000_000, 1_000_000, 90.00), + ("claude-haiku-4-5", 1_000_000, 1_000_000, 6.00), + ] { + let cost = ClaudePricing::cost_usd_with_cache_ttl(model, input, 0, 0, 0, output); assert!( - (cost - 90.00).abs() < 0.001, - "{model} should bill $90 ($15 in + $75 out), got {cost}" + (cost - expected).abs() < 0.001, + "{model} should bill ${expected}, got {cost}" ); } } -#[test] -fn test_haiku_45_uses_current_pricing() { - // Haiku 4.5 bills at $1/1M input + $5/1M output = $6 via the canonical - // table (previously the scanner under-priced it at the Haiku 3 rate). - let cost = - ClaudePricing::cost_usd_with_cache_ttl("claude-haiku-4-5", 1_000_000, 0, 0, 0, 1_000_000); - assert!( - (cost - 6.00).abs() < 0.001, - "haiku-4-5 should bill $6 ($1 in + $5 out), got {cost}" - ); -} - #[test] fn parses_current_codex_payload_token_count_events() { let path = std::env::temp_dir().join(format!( From ebb53a17003521e2a951d2b4ca95f34f92d59015 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:36:22 +0700 Subject: [PATCH 32/41] Pin the live host-cost renderer's dash, partial and local-title text --- rust/src/codex_costs/host_costs.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/rust/src/codex_costs/host_costs.rs b/rust/src/codex_costs/host_costs.rs index ace018c0dc..48279d1bcd 100644 --- a/rust/src/codex_costs/host_costs.rs +++ b/rust/src/codex_costs/host_costs.rs @@ -228,4 +228,28 @@ mod tests { assert!(text.contains("Snapshot updated: 2000-01-01T00:00:00Z")); assert!(text.contains("Day boundaries: Asia/Tokyo")); } + + #[test] + fn local_text_shows_unknown_values_as_dashes_and_flags_partial_history() { + let partial = CodexCostSummary::from_summaries_at( + &CostSummary::default(), + &CostSummary::default(), + 30, + chrono::DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + "UTC", + ); + + let text = + render_codex_host_report(&CodexHostCostReport::success("local", "local", partial)); + + assert_eq!( + text, + "This machine — Codex API-equivalent estimate (not billed)\n\ + Today: — · — tokens\n\ + Last 30 days: — · — tokens\n\ + Snapshot updated: 2023-11-14T22:13:20Z\n\ + Day boundaries: UTC\n\ + Partial history; scan is incomplete." + ); + } } From 35d4b291ddebd433c8c15e85cd60773688a7ff51 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:42:42 +0700 Subject: [PATCH 33/41] Share cost scanner test setup through a support module (TST-01) --- rust/src/cost_scanner/tests.rs | 430 +++++-------------- rust/src/cost_scanner/tests/archived.rs | 7 - rust/src/cost_scanner/tests/copied_prefix.rs | 56 +-- rust/src/cost_scanner/tests/direct_fork.rs | 23 +- rust/src/cost_scanner/tests/fork_resume.rs | 5 +- rust/src/cost_scanner/tests/lineage_cache.rs | 84 +--- rust/src/cost_scanner/tests/paginated.rs | 71 +-- rust/src/cost_scanner/tests/period.rs | 5 +- rust/src/cost_scanner/tests/support.rs | 38 ++ 9 files changed, 190 insertions(+), 529 deletions(-) create mode 100644 rust/src/cost_scanner/tests/support.rs diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 24150eb5df..4b134a7265 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -5,6 +5,7 @@ use crate::core::{CodexSessionLineage, CostUsagePricing}; use chrono::{FixedOffset, Local, NaiveTime, TimeZone}; use claude_pricing::FALLBACK_CLAUDE_MODEL; use std::io::Write; +use support::{app_scanner, cached_file, codex_scan_dirs, partition_dir}; /// Scanner Claude pricing without the per-scan memo, as an oracle. struct ClaudePricing; @@ -312,15 +313,10 @@ fn parses_current_codex_payload_token_count_events() { #[test] fn scans_gpt6_astra_usage_with_cached_and_reasoning_tokens() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let today = Local::now().date_naive(); let day = today.format("%Y-%m-%d").to_string(); - let day_dir = sessions - .join(today.format("%Y").to_string()) - .join(today.format("%m").to_string()) - .join(today.format("%d").to_string()); + let day_dir = partition_dir(&sessions, today); std::fs::create_dir_all(&day_dir).unwrap(); let line = serde_json::json!({ "timestamp": Local::now().to_rfc3339(), @@ -340,10 +336,7 @@ fn scans_gpt6_astra_usage_with_cached_and_reasoning_tokens() { }); std::fs::write(day_dir.join("astra.jsonl"), format!("{line}\n")).unwrap(); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(summary.input_tokens, 1000); @@ -1187,10 +1180,7 @@ fn recent_codex_fixture_time_stays_on_the_local_day() { fn write_codex_session_fixture(sessions_root: &Path, name: &str, input_tokens: u64) -> PathBuf { let event_time = recent_codex_fixture_time(); let today = event_time.with_timezone(&Local).date_naive(); - let day_dir = sessions_root - .join(today.format("%Y").to_string()) - .join(today.format("%m").to_string()) - .join(today.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, today); std::fs::create_dir_all(&day_dir).unwrap(); let path = day_dir.join(name); let ts = event_time.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); @@ -1209,10 +1199,7 @@ fn write_codex_session_fixture_with_inputs( ) -> PathBuf { let base = recent_codex_fixture_time(); let today = base.with_timezone(&Local).date_naive(); - let day_dir = sessions_root - .join(today.format("%Y").to_string()) - .join(today.format("%m").to_string()) - .join(today.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, today); std::fs::create_dir_all(&day_dir).unwrap(); let mut body = String::new(); for (index, input) in input_tokens.iter().enumerate() { @@ -1381,19 +1368,14 @@ fn retained_report_sums_multiple_days_above_i32_max() { #[test] fn reasoning_survives_scan_rebuild_and_cache_reload() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); // The event timestamp (now − 1h) decides the parsed day key, so derive the // fixture day from that same instant: at local 00:00–01:00 now − 1h falls // on the previous local day and the row would land there, not on today. let event_time = Utc::now() - Duration::hours(1); let today = event_time.with_timezone(&Local).date_naive(); let day = today.format("%Y-%m-%d").to_string(); - let day_dir = sessions - .join(today.format("%Y").to_string()) - .join(today.format("%m").to_string()) - .join(today.format("%d").to_string()); + let day_dir = partition_dir(&sessions, today); std::fs::create_dir_all(&day_dir).unwrap(); let timestamp = event_time.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); let reasoning_line = serde_json::json!({ @@ -1418,10 +1400,7 @@ fn reasoning_survives_scan_rebuild_and_cache_reload() { ) .unwrap(); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(summary.output_tokens, 20); assert_eq!(summary.reasoning_tokens, Some(7)); @@ -1441,10 +1420,7 @@ fn reasoning_survives_scan_rebuild_and_cache_reload() { let legacy_root = tempfile::tempdir().unwrap(); let legacy_sessions = legacy_root.path().join("sessions"); let legacy_cache_root = legacy_root.path().join("cache"); - let legacy_day_dir = legacy_sessions - .join(today.format("%Y").to_string()) - .join(today.format("%m").to_string()) - .join(today.format("%d").to_string()); + let legacy_day_dir = partition_dir(&legacy_sessions, today); std::fs::create_dir_all(&legacy_day_dir).unwrap(); let legacy_line = serde_json::json!({ "timestamp": timestamp, @@ -1467,10 +1443,7 @@ fn reasoning_survives_scan_rebuild_and_cache_reload() { ) .unwrap(); - let legacy_scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&legacy_cache_root) - .with_sessions_dirs(vec![legacy_sessions]); + let legacy_scanner = app_scanner(7, &legacy_cache_root, &legacy_sessions); let (legacy_summary, _, legacy_cache) = legacy_scanner.scan_codex_detailed_with_cache(None); assert_eq!(legacy_summary.output_tokens, 20); assert_eq!(legacy_summary.reasoning_tokens, None); @@ -1491,10 +1464,7 @@ fn write_codex_fork_session_fixture( totals: &[i64], ) -> PathBuf { let today = Local::now().date_naive(); - let day_dir = sessions_root - .join(today.format("%Y").to_string()) - .join(today.format("%m").to_string()) - .join(today.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, today); std::fs::create_dir_all(&day_dir).unwrap(); let mut body = format!( @@ -1546,9 +1516,7 @@ fn cached_input_total(usage: &CostUsageFileUsage) -> i64 { #[test] fn ordinary_non_fork_session_keeps_cumulative_accounting() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture_with_inputs(&sessions, "ordinary.jsonl", &[100, 140]); let mut options = CostScanOptions::app_driven(); @@ -1560,10 +1528,7 @@ fn ordinary_non_fork_session_keeps_cumulative_accounting() { let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(summary.input_tokens, 140); - let usage = cache - .files - .get(&path.to_string_lossy().to_string()) - .unwrap(); + let usage = cached_file(&cache, &path); assert_eq!(cached_input_total(usage), 140); assert_eq!(usage.codex_forked_from_id, None); assert!(!usage.codex_unresolved_fork_parent); @@ -1571,9 +1536,7 @@ fn ordinary_non_fork_session_keeps_cumulative_accounting() { #[test] fn fork_child_counts_only_growth_above_parent_baseline() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let fork = base + Duration::seconds(2); let parent = write_codex_fork_session_fixture( @@ -1619,10 +1582,7 @@ fn fork_child_counts_only_growth_above_parent_baseline() { assert_eq!(summary.input_tokens, 1_000_140); assert_eq!(summary.sessions_count, 2); - let child_usage = cache - .files - .get(&child.to_string_lossy().to_string()) - .unwrap(); + let child_usage = cached_file(&cache, &child); assert_eq!(cached_input_total(child_usage), 140); assert!(!child_usage.codex_unresolved_fork_parent); assert!(cache.codex_pending_paths.is_empty()); @@ -1635,9 +1595,7 @@ fn fork_child_counts_only_growth_above_parent_baseline() { #[test] fn replaced_fork_child_does_not_reuse_cached_identity() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let fork = base + Duration::seconds(2); let _parent = write_codex_fork_session_fixture( @@ -1703,9 +1661,7 @@ fn replaced_fork_child_does_not_reuse_cached_identity() { #[test] fn missing_fork_parent_fails_closed_and_persists_pending() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_codex_fork_session_fixture( &sessions, @@ -1717,10 +1673,7 @@ fn missing_fork_parent_fails_closed_and_persists_pending() { &[1_000_000, 1_000_140], ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); let child_key = child.to_string_lossy().to_string(); @@ -1736,9 +1689,7 @@ fn missing_fork_parent_fails_closed_and_persists_pending() { #[test] fn fork_child_resolves_after_parent_is_cached() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let fork = base + Duration::seconds(2); let child = write_codex_fork_session_fixture( @@ -1786,10 +1737,7 @@ fn fork_child_resolves_after_parent_is_cached() { } } let (summary, cache) = resolved.expect("later bounded pass resolves the child"); - let child_usage = cache - .files - .get(&child.to_string_lossy().to_string()) - .unwrap(); + let child_usage = cached_file(&cache, &child); assert_eq!(summary.input_tokens, 1_000_140); assert_eq!(cached_input_total(child_usage), 140); assert!(!child_usage.codex_unresolved_fork_parent); @@ -1798,9 +1746,7 @@ fn fork_child_resolves_after_parent_is_cached() { #[test] fn parent_last_token_after_fork_keeps_child_unresolved() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let parent = write_codex_fork_session_fixture( &sessions, @@ -1821,16 +1767,10 @@ fn parent_last_token_after_fork_keeps_child_unresolved() { &[1_000_000, 1_000_140], ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - let child_usage = cache - .files - .get(&child.to_string_lossy().to_string()) - .unwrap(); + let child_usage = cached_file(&cache, &child); assert_eq!(summary.input_tokens, 1_000_000); assert_eq!(cached_input_total(child_usage), 0); assert!(child_usage.codex_unresolved_fork_parent); @@ -1848,9 +1788,7 @@ fn parent_last_token_after_fork_keeps_child_unresolved() { #[test] fn deleted_unresolved_child_is_pruned_without_resurrection() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_codex_fork_session_fixture( &sessions, @@ -1861,10 +1799,7 @@ fn deleted_unresolved_child_is_pruned_without_resurrection() { base + Duration::seconds(2), &[1_000_000, 1_000_140], ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); assert!( first_cache @@ -1886,9 +1821,7 @@ fn deleted_unresolved_child_is_pruned_without_resurrection() { #[test] fn fork_baseline_reset_fails_closed_instead_of_billing_fresh_usage() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let _parent = write_codex_fork_session_fixture( &sessions, @@ -1909,15 +1842,9 @@ fn fork_baseline_reset_fails_closed_instead_of_billing_fresh_usage() { &[1_000_000, 999_900, 1_000_140], ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - let child_usage = cache - .files - .get(&child.to_string_lossy().to_string()) - .unwrap(); + let child_usage = cached_file(&cache, &child); assert_eq!(summary.input_tokens, 1_000_000); assert_eq!(cached_input_total(child_usage), 0); @@ -1927,16 +1854,11 @@ fn fork_baseline_reset_fails_closed_instead_of_billing_fresh_usage() { #[test] fn cost_scan_second_pass_skips_unchanged_files_via_cache() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); write_codex_session_fixture(&sessions, "a.jsonl", 100); write_codex_session_fixture(&sessions, "b.jsonl", 200); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary1, stats1) = scanner.scan_codex_detailed(None); assert_eq!(stats1.files_parsed, 2, "first pass parses both files"); @@ -1980,10 +1902,7 @@ fn cost_scan_second_pass_skips_unchanged_files_via_cache() { assert_eq!(summary3.input_tokens, summary1.input_tokens); // app_driven after debounce still re-reads (skip via mtime, not full re-parse). - let forced = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let forced = app_scanner(7, &cache_root, &sessions); let (_, stats4) = forced.scan_codex_detailed(None); assert!(!stats4.used_cache_debounce); assert_eq!(stats4.files_skipped, 2); @@ -1993,15 +1912,10 @@ fn cost_scan_second_pass_skips_unchanged_files_via_cache() { #[test] fn codex_lazy_history_receipt_reads_only_changed_file_and_matches_fresh_parse() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (root, sessions, cache_root) = codex_scan_dirs(); let first_path = write_codex_session_fixture_with_inputs(&sessions, "first.jsonl", &[100]); let second_path = write_codex_session_fixture_with_inputs(&sessions, "second.jsonl", &[200]); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (initial, _, _) = scanner.scan_codex_detailed_with_cache(None); let (unchanged, unchanged_stats, _) = scanner.scan_codex_detailed_with_cache(None); @@ -2036,10 +1950,7 @@ fn codex_lazy_history_receipt_reads_only_changed_file_and_matches_fresh_parse() assert_eq!(incremental_stats.codex_read_receipt.history_reads, 1); assert_eq!(incremental.input_tokens, 350); - let fresh = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(root.path().join("fresh-cache")) - .with_sessions_dirs(vec![sessions]); + let fresh = app_scanner(7, root.path().join("fresh-cache"), &sessions); let (full, full_stats) = fresh.scan_codex_detailed(None); assert_eq!(full_stats.codex_history_read_paths.len(), 2); assert_eq!(incremental.input_tokens, full.input_tokens); @@ -2052,14 +1963,9 @@ fn codex_lazy_history_receipt_reads_only_changed_file_and_matches_fresh_parse() #[test] fn codex_source_recovery_keeps_appended_duplicate_unpriced_after_cache_reload() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture_with_inputs(&sessions, "recovery.jsonl", &[100]); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, mut first_cache) = scanner.scan_codex_detailed_with_cache(None); let path_key = path.to_string_lossy().to_string(); @@ -2108,14 +2014,9 @@ fn codex_source_recovery_keeps_appended_duplicate_unpriced_after_cache_reload() #[test] fn codex_file_identity_invalidates_same_path_cache_without_eager_history_read() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "replacement.jsonl", 100); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, _) = scanner.scan_codex_detailed_with_cache(None); let old_mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); let rotated = path.with_extension("old"); @@ -2186,18 +2087,13 @@ fn cancelled_fresh_cache_hit_is_not_authoritative() { #[test] fn cost_scan_cancel_stops_between_files() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); write_codex_session_fixture(&sessions, "a.jsonl", 100); write_codex_session_fixture(&sessions, "b.jsonl", 200); write_codex_session_fixture(&sessions, "c.jsonl", 300); let cancel = AtomicBool::new(true); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, cache_root, &sessions); let (summary, stats) = scanner.scan_codex_detailed(Some(&cancel)); assert_eq!(stats.files_seen, 0, "cancel before first file stops walk"); assert_eq!(summary.sessions_count, 0); @@ -2205,15 +2101,10 @@ fn cost_scan_cancel_stops_between_files() { #[test] fn cost_scan_reconciles_deleted_file_to_known_zero() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (first, _) = scanner.scan_codex_detailed(None); assert_eq!(first.sessions_count, 1); assert!(first.total_cost_usd > 0.0); @@ -2232,16 +2123,11 @@ fn cost_scan_reconciles_deleted_file_to_known_zero() { #[test] fn cost_scan_reconciliation_preserves_sibling_totals_once() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let deleted = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); let sibling = write_codex_session_fixture(&sessions, "sibling.jsonl", 200); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (first, _) = scanner.scan_codex_detailed(None); assert_eq!(first.sessions_count, 2); @@ -2267,15 +2153,10 @@ fn cost_scan_reconciliation_preserves_sibling_totals_once() { #[test] fn cancelled_scan_after_deletion_preserves_stale_cache_row() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (first, _) = scanner.scan_codex_detailed(None); assert_eq!(first.sessions_count, 1); std::fs::remove_file(&path).unwrap(); @@ -2297,15 +2178,10 @@ fn cancelled_scan_after_deletion_preserves_stale_cache_row() { #[test] fn cost_scan_resumes_appended_bytes() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "grow.jsonl", 50); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (s1, st1) = scanner.scan_codex_detailed(None); assert_eq!(st1.files_parsed, 1); assert_eq!(st1.token_timestamp_comparisons, 0); @@ -2337,10 +2213,7 @@ fn cost_scan_resumes_appended_bytes() { // The append-only path must publish the same aggregate as a fresh // full parse; the optimization is allowed to change work, not data. - let full_scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(root.path().join("fresh-cache")) - .with_sessions_dirs(vec![sessions]); + let full_scanner = app_scanner(7, root.path().join("fresh-cache"), &sessions); let (full, full_stats) = full_scanner.scan_codex_detailed(None); assert_eq!(full_stats.files_parsed, 1); assert_eq!(s2.input_tokens, full.input_tokens); @@ -2370,18 +2243,13 @@ fn codex_partial_rescan_replaces_changed_session_after_cache_reopen() { // session must replace the cached file aggregate even when the first // refresh only consumes a bounded prefix and the next refresh reloads the // cache from disk. - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture_with_inputs(&sessions, "changed.jsonl", &[100, 200]); let old_metadata = std::fs::metadata(&path).unwrap(); let old_size = old_metadata.len(); let old_mtime = old_metadata.modified().unwrap(); - let initial_scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let initial_scanner = app_scanner(7, &cache_root, &sessions); let (initial, initial_stats) = initial_scanner.scan_codex_detailed(None); assert_eq!(initial_stats.files_parsed, 1); assert_eq!(initial.input_tokens, 200); @@ -2454,10 +2322,7 @@ fn codex_partial_rescan_replaces_changed_session_after_cache_reopen() { // Resumption may change the amount of work, but it must publish the same // cost aggregate as a clean full parse of the rewritten session. - let fresh_scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(root.path().join("fresh-cache")) - .with_sessions_dirs(vec![sessions]); + let fresh_scanner = app_scanner(7, root.path().join("fresh-cache"), &sessions); let (fresh, fresh_stats) = fresh_scanner.scan_codex_detailed(None); assert_eq!(fresh_stats.files_parsed, 1); assert_eq!(resumed.input_tokens, fresh.input_tokens); @@ -2475,9 +2340,7 @@ fn codex_partial_rescan_replaces_changed_session_after_cache_reopen() { #[test] fn bounded_growing_rollout_freezes_target_and_resumes_a_retained_tail() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture_with_inputs(&sessions, "growing.jsonl", &[100, 200, 300]); let initial_size = i64::try_from(std::fs::metadata(&path).unwrap().len()) @@ -2503,10 +2366,7 @@ fn bounded_growing_rollout_freezes_target_and_resumes_a_retained_tail() { let (first, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(first.input_tokens, 100); - let first_usage = first_cache - .files - .get(&path.to_string_lossy().to_string()) - .unwrap(); + let first_usage = cached_file(&first_cache, &path); assert_eq!(first_usage.codex_scan_target_size, Some(initial_size)); assert_eq!(first_usage.parsed_bytes, Some(first_line_bytes)); assert!(first_cache.codex_scan_incomplete); @@ -2535,10 +2395,7 @@ fn bounded_growing_rollout_freezes_target_and_resumes_a_retained_tail() { drop(file); (bounded_summary, _, bounded_cache) = scanner.scan_codex_detailed_with_cache(None); - let usage = bounded_cache - .files - .get(&path.to_string_lossy().to_string()) - .unwrap(); + let usage = cached_file(&bounded_cache, &path); assert_eq!(usage.codex_scan_target_size, Some(initial_size)); assert!(usage.parsed_bytes.unwrap_or_default() <= initial_size); if usage.parsed_bytes == Some(initial_size) { @@ -2547,10 +2404,7 @@ fn bounded_growing_rollout_freezes_target_and_resumes_a_retained_tail() { } assert_eq!(bounded_summary.input_tokens, 300); - let bounded_usage = bounded_cache - .files - .get(&path.to_string_lossy().to_string()) - .unwrap(); + let bounded_usage = cached_file(&bounded_cache, &path); assert_eq!(bounded_usage.parsed_bytes, Some(initial_size)); assert_eq!(bounded_usage.codex_scan_target_size, Some(initial_size)); assert!( @@ -2578,10 +2432,7 @@ fn bounded_growing_rollout_freezes_target_and_resumes_a_retained_tail() { file.write_all(&partial_line.as_bytes()[..split]).unwrap(); drop(file); let (partial_summary, _, partial_cache) = scanner.scan_codex_detailed_with_cache(None); - let partial_usage = partial_cache - .files - .get(&path.to_string_lossy().to_string()) - .unwrap(); + let partial_usage = cached_file(&partial_cache, &path); assert_eq!(partial_summary.input_tokens, stable_summary.input_tokens); assert_eq!(partial_usage.parsed_bytes, Some(stable_size)); assert_eq!(partial_usage.codex_scan_target_size, Some(stable_size)); @@ -2597,10 +2448,7 @@ fn bounded_growing_rollout_freezes_target_and_resumes_a_retained_tail() { let (resumed_summary, _, resumed_cache) = scanner.scan_codex_detailed_with_cache(None); assert!(!resumed_cache.codex_scan_incomplete); assert_eq!(resumed_summary.input_tokens, next_total - 100); - let resumed_usage = resumed_cache - .files - .get(&path.to_string_lossy().to_string()) - .unwrap(); + let resumed_usage = cached_file(&resumed_cache, &path); assert_eq!( resumed_usage.parsed_bytes, Some( @@ -2620,15 +2468,10 @@ fn cost_scan_midline_rewrite_forces_full_parse_not_resume() { // cached resume offset is now mid-line (byte before offset is not \n), // the scanner must fall through to a full re-parse from offset 0 rather // than resuming from the stale mid-line offset. - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let _path = write_codex_session_fixture(&sessions, "a.jsonl", 100); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (s1, st1) = scanner.scan_codex_detailed(None); assert_eq!(st1.files_parsed, 1); assert_eq!(s1.input_tokens, 100); @@ -2636,10 +2479,7 @@ fn cost_scan_midline_rewrite_forces_full_parse_not_resume() { // Rewrite the file with a shorter body at the same path so the cached // parsed_bytes offset now points mid-line in the new content. let today = Local::now().date_naive(); - let day_dir = sessions - .join(today.format("%Y").to_string()) - .join(today.format("%m").to_string()) - .join(today.format("%d").to_string()); + let day_dir = partition_dir(&sessions, today); let ts = (Utc::now() - Duration::minutes(30)) .format("%Y-%m-%dT%H:%M:%S%.3fZ") .to_string(); @@ -2667,15 +2507,10 @@ fn cost_scan_midline_rewrite_forces_full_parse_not_resume() { fn previous_report_clears_after_successful_full_scan() { // F8 (upstream 0.48.0): a completed full scan clears previous_report so // the refreshing indicator does not stay permanently on. - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); write_codex_session_fixture(&sessions, "a.jsonl", 100); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); // First scan: builds cache fresh; no previous_report expected. let (summary1, _) = scanner.scan_codex_detailed(None); @@ -2725,15 +2560,10 @@ fn previous_report_clears_after_successful_full_scan() { #[test] fn known_zero_is_set_when_scan_completes_with_no_sessions() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); std::fs::create_dir_all(&sessions).unwrap(); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _) = scanner.scan_codex_detailed(None); assert!(summary.history_coverage_established, "scan completed"); @@ -2743,15 +2573,10 @@ fn known_zero_is_set_when_scan_completes_with_no_sessions() { #[test] fn known_zero_is_not_set_when_scan_has_results() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); write_codex_session_fixture(&sessions, "a.jsonl", 100); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _) = scanner.scan_codex_detailed(None); assert!(summary.history_coverage_established); @@ -2761,9 +2586,7 @@ fn known_zero_is_not_set_when_scan_has_results() { #[test] fn tiny_candidate_limit_prefers_newest_dirty_file_and_persists_older_pending() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let older = write_codex_session_fixture(&sessions, "a-older.jsonl", 100); let newer = write_codex_session_fixture(&sessions, "z-newer.jsonl", 200); @@ -2851,10 +2674,7 @@ fn tiny_byte_limit_resumes_and_drains_to_unbounded_totals() { let final_bounded = final_bounded.expect("bounded passes drain"); assert!(saw_resume, "later passes resume the cached prefix"); - let full = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(root.path().join("full-cache")) - .with_sessions_dirs(vec![sessions]); + let full = app_scanner(7, root.path().join("full-cache"), &sessions); let (full_summary, _, full_cache) = full.scan_codex_detailed_with_cache(None); assert!(!full_cache.codex_scan_incomplete); assert_eq!(final_bounded.input_tokens, full_summary.input_tokens); @@ -2867,9 +2687,7 @@ fn tiny_byte_limit_resumes_and_drains_to_unbounded_totals() { #[test] fn incomplete_summary_preserves_previous_report_and_marks_it_non_authoritative() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); write_codex_session_fixture_with_inputs(&sessions, "multi.jsonl", &[100, 200]); let report = CachedCostReport { @@ -2889,10 +2707,7 @@ fn incomplete_summary_preserves_previous_report_and_marks_it_non_authoritative() }; JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let cancel = AtomicBool::new(true); let (summary, _, saved) = scanner.scan_codex_detailed_with_cache(Some(&cancel)); @@ -2914,9 +2729,7 @@ fn incomplete_summary_preserves_previous_report_and_marks_it_non_authoritative() #[test] fn failed_catch_up_pause_preserves_cursor_and_report_until_explicit_refresh() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (root, sessions, cache_root) = codex_scan_dirs(); let pending = write_codex_session_fixture(&sessions, "pending.jsonl", 100); let report = CachedCostReport { total_cost_usd: 42.5, @@ -2972,10 +2785,7 @@ fn failed_catch_up_pause_preserves_cursor_and_report_until_explicit_refresh() { failed_cache.codex_scan_pause_reason ); - let explicit = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let explicit = app_scanner(7, &cache_root, &sessions); let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); assert!(resumed_cache.codex_scan_pause_reason.is_none()); assert!(!resumed_cache.codex_scan_incomplete); @@ -2993,10 +2803,7 @@ fn missing_unobserved_sessions_root_does_not_pause_validated_cache() { let cache_root = root.path().join("cache"); write_codex_session_fixture(&sessions, "observed.jsonl", 100); - let initial = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let initial = app_scanner(7, &cache_root, &sessions); let (initial_summary, _) = initial.scan_codex_detailed(None); assert!(initial_summary.history_coverage_established); @@ -3017,15 +2824,10 @@ fn missing_unobserved_sessions_root_does_not_pause_validated_cache() { #[test] fn trace_pruning_preserves_cursor_and_validated_history_until_refresh() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "pruned.jsonl", 100); - let initial = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let initial = app_scanner(7, &cache_root, &sessions); let (initial_summary, _, initial_cache) = initial.scan_codex_detailed_with_cache(None); assert_eq!(initial_summary.input_tokens, 100); assert!(!initial_cache.codex_scan_incomplete); @@ -3052,10 +2854,7 @@ fn trace_pruning_preserves_cursor_and_validated_history_until_refresh() { ); assert!(paused_cache.previous_report.is_some()); - let explicit = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let explicit = app_scanner(7, &cache_root, &sessions); let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); assert_eq!(resumed_summary.input_tokens, 1); assert!(resumed_cache.codex_scan_pause_reason.is_none()); @@ -3081,9 +2880,7 @@ fn pending_codex_options(force: bool) -> CostScanOptions { #[test] fn pending_codex_scan_30_to_7_keeps_wide_start_and_narrow_report() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); for index in 0..3 { write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); } @@ -3112,9 +2909,7 @@ fn pending_codex_scan_30_to_7_keeps_wide_start_and_narrow_report() { #[test] fn pending_codex_scan_7_to_30_expands_to_earliest_start() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); for index in 0..3 { write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); } @@ -3140,9 +2935,7 @@ fn pending_codex_scan_7_to_30_expands_to_earliest_start() { #[test] fn pending_codex_scan_repeated_narrow_wide_alternation_is_monotonic() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); for index in 0..8 { write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); } @@ -3223,9 +3016,7 @@ fn pending_codex_scan_incompatible_root_timezone_or_end_resets_start() { #[test] fn pending_codex_scan_force_rescan_resets_start() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); for index in 0..3 { write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); } @@ -3251,9 +3042,7 @@ fn pending_codex_scan_force_rescan_resets_start() { #[test] fn pending_codex_scan_start_survives_restart() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); for index in 0..3 { write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); } @@ -3287,15 +3076,10 @@ fn pending_codex_scan_start_survives_restart() { #[test] fn disappeared_trace_path_waits_for_explicit_validation() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "disappeared.jsonl", 100); - let initial = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let initial = app_scanner(7, &cache_root, &sessions); let (initial_summary, _, _) = initial.scan_codex_detailed_with_cache(None); assert_eq!(initial_summary.input_tokens, 100); @@ -3318,10 +3102,7 @@ fn disappeared_trace_path_waits_for_explicit_validation() { vec![path.to_string_lossy().to_string()] ); - let explicit = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let explicit = app_scanner(7, &cache_root, &sessions); let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); assert_eq!(resumed_summary.sessions_count, 0); assert!(resumed_summary.history_coverage_established); @@ -3332,9 +3113,7 @@ fn disappeared_trace_path_waits_for_explicit_validation() { #[test] fn paused_catch_up_round_trips_without_retrying_in_background() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let pending = write_codex_session_fixture(&sessions, "pending.jsonl", 100); let report = CachedCostReport { total_cost_usd: 9.25, @@ -3395,9 +3174,7 @@ fn pending_and_incomplete_round_trip_through_cache_json() { #[test] fn deleted_pending_path_is_pruned_after_complete_discovery() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture_with_inputs(&sessions, "partial.jsonl", &[100, 200]); let first_line_bytes = i64::try_from( std::fs::read(&path) @@ -3444,17 +3221,12 @@ fn legacy_cache_json_defaults_bounded_scan_state() { #[test] fn complete_empty_codex_fragment_persists_in_cache() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "empty.jsonl", 100); std::fs::write(&path, b"\n").unwrap(); let key = path.to_string_lossy().to_string(); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(summary.input_tokens, 0); @@ -3469,17 +3241,12 @@ fn complete_empty_codex_fragment_persists_in_cache() { #[test] fn complete_empty_codex_fragment_reparses_from_start_after_growth() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "empty.jsonl", 100); std::fs::write(&path, b"\n").unwrap(); let key = path.to_string_lossy().to_string(); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); let first = first_cache.files.get(&key).expect("initial empty fragment"); assert_eq!(first.parsed_bytes, Some(1)); @@ -3495,17 +3262,12 @@ fn complete_empty_codex_fragment_reparses_from_start_after_growth() { #[test] fn incomplete_or_buffered_empty_codex_fragment_is_not_marked_complete() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let path = write_codex_session_fixture(&sessions, "incomplete.jsonl", 100); std::fs::write(&path, br#"{"timestamp":"2026-09-07T00:00:00Z""#).unwrap(); let key = path.to_string_lossy().to_string(); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); let entry = cache @@ -3627,7 +3389,6 @@ fn claude_day_total_includes_cache_and_matches_summary_and_model_totals() { assert_eq!(model_total, window_total); } -#[cfg(test)] #[path = "tests/claude_swap.rs"] mod claude_swap; #[cfg(test)] @@ -3651,6 +3412,9 @@ mod lineage_cache; #[cfg(test)] #[path = "tests/paginated.rs"] mod paginated; +#[cfg(test)] +#[path = "tests/support.rs"] +mod support; #[path = "tests/period.rs"] mod period; diff --git a/rust/src/cost_scanner/tests/archived.rs b/rust/src/cost_scanner/tests/archived.rs index e75f690518..f1175ad724 100644 --- a/rust/src/cost_scanner/tests/archived.rs +++ b/rust/src/cost_scanner/tests/archived.rs @@ -41,13 +41,6 @@ fn rollout_body(day: NaiveDate, session: &str, input: u64) -> String { ) } -fn partition_dir(sessions: &Path, day: NaiveDate) -> PathBuf { - sessions - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()) -} - fn write_file(dir: &Path, name: &str, body: &str) -> PathBuf { std::fs::create_dir_all(dir).unwrap(); let path = dir.join(name); diff --git a/rust/src/cost_scanner/tests/copied_prefix.rs b/rust/src/cost_scanner/tests/copied_prefix.rs index 23e1dccb22..cbca2cef1b 100644 --- a/rust/src/cost_scanner/tests/copied_prefix.rs +++ b/rust/src/cost_scanner/tests/copied_prefix.rs @@ -9,10 +9,7 @@ fn write_copied_prefix_subagent_fixture( owned: bool, ) -> PathBuf { let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions_root - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, day); std::fs::create_dir_all(&day_dir).unwrap(); let path = day_dir.join(name); let mut lines = vec![ @@ -97,9 +94,7 @@ fn token_row( #[test] fn copied_prefix_subagent_infers_advancing_baseline_without_parent() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let child = write_copied_prefix_subagent_fixture( &sessions, "child.jsonl", @@ -108,10 +103,7 @@ fn copied_prefix_subagent_infers_advancing_baseline_without_parent() { Utc::now() - Duration::hours(1), true, ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(summary.input_tokens, 70); @@ -138,9 +130,7 @@ fn copied_prefix_subagent_infers_advancing_baseline_without_parent() { #[test] fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let child = write_copied_prefix_subagent_fixture( &sessions, "child.jsonl", @@ -149,10 +139,7 @@ fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { Utc::now() - Duration::hours(1), false, ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(summary.input_tokens, 0); @@ -174,9 +161,7 @@ fn copied_prefix_subagent_inherited_only_suffix_is_not_billed() { #[test] fn copied_prefix_subagent_prefers_validated_parent_baseline() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let parent = write_codex_fork_session_fixture( &sessions, @@ -240,9 +225,7 @@ fn copied_prefix_subagent_prefers_validated_parent_baseline() { #[test] fn candidate_limit_counts_each_child_parent_candidate_once() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let parent = write_codex_fork_session_fixture( &sessions, @@ -299,9 +282,7 @@ fn candidate_limit_counts_each_child_parent_candidate_once() { #[test] fn cold_scan_orders_multi_level_parent_chain_before_children() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let ancestor = write_codex_fork_session_fixture( &sessions, @@ -395,9 +376,7 @@ fn assert_unsafe_lineage_is_unresolved( #[test] fn duplicate_parent_session_ids_fail_closed_with_their_child() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let first_parent = write_codex_fork_session_fixture( &sessions, @@ -444,9 +423,7 @@ fn duplicate_parent_session_ids_fail_closed_with_their_child() { #[test] fn two_node_subagent_cycle_fails_closed() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let first = write_copied_prefix_subagent_fixture( &sessions, @@ -478,9 +455,7 @@ fn two_node_subagent_cycle_fails_closed() { #[test] fn self_referential_subagent_fails_closed() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let session = write_copied_prefix_subagent_fixture( &sessions, "self-cycle.jsonl", @@ -489,10 +464,7 @@ fn self_referential_subagent_fails_closed() { Utc::now() - Duration::hours(1), true, ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, stats, cache) = scanner.scan_codex_detailed_with_cache(None); @@ -502,9 +474,7 @@ fn self_referential_subagent_fails_closed() { fn assert_cached_inference_is_replaced_when_parent_appears( prefer_newest_codex_sessions_first: bool, ) { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_copied_prefix_subagent_fixture( &sessions, diff --git a/rust/src/cost_scanner/tests/direct_fork.rs b/rust/src/cost_scanner/tests/direct_fork.rs index c3099fef7f..84d65b4470 100644 --- a/rust/src/cost_scanner/tests/direct_fork.rs +++ b/rust/src/cost_scanner/tests/direct_fork.rs @@ -79,15 +79,10 @@ impl Chain { /// `child` forked at t=4. Upstream covers no parent event and events at t=3 /// and t=8; t=2 additionally puts the parent's event at its own fork instant. fn write_chain(parent_event_time: Option) -> Chain { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(&sessions, day); let mut chain = Chain { sessions, cache_root, @@ -276,15 +271,10 @@ fn changed_root_revalidates_empty_parent_descendants() { #[test] fn cyclic_empty_fork_ancestry_stays_unresolved() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(&sessions, day); let files = [("a", "b"), ("b", "a")].map(|(id, parent)| { write_rows( &day_dir, @@ -312,10 +302,7 @@ fn scan_empty_fork_ladder(depth: usize) -> (CostSummary, CostUsageCache, PathBuf let sessions = root.path().join("sessions"); let base = Utc::now() - Duration::hours(1); let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(&sessions, day); write_rows( &day_dir, "s000.jsonl", diff --git a/rust/src/cost_scanner/tests/fork_resume.rs b/rust/src/cost_scanner/tests/fork_resume.rs index d463f84cd1..38ea8ce105 100644 --- a/rust/src/cost_scanner/tests/fork_resume.rs +++ b/rust/src/cost_scanner/tests/fork_resume.rs @@ -37,10 +37,7 @@ impl Sessions { let sessions = root.path().join("sessions"); let base = Utc::now() - Duration::hours(1); let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(&sessions, day); Self { root, sessions, diff --git a/rust/src/cost_scanner/tests/lineage_cache.rs b/rust/src/cost_scanner/tests/lineage_cache.rs index 124733b098..c000d9edc6 100644 --- a/rust/src/cost_scanner/tests/lineage_cache.rs +++ b/rust/src/cost_scanner/tests/lineage_cache.rs @@ -8,10 +8,7 @@ fn write_subagent( timestamp: DateTime, ) -> PathBuf { let day = timestamp.with_timezone(&Local).date_naive(); - let day_dir = sessions_root - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, day); std::fs::create_dir_all(&day_dir).unwrap(); let path = day_dir.join(name); let rows = [ @@ -70,10 +67,7 @@ fn write_missing_ordinal_subagent( include_owned_usage: bool, ) -> PathBuf { let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions_root - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, day); std::fs::create_dir_all(&day_dir).unwrap(); let path = day_dir.join(name); let mut missing_ordinal = lineage_token_row(base, 11, 100, 0); @@ -141,9 +135,7 @@ fn assert_unresolved(cache: &CostUsageCache, path: &Path) { fn appended_owned_token_row_reinfers_locally_resolved_subagent_from_start() { use std::io::Write as _; - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_subagent(&sessions, "child.jsonl", "child-id", "missing-parent", base); let scanner = bounded_scanner(&sessions, &cache_root); @@ -167,9 +159,7 @@ fn appended_owned_token_row_reinfers_locally_resolved_subagent_from_start() { #[test] fn replaced_parent_with_same_path_size_and_mtime_cannot_author_lineage() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let parent = write_codex_fork_session_fixture( &sessions, @@ -187,10 +177,7 @@ fn replaced_parent_with_same_path_size_and_mtime_cannot_author_lineage() { "parent-id", base + Duration::seconds(10), ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); let parent_key = parent.to_string_lossy().to_string(); let child_usage = &cache.files[&child.to_string_lossy().to_string()]; @@ -238,15 +225,10 @@ fn replaced_parent_with_same_path_size_and_mtime_cannot_author_lineage() { #[test] fn missing_explicit_ordinal_keeps_subagent_cache_unresolved() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_missing_ordinal_subagent(&sessions, "child.jsonl", base, true); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); @@ -257,15 +239,10 @@ fn missing_explicit_ordinal_keeps_subagent_cache_unresolved() { #[test] fn missing_ordinal_cannot_complete_zero_usage_subagent_cache() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_missing_ordinal_subagent(&sessions, "child.jsonl", base, false); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); @@ -278,9 +255,7 @@ fn missing_ordinal_cannot_complete_zero_usage_subagent_cache() { fn missing_ordinal_after_local_resolution_keeps_subagent_cache_unresolved() { use std::io::Write as _; - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_subagent( &sessions, @@ -308,9 +283,7 @@ fn missing_ordinal_after_local_resolution_keeps_subagent_cache_unresolved() { #[test] fn legacy_cache_without_file_identity_is_reparsed() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let session = write_codex_fork_session_fixture( &sessions, @@ -321,10 +294,7 @@ fn legacy_cache_without_file_identity_is_reparsed() { base, &[1_000], ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, _) = scanner.scan_codex_detailed_with_cache(None); let session_key = session.to_string_lossy().to_string(); @@ -348,9 +318,7 @@ fn legacy_cache_without_file_identity_is_reparsed() { #[test] fn bounded_refresh_detects_duplicate_parent_owners_across_cache_and_candidate() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_subagent(&sessions, "child.jsonl", "child-id", "parent-id", base); let scanner = bounded_scanner(&sessions, &cache_root); @@ -395,9 +363,7 @@ fn bounded_refresh_detects_duplicate_parent_owners_across_cache_and_candidate() #[test] fn bounded_refresh_detects_equal_timestamp_two_node_cycle() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let first = write_subagent(&sessions, "first.jsonl", "first-id", "second-id", base); let scanner = bounded_scanner(&sessions, &cache_root); @@ -416,9 +382,7 @@ fn bounded_refresh_detects_equal_timestamp_two_node_cycle() { #[test] fn bounded_refresh_rejects_self_cycle_migration() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let session = write_subagent(&sessions, "self.jsonl", "self-id", "missing-id", base); let scanner = bounded_scanner(&sessions, &cache_root); @@ -442,9 +406,7 @@ fn bounded_refresh_rejects_self_cycle_migration() { #[test] fn bounded_refresh_rejects_dependent_of_locally_inferred_parent() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let parent = write_subagent(&sessions, "parent.jsonl", "parent-id", "missing-id", base); let scanner = bounded_scanner(&sessions, &cache_root); @@ -468,16 +430,11 @@ fn bounded_refresh_rejects_dependent_of_locally_inferred_parent() { #[test] fn current_refresh_scopes_unsafe_cache_invalidation_to_range_and_dependencies() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let active_time = Utc::now() - Duration::hours(1); let old_date = Local::now().date_naive() - Duration::days(30); let old_day = old_date.format("%Y-%m-%d").to_string(); - let old_dir = sessions - .join(old_date.format("%Y").to_string()) - .join(old_date.format("%m").to_string()) - .join(old_date.format("%d").to_string()); + let old_dir = partition_dir(&sessions, old_date); let mut cache = CostUsageCache::default(); { @@ -504,10 +461,7 @@ fn current_refresh_scopes_unsafe_cache_invalidation_to_range_and_dependencies() active_time + Duration::seconds(1), &[1_000_000, 1_000_140], ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, refreshed) = scanner.scan_codex_detailed_with_cache(None); let cached_path = |name: &str| old_dir.join(name).to_string_lossy().to_string(); diff --git a/rust/src/cost_scanner/tests/paginated.rs b/rust/src/cost_scanner/tests/paginated.rs index fd4157eed7..ec625fe13e 100644 --- a/rust/src/cost_scanner/tests/paginated.rs +++ b/rust/src/cost_scanner/tests/paginated.rs @@ -10,10 +10,7 @@ fn write_codex_paginated_subagent_fixture( base: DateTime, ) -> PathBuf { let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions_root - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, day); std::fs::create_dir_all(&day_dir).unwrap(); let path = day_dir.join(name); // Desktop v2 aliases session_id to the parent, while id identifies the @@ -80,10 +77,7 @@ fn write_codex_paginated_continuation_fixture( base: DateTime, ) -> PathBuf { let day = base.with_timezone(&Local).date_naive(); - let day_dir = sessions_root - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, day); std::fs::create_dir_all(&day_dir).unwrap(); let path = day_dir.join(name); let metadata = serde_json::json!({ @@ -137,9 +131,7 @@ fn write_codex_paginated_continuation_fixture( #[test] fn paginated_continuation_raises_inherited_baseline_from_total_last() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = recent_codex_fixture_time(); write_codex_fork_session_fixture( &sessions, @@ -247,9 +239,7 @@ fn bounded_paginated_continuation_raises_its_baseline_once() { #[test] fn paginated_history_base_equal_parent_keeps_true_fork_subtraction() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = recent_codex_fixture_time(); write_codex_fork_session_fixture( &sessions, @@ -293,19 +283,14 @@ fn paginated_history_base_equal_parent_keeps_true_fork_subtraction() { #[test] fn paginated_v2_subagent_counts_own_usage_without_parent() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let child = write_codex_paginated_subagent_fixture( &sessions, "child.jsonl", "child-id", Utc::now() - Duration::hours(1), ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(summary.input_tokens, 42_005); @@ -328,9 +313,7 @@ fn paginated_v2_subagent_counts_own_usage_without_parent() { #[test] fn paginated_v2_subagent_does_not_subtract_or_double_count_continued_parent() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let parent = write_codex_fork_session_fixture( &sessions, @@ -347,10 +330,7 @@ fn paginated_v2_subagent_does_not_subtract_or_double_count_continued_parent() { "child-id", base + Duration::seconds(1), ); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); assert_eq!(summary.input_tokens, 1_042_005); @@ -390,15 +370,10 @@ fn paginated_v2_subagent_does_not_subtract_or_double_count_continued_parent() { #[test] fn paginated_v2_subagent_recovers_unchanged_previously_unresolved_cache() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_codex_paginated_subagent_fixture(&sessions, "child.jsonl", "child-id", base); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, mut old_cache) = scanner.scan_codex_detailed_with_cache(None); let child_key = child.to_string_lossy().to_string(); let old_usage = old_cache.files.get_mut(&child_key).unwrap(); @@ -447,15 +422,10 @@ fn paginated_v2_subagent_recovers_unchanged_previously_unresolved_cache() { #[test] fn paginated_v2_subagent_repairs_stale_complete_cache_metadata() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let child = write_codex_paginated_subagent_fixture(&sessions, "child.jsonl", "child-id", base); - let scanner = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); + let scanner = app_scanner(7, &cache_root, &sessions); let (_, _, mut cache) = scanner.scan_codex_detailed_with_cache(None); let child_key = child.to_string_lossy().to_string(); let usage = cache.files.get_mut(&child_key).unwrap(); @@ -474,9 +444,7 @@ fn paginated_v2_subagent_repairs_stale_complete_cache_metadata() { #[test] fn paginated_v2_subagent_background_refresh_adds_new_local_day_without_repair() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let today = Local::now().date_naive(); let yesterday = today.pred_opt().unwrap(); let local_noon = |day: chrono::NaiveDate| { @@ -546,9 +514,7 @@ fn paginated_v2_subagent_background_refresh_adds_new_local_day_without_repair() #[test] fn unresolved_legacy_fork_keeps_background_daily_usage_live() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let today = Local::now().date_naive(); let yesterday = today.pred_opt().unwrap(); let local_noon = |day: chrono::NaiveDate| { @@ -670,9 +636,7 @@ fn unresolved_legacy_fork_keeps_background_daily_usage_live() { #[test] fn unresolved_legacy_fork_old_no_progress_pause_retains_report_in_background() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("cache"); + let (_root, sessions, cache_root) = codex_scan_dirs(); let base = Utc::now() - Duration::hours(1); let unresolved = write_codex_fork_session_fixture( &sessions, @@ -692,10 +656,7 @@ fn unresolved_legacy_fork_old_no_progress_pause_retains_report_in_background() { base, &[100], ); - let initial = CostScanner::new(7) - .with_options(CostScanOptions::app_driven()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); + let initial = app_scanner(7, &cache_root, &sessions); let (_, _, mut paused) = initial.scan_codex_detailed_with_cache(None); let unresolved_key = unresolved.to_string_lossy().to_string(); let unchanged_fork = paused.files[&unresolved_key].clone(); diff --git a/rust/src/cost_scanner/tests/period.rs b/rust/src/cost_scanner/tests/period.rs index 68d84c72a0..1daf48ff68 100644 --- a/rust/src/cost_scanner/tests/period.rs +++ b/rust/src/cost_scanner/tests/period.rs @@ -15,10 +15,7 @@ fn date(y: i32, m: u32, d: u32) -> NaiveDate { /// Writes one Codex session under the `YYYY/MM/DD` partition for `day`. fn write_session_on(sessions_root: &Path, day: NaiveDate, input_tokens: u64) { - let day_dir = sessions_root - .join(day.format("%Y").to_string()) - .join(day.format("%m").to_string()) - .join(day.format("%d").to_string()); + let day_dir = partition_dir(sessions_root, day); std::fs::create_dir_all(&day_dir).unwrap(); // Local noon keeps the record on `day` in the local zone. let local_noon = Local diff --git a/rust/src/cost_scanner/tests/support.rs b/rust/src/cost_scanner/tests/support.rs new file mode 100644 index 0000000000..1dc4aa823c --- /dev/null +++ b/rust/src/cost_scanner/tests/support.rs @@ -0,0 +1,38 @@ +//! Setup shared by the cost scanner tests. + +use super::*; + +/// A temp root holding the `sessions` and `cache` dirs most Codex scans use. +pub(super) fn codex_scan_dirs() -> (tempfile::TempDir, PathBuf, PathBuf) { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("cache"); + (root, sessions, cache_root) +} + +/// The app-driven scanner over one sessions dir. +pub(super) fn app_scanner( + days: u32, + cache_root: impl Into, + sessions: &Path, +) -> CostScanner { + CostScanner::new(days) + .with_options(CostScanOptions::app_driven()) + .with_cache_root(cache_root) + .with_sessions_dirs(vec![sessions.to_path_buf()]) +} + +/// The `YYYY/MM/DD` partition for `day` under a sessions dir. +pub(super) fn partition_dir(sessions: &Path, day: NaiveDate) -> PathBuf { + sessions + .join(day.format("%Y").to_string()) + .join(day.format("%m").to_string()) + .join(day.format("%d").to_string()) +} + +pub(super) fn cached_file<'a>(cache: &'a CostUsageCache, path: &Path) -> &'a CostUsageFileUsage { + cache + .files + .get(&path.to_string_lossy().to_string()) + .unwrap() +} From 3e0484ac288bf8815d5207208ca8ad3bd8808c42 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:49:56 +0700 Subject: [PATCH 34/41] Move the pi session cost tests into their own file --- rust/src/pi_session_cost.rs | 371 +---------------------------- rust/src/pi_session_cost/tests.rs | 372 ++++++++++++++++++++++++++++++ 2 files changed, 373 insertions(+), 370 deletions(-) create mode 100644 rust/src/pi_session_cost/tests.rs diff --git a/rust/src/pi_session_cost.rs b/rust/src/pi_session_cost.rs index d02c041fe0..b1273dc13c 100644 --- a/rust/src/pi_session_cost.rs +++ b/rust/src/pi_session_cost.rs @@ -653,373 +653,4 @@ fn num(usage: &Value, keys: &[&str]) -> u64 { } #[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - use tempfile::tempdir; - - fn parse_pi_assistant_entry(value: &Value, target: PiMappedProvider) -> Option { - let entry = parse_pi_assistant_entry_any(value)?; - (entry.provider == target).then_some(entry) - } - - #[test] - fn maps_openai_codex_and_anthropic_providers() { - assert_eq!( - map_provider("openai-codex-responses"), - Some(PiMappedProvider::Codex) - ); - assert_eq!(map_provider("anthropic"), Some(PiMappedProvider::Claude)); - assert_eq!(map_provider("google"), None); - } - - #[test] - fn parses_assistant_usage_row() { - let raw = serde_json::json!({ - "id": "msg-1", - "role": "assistant", - "provider": "openai-codex", - "model": "gpt-5", - "timestamp": "2026-07-20T12:00:00Z", - "usage": { "input": 100, "output": 20, "cacheRead": 10 } - }); - let entry = parse_pi_assistant_entry(&raw, PiMappedProvider::Codex).unwrap(); - assert_eq!(entry.input, 100); - assert_eq!(entry.output, 20); - assert_eq!(entry.cache_read, 10); - assert_eq!(entry.model, "gpt-5"); - assert!(entry.pricing_known); - } - - #[test] - fn parses_astra_cache_write_and_prices_it() { - let raw = serde_json::json!({ - "id": "astra-msg-1", - "role": "assistant", - "provider": "openai-codex", - "model": "gpt-6-astra", - "usage": { - "input": 1_000, - "output": 100, - "cacheRead": 200, - "cacheWrite": 300 - } - }); - let entry = parse_pi_assistant_entry(&raw, PiMappedProvider::Codex).unwrap(); - let expected = 500.0 * 1e-5 + 200.0 * 1e-6 + 300.0 * 1.25e-5 + 100.0 * 5e-5; - assert_eq!(entry.cache_create, 300); - assert!((entry.cost - expected).abs() < 1e-12); - assert!(entry.pricing_known); - } - - #[test] - fn prices_codex_rows_at_their_utc_timestamp() { - let cost = |timestamp: Option<&str>| { - let mut raw = serde_json::json!({ - "id": "sol-1", "role": "assistant", "provider": "openai-codex", - "model": "gpt-5.6-sol", - "usage": { "input": 100, "output": 5, "cacheRead": 10, "cacheWrite": 20 } - }); - if let Some(timestamp) = timestamp { - raw["timestamp"] = timestamp.into(); - } - parse_pi_assistant_entry(&raw, PiMappedProvider::Codex) - .unwrap() - .cost - }; - let historical = 70.0 * 5e-6 + 10.0 * 5e-7 + 20.0 * 6.25e-6 + 5.0 * 3e-5; - let current = 70.0 * 4e-6 + 10.0 * 4e-7 + 20.0 * 5e-6 + 5.0 * 2e-5; - for (timestamp, expected) in [ - (Some("2026-07-10T12:00:00Z"), historical), - (Some("2026-08-20T23:59:59Z"), historical), - (Some("2026-08-21T00:00:00Z"), current), - (Some("2026-09-10T12:00:00Z"), current), - (None, current), - ] { - assert!((cost(timestamp) - expected).abs() < 1e-12, "{timestamp:?}"); - } - } - - #[test] - fn unknown_model_keeps_tokens_and_marks_pricing_incomplete() { - let raw = serde_json::json!({ - "id": "unknown-model-1", - "role": "assistant", - "provider": "openai-codex", - "model": "future-model-without-a-rate", - "usage": { "input": 11, "output": 3 } - }); - let entry = parse_pi_assistant_entry_any(&raw).unwrap(); - assert!(!entry.pricing_known); - assert_eq!(entry.cost, 0.0); - - let mut summary = CostSummary::default(); - apply_entry(&mut summary, &entry); - assert_eq!(summary.input_tokens, 11); - assert_eq!(summary.output_tokens, 3); - assert!( - summary - .unknown_models - .contains("future-model-without-a-rate") - ); - assert!(summary.model_pricing_completeness.is_partial()); - } - - #[test] - fn standalone_parser_keeps_the_mapped_provider_for_mixed_history() { - let codex = serde_json::json!({ - "id": "codex-1", - "role": "assistant", - "provider": "openai-codex", - "model": "gpt-5", - "usage": { "input": 100, "output": 20 } - }); - let claude = serde_json::json!({ - "id": "claude-1", - "role": "assistant", - "provider": "anthropic", - "model": "claude-sonnet-4-6", - "usage": { "input": 100, "output": 20 } - }); - - assert_eq!( - parse_pi_assistant_entry_any(&codex) - .expect("Codex row should parse") - .provider, - PiMappedProvider::Codex - ); - assert_eq!( - parse_pi_assistant_entry_any(&claude) - .expect("Claude row should parse") - .provider, - PiMappedProvider::Claude - ); - } - - #[test] - fn dedupes_shared_entry_ids_across_files() { - let dir = tempdir().unwrap(); - let sessions = dir.path().join("agent").join("sessions"); - std::fs::create_dir_all(&sessions).unwrap(); - let header = r#"{"type":"session","id":"session-1"}"#; - let line = r#"{"id":"shared-1","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":50,"output":5}}"#; - for name in ["a.jsonl", "b.jsonl"] { - let mut f = File::create(sessions.join(name)).unwrap(); - writeln!(f, "{header}").unwrap(); - writeln!(f, "{line}").unwrap(); - } - - // Point home at temp so roots resolve under .omp - let home = dir.path().to_path_buf(); - // Manually walk the sessions we created via for_each - let mut seen = HashSet::new(); - let mut summary = CostSummary::default(); - let mut total = 0u32; - for name in ["a.jsonl", "b.jsonl"] { - total += for_each_pi_entry( - &sessions.join(name), - DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc), - Some(PiMappedProvider::Codex), - &mut seen, - |entry| apply_entry(&mut summary, &entry), - ) - .counted; - } - assert_eq!(seen.len(), 1); - assert_eq!(summary.input_tokens, 50); - assert_eq!(total, 1); // second file deduped - let _ = home; - } - - #[test] - fn same_message_id_in_distinct_sessions_is_not_deduped() { - let dir = tempdir().unwrap(); - let sessions = dir.path().join("agent").join("sessions"); - std::fs::create_dir_all(&sessions).unwrap(); - let line = r#"{"id":"reused-message","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":50,"output":5}}"#; - for session in ["session-a", "session-b"] { - let mut f = File::create(sessions.join(format!("{session}.jsonl"))).unwrap(); - writeln!(f, "{{\"type\":\"session\",\"id\":\"{session}\"}}").unwrap(); - writeln!(f, "{line}").unwrap(); - } - - let cutoff = DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc); - let mut seen = HashSet::new(); - let mut summary = CostSummary::default(); - let mut total = 0u32; - for session in ["session-a", "session-b"] { - total += for_each_pi_entry( - &sessions.join(format!("{session}.jsonl")), - cutoff, - Some(PiMappedProvider::Codex), - &mut seen, - |entry| apply_entry(&mut summary, &entry), - ) - .counted; - } - assert_eq!(total, 2); - assert_eq!(summary.input_tokens, 100); - } - - #[test] - fn configured_roots_include_explicit_pi_and_omp_profile_sources() { - let home = tempdir().unwrap(); - let cwd = home.path().join("project"); - std::fs::create_dir_all(&cwd).unwrap(); - let mut environment = EnvMap::new(); - environment.insert( - "PI_CODING_AGENT_SESSION_DIR".to_string(), - "pi-sessions".to_string(), - ); - environment.insert("PI_CONFIG_DIR".to_string(), "config".to_string()); - environment.insert("OMP_PROFILE".to_string(), "work".to_string()); - - let roots = pi_compatible_session_roots_for(home.path(), &cwd, &environment); - let pi_root = cwd.join("pi-sessions"); - let omp_root = home - .path() - .join("config") - .join("profiles") - .join("work") - .join("agent") - .join("sessions"); - assert!(roots.iter().any(|root| root == &pi_root)); - assert!(roots.iter().any(|root| root == &omp_root)); - - let duplicate = concat!( - r#"{"type":"session","id":"session-1"}"#, - "\n", - r#"{"id":"shared","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":50,"output":5}}"#, - "\n" - ); - let unique = concat!( - r#"{"type":"session","id":"session-2"}"#, - "\n", - r#"{"id":"unique","role":"assistant","provider":"anthropic","model":"claude-sonnet-4-6","timestamp":"2026-07-20T13:00:00Z","usage":{"input":70,"output":7}}"#, - "\n" - ); - std::fs::create_dir_all(&pi_root).unwrap(); - std::fs::create_dir_all(&omp_root).unwrap(); - std::fs::write(pi_root.join("session.jsonl"), duplicate).unwrap(); - std::fs::write(omp_root.join("session.jsonl"), duplicate).unwrap(); - std::fs::write(omp_root.join("unique.jsonl"), unique).unwrap(); - - let scan = scan_pi_daily_from_roots( - DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc), - None, - roots, - ); - assert!(scan.history_coverage_established); - assert_eq!(scan.tokens.values().sum::(), 132); - } - - #[test] - fn malformed_usage_input_keeps_valid_rows_but_marks_source_incomplete() { - let dir = tempdir().unwrap(); - let sessions = dir.path().join("agent").join("sessions"); - std::fs::create_dir_all(&sessions).unwrap(); - let valid = r#"{"id":"valid","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":11,"output":3}}"#; - std::fs::write( - sessions.join("mixed.jsonl"), - format!("{valid}\n{{\"role\":\"assistant\",\"usage\":\n"), - ) - .unwrap(); - - let scan = scan_pi_daily_from_roots( - DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc), - None, - vec![sessions.clone()], - ); - assert!(!scan.history_coverage_established); - assert_eq!(scan.tokens.values().sum::(), 14); - - let mut summary = CostSummary::default(); - let mut seen = HashSet::new(); - let evidence = scan_roots_into( - &mut summary, - Utc::now() - Duration::days(365), - None, - &mut seen, - vec![sessions], - None, - ); - assert!(!evidence.complete); - assert_eq!(summary.input_tokens, 11); - } - - #[test] - fn standalone_daily_scan_dedupes_pi_and_omp_roots() { - let dir = tempdir().unwrap(); - let pi_sessions = dir.path().join(".pi").join("agent").join("sessions"); - let omp_sessions = dir.path().join(".omp").join("agent").join("sessions"); - std::fs::create_dir_all(&pi_sessions).unwrap(); - std::fs::create_dir_all(&omp_sessions).unwrap(); - let codex = r#"{"id":"shared","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":50,"output":5}}"#; - let claude = r#"{"id":"claude-only","role":"assistant","provider":"anthropic","model":"claude-sonnet-4-6","timestamp":"2026-07-20T13:00:00Z","usage":{"input":70,"output":7}}"#; - std::fs::write( - pi_sessions.join("one.jsonl"), - format!("{codex}\n{claude}\n"), - ) - .unwrap(); - std::fs::write(omp_sessions.join("one.jsonl"), format!("{codex}\n")).unwrap(); - - let scan = scan_pi_daily_from_roots( - DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc), - None, - vec![pi_sessions, omp_sessions], - ); - assert!(scan.history_coverage_established); - assert_eq!(scan.tokens.values().sum::(), 132); - assert_eq!(scan.tokens.len(), 1); - } - - #[test] - fn daily_tokens_count_cache_like_the_window_total() { - let dir = tempdir().unwrap(); - let sessions = dir.path().join("agent").join("sessions"); - std::fs::create_dir_all(&sessions).unwrap(); - let codex = r#"{"id":"cached-codex","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":100,"output":20,"cacheRead":10,"cacheWrite":5}}"#; - let claude = r#"{"id":"cached-claude","role":"assistant","provider":"anthropic","model":"claude-sonnet-4-6","timestamp":"2026-07-20T13:00:00Z","usage":{"input":40,"output":4,"cacheRead":300,"cacheWrite":30}}"#; - std::fs::write( - sessions.join("cached.jsonl"), - format!("{codex}\n{claude}\n"), - ) - .unwrap(); - let cutoff = DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc); - - let scan = scan_pi_daily_from_roots(cutoff, None, vec![sessions.clone()]); - assert!(scan.history_coverage_established); - let daily_total = scan.tokens.values().sum::(); - assert_eq!(daily_total, 135 + 374); - - let mut summary = CostSummary::default(); - let mut seen = HashSet::new(); - let evidence = scan_roots_into(&mut summary, cutoff, None, &mut seen, vec![sessions], None); - assert!(evidence.complete); - assert_eq!(summary.total_tokens_for_provider("pi"), daily_total); - } - - #[test] - fn session_roots_include_pi_and_omp() { - let home = PathBuf::from("/home/user"); - let roots = pi_compatible_session_roots_for(&home, &home, &EnvMap::new()); - assert!( - roots - .iter() - .any(|p| p.ends_with(".pi/agent/sessions") || p.ends_with(".pi\\agent\\sessions")) - ); - assert!(roots.iter().any(|p| p.ends_with(".omp/agent/sessions") || p.ends_with(".omp\\agent\\sessions"))); - } -} +mod tests; diff --git a/rust/src/pi_session_cost/tests.rs b/rust/src/pi_session_cost/tests.rs new file mode 100644 index 0000000000..3d744d9596 --- /dev/null +++ b/rust/src/pi_session_cost/tests.rs @@ -0,0 +1,372 @@ +use super::*; +use std::io::Write; +use tempfile::tempdir; + +fn parse_pi_assistant_entry(value: &Value, target: PiMappedProvider) -> Option { + let entry = parse_pi_assistant_entry_any(value)?; + (entry.provider == target).then_some(entry) +} + +#[test] +fn maps_openai_codex_and_anthropic_providers() { + assert_eq!( + map_provider("openai-codex-responses"), + Some(PiMappedProvider::Codex) + ); + assert_eq!(map_provider("anthropic"), Some(PiMappedProvider::Claude)); + assert_eq!(map_provider("google"), None); +} + +#[test] +fn parses_assistant_usage_row() { + let raw = serde_json::json!({ + "id": "msg-1", + "role": "assistant", + "provider": "openai-codex", + "model": "gpt-5", + "timestamp": "2026-07-20T12:00:00Z", + "usage": { "input": 100, "output": 20, "cacheRead": 10 } + }); + let entry = parse_pi_assistant_entry(&raw, PiMappedProvider::Codex).unwrap(); + assert_eq!(entry.input, 100); + assert_eq!(entry.output, 20); + assert_eq!(entry.cache_read, 10); + assert_eq!(entry.model, "gpt-5"); + assert!(entry.pricing_known); +} + +#[test] +fn parses_astra_cache_write_and_prices_it() { + let raw = serde_json::json!({ + "id": "astra-msg-1", + "role": "assistant", + "provider": "openai-codex", + "model": "gpt-6-astra", + "usage": { + "input": 1_000, + "output": 100, + "cacheRead": 200, + "cacheWrite": 300 + } + }); + let entry = parse_pi_assistant_entry(&raw, PiMappedProvider::Codex).unwrap(); + let expected = 500.0 * 1e-5 + 200.0 * 1e-6 + 300.0 * 1.25e-5 + 100.0 * 5e-5; + assert_eq!(entry.cache_create, 300); + assert!((entry.cost - expected).abs() < 1e-12); + assert!(entry.pricing_known); +} + +#[test] +fn prices_codex_rows_at_their_utc_timestamp() { + let cost = |timestamp: Option<&str>| { + let mut raw = serde_json::json!({ + "id": "sol-1", "role": "assistant", "provider": "openai-codex", + "model": "gpt-5.6-sol", + "usage": { "input": 100, "output": 5, "cacheRead": 10, "cacheWrite": 20 } + }); + if let Some(timestamp) = timestamp { + raw["timestamp"] = timestamp.into(); + } + parse_pi_assistant_entry(&raw, PiMappedProvider::Codex) + .unwrap() + .cost + }; + let historical = 70.0 * 5e-6 + 10.0 * 5e-7 + 20.0 * 6.25e-6 + 5.0 * 3e-5; + let current = 70.0 * 4e-6 + 10.0 * 4e-7 + 20.0 * 5e-6 + 5.0 * 2e-5; + for (timestamp, expected) in [ + (Some("2026-07-10T12:00:00Z"), historical), + (Some("2026-08-20T23:59:59Z"), historical), + (Some("2026-08-21T00:00:00Z"), current), + (Some("2026-09-10T12:00:00Z"), current), + (None, current), + ] { + assert!((cost(timestamp) - expected).abs() < 1e-12, "{timestamp:?}"); + } +} + +#[test] +fn unknown_model_keeps_tokens_and_marks_pricing_incomplete() { + let raw = serde_json::json!({ + "id": "unknown-model-1", + "role": "assistant", + "provider": "openai-codex", + "model": "future-model-without-a-rate", + "usage": { "input": 11, "output": 3 } + }); + let entry = parse_pi_assistant_entry_any(&raw).unwrap(); + assert!(!entry.pricing_known); + assert_eq!(entry.cost, 0.0); + + let mut summary = CostSummary::default(); + apply_entry(&mut summary, &entry); + assert_eq!(summary.input_tokens, 11); + assert_eq!(summary.output_tokens, 3); + assert!( + summary + .unknown_models + .contains("future-model-without-a-rate") + ); + assert!(summary.model_pricing_completeness.is_partial()); +} + +#[test] +fn standalone_parser_keeps_the_mapped_provider_for_mixed_history() { + let codex = serde_json::json!({ + "id": "codex-1", + "role": "assistant", + "provider": "openai-codex", + "model": "gpt-5", + "usage": { "input": 100, "output": 20 } + }); + let claude = serde_json::json!({ + "id": "claude-1", + "role": "assistant", + "provider": "anthropic", + "model": "claude-sonnet-4-6", + "usage": { "input": 100, "output": 20 } + }); + + assert_eq!( + parse_pi_assistant_entry_any(&codex) + .expect("Codex row should parse") + .provider, + PiMappedProvider::Codex + ); + assert_eq!( + parse_pi_assistant_entry_any(&claude) + .expect("Claude row should parse") + .provider, + PiMappedProvider::Claude + ); +} + +#[test] +fn dedupes_shared_entry_ids_across_files() { + let dir = tempdir().unwrap(); + let sessions = dir.path().join("agent").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let header = r#"{"type":"session","id":"session-1"}"#; + let line = r#"{"id":"shared-1","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":50,"output":5}}"#; + for name in ["a.jsonl", "b.jsonl"] { + let mut f = File::create(sessions.join(name)).unwrap(); + writeln!(f, "{header}").unwrap(); + writeln!(f, "{line}").unwrap(); + } + + // Point home at temp so roots resolve under .omp + let home = dir.path().to_path_buf(); + // Manually walk the sessions we created via for_each + let mut seen = HashSet::new(); + let mut summary = CostSummary::default(); + let mut total = 0u32; + for name in ["a.jsonl", "b.jsonl"] { + total += for_each_pi_entry( + &sessions.join(name), + DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc), + Some(PiMappedProvider::Codex), + &mut seen, + |entry| apply_entry(&mut summary, &entry), + ) + .counted; + } + assert_eq!(seen.len(), 1); + assert_eq!(summary.input_tokens, 50); + assert_eq!(total, 1); // second file deduped + let _ = home; +} + +#[test] +fn same_message_id_in_distinct_sessions_is_not_deduped() { + let dir = tempdir().unwrap(); + let sessions = dir.path().join("agent").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let line = r#"{"id":"reused-message","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":50,"output":5}}"#; + for session in ["session-a", "session-b"] { + let mut f = File::create(sessions.join(format!("{session}.jsonl"))).unwrap(); + writeln!(f, "{{\"type\":\"session\",\"id\":\"{session}\"}}").unwrap(); + writeln!(f, "{line}").unwrap(); + } + + let cutoff = DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + let mut seen = HashSet::new(); + let mut summary = CostSummary::default(); + let mut total = 0u32; + for session in ["session-a", "session-b"] { + total += for_each_pi_entry( + &sessions.join(format!("{session}.jsonl")), + cutoff, + Some(PiMappedProvider::Codex), + &mut seen, + |entry| apply_entry(&mut summary, &entry), + ) + .counted; + } + assert_eq!(total, 2); + assert_eq!(summary.input_tokens, 100); +} + +#[test] +fn configured_roots_include_explicit_pi_and_omp_profile_sources() { + let home = tempdir().unwrap(); + let cwd = home.path().join("project"); + std::fs::create_dir_all(&cwd).unwrap(); + let mut environment = EnvMap::new(); + environment.insert( + "PI_CODING_AGENT_SESSION_DIR".to_string(), + "pi-sessions".to_string(), + ); + environment.insert("PI_CONFIG_DIR".to_string(), "config".to_string()); + environment.insert("OMP_PROFILE".to_string(), "work".to_string()); + + let roots = pi_compatible_session_roots_for(home.path(), &cwd, &environment); + let pi_root = cwd.join("pi-sessions"); + let omp_root = home + .path() + .join("config") + .join("profiles") + .join("work") + .join("agent") + .join("sessions"); + assert!(roots.iter().any(|root| root == &pi_root)); + assert!(roots.iter().any(|root| root == &omp_root)); + + let duplicate = concat!( + r#"{"type":"session","id":"session-1"}"#, + "\n", + r#"{"id":"shared","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":50,"output":5}}"#, + "\n" + ); + let unique = concat!( + r#"{"type":"session","id":"session-2"}"#, + "\n", + r#"{"id":"unique","role":"assistant","provider":"anthropic","model":"claude-sonnet-4-6","timestamp":"2026-07-20T13:00:00Z","usage":{"input":70,"output":7}}"#, + "\n" + ); + std::fs::create_dir_all(&pi_root).unwrap(); + std::fs::create_dir_all(&omp_root).unwrap(); + std::fs::write(pi_root.join("session.jsonl"), duplicate).unwrap(); + std::fs::write(omp_root.join("session.jsonl"), duplicate).unwrap(); + std::fs::write(omp_root.join("unique.jsonl"), unique).unwrap(); + + let scan = scan_pi_daily_from_roots( + DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc), + None, + roots, + ); + assert!(scan.history_coverage_established); + assert_eq!(scan.tokens.values().sum::(), 132); +} + +#[test] +fn malformed_usage_input_keeps_valid_rows_but_marks_source_incomplete() { + let dir = tempdir().unwrap(); + let sessions = dir.path().join("agent").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let valid = r#"{"id":"valid","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":11,"output":3}}"#; + std::fs::write( + sessions.join("mixed.jsonl"), + format!("{valid}\n{{\"role\":\"assistant\",\"usage\":\n"), + ) + .unwrap(); + + let scan = scan_pi_daily_from_roots( + DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc), + None, + vec![sessions.clone()], + ); + assert!(!scan.history_coverage_established); + assert_eq!(scan.tokens.values().sum::(), 14); + + let mut summary = CostSummary::default(); + let mut seen = HashSet::new(); + let evidence = scan_roots_into( + &mut summary, + Utc::now() - Duration::days(365), + None, + &mut seen, + vec![sessions], + None, + ); + assert!(!evidence.complete); + assert_eq!(summary.input_tokens, 11); +} + +#[test] +fn standalone_daily_scan_dedupes_pi_and_omp_roots() { + let dir = tempdir().unwrap(); + let pi_sessions = dir.path().join(".pi").join("agent").join("sessions"); + let omp_sessions = dir.path().join(".omp").join("agent").join("sessions"); + std::fs::create_dir_all(&pi_sessions).unwrap(); + std::fs::create_dir_all(&omp_sessions).unwrap(); + let codex = r#"{"id":"shared","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":50,"output":5}}"#; + let claude = r#"{"id":"claude-only","role":"assistant","provider":"anthropic","model":"claude-sonnet-4-6","timestamp":"2026-07-20T13:00:00Z","usage":{"input":70,"output":7}}"#; + std::fs::write( + pi_sessions.join("one.jsonl"), + format!("{codex}\n{claude}\n"), + ) + .unwrap(); + std::fs::write(omp_sessions.join("one.jsonl"), format!("{codex}\n")).unwrap(); + + let scan = scan_pi_daily_from_roots( + DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc), + None, + vec![pi_sessions, omp_sessions], + ); + assert!(scan.history_coverage_established); + assert_eq!(scan.tokens.values().sum::(), 132); + assert_eq!(scan.tokens.len(), 1); +} + +#[test] +fn daily_tokens_count_cache_like_the_window_total() { + let dir = tempdir().unwrap(); + let sessions = dir.path().join("agent").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let codex = r#"{"id":"cached-codex","role":"assistant","provider":"openai-codex","model":"gpt-5","timestamp":"2026-07-20T12:00:00Z","usage":{"input":100,"output":20,"cacheRead":10,"cacheWrite":5}}"#; + let claude = r#"{"id":"cached-claude","role":"assistant","provider":"anthropic","model":"claude-sonnet-4-6","timestamp":"2026-07-20T13:00:00Z","usage":{"input":40,"output":4,"cacheRead":300,"cacheWrite":30}}"#; + std::fs::write( + sessions.join("cached.jsonl"), + format!("{codex}\n{claude}\n"), + ) + .unwrap(); + let cutoff = DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + + let scan = scan_pi_daily_from_roots(cutoff, None, vec![sessions.clone()]); + assert!(scan.history_coverage_established); + let daily_total = scan.tokens.values().sum::(); + assert_eq!(daily_total, 135 + 374); + + let mut summary = CostSummary::default(); + let mut seen = HashSet::new(); + let evidence = scan_roots_into(&mut summary, cutoff, None, &mut seen, vec![sessions], None); + assert!(evidence.complete); + assert_eq!(summary.total_tokens_for_provider("pi"), daily_total); +} + +#[test] +fn session_roots_include_pi_and_omp() { + let home = PathBuf::from("/home/user"); + let roots = pi_compatible_session_roots_for(&home, &home, &EnvMap::new()); + assert!( + roots + .iter() + .any(|p| p.ends_with(".pi/agent/sessions") || p.ends_with(".pi\\agent\\sessions")) + ); + assert!( + roots + .iter() + .any(|p| p.ends_with(".omp/agent/sessions") || p.ends_with(".omp\\agent\\sessions")) + ); +} From 8c8980050768f6a9bbe7c0f746d17b07c181f4fe Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:56:32 +0700 Subject: [PATCH 35/41] Split the cost scanner tests by topic --- rust/src/cost_scanner/tests.rs | 3462 +---------------- rust/src/cost_scanner/tests/claude_daily.rs | 511 +++ rust/src/cost_scanner/tests/claude_rates.rs | 232 ++ rust/src/cost_scanner/tests/claude_records.rs | 280 ++ rust/src/cost_scanner/tests/codex_bounded.rs | 361 ++ rust/src/cost_scanner/tests/codex_cache.rs | 707 ++++ rust/src/cost_scanner/tests/codex_fork.rs | 341 ++ rust/src/cost_scanner/tests/codex_pending.rs | 445 +++ rust/src/cost_scanner/tests/codex_usage.rs | 329 ++ rust/src/cost_scanner/tests/support.rs | 175 + rust/src/cost_scanner/tests/totals.rs | 88 + 11 files changed, 3499 insertions(+), 3432 deletions(-) create mode 100644 rust/src/cost_scanner/tests/claude_daily.rs create mode 100644 rust/src/cost_scanner/tests/claude_rates.rs create mode 100644 rust/src/cost_scanner/tests/claude_records.rs create mode 100644 rust/src/cost_scanner/tests/codex_bounded.rs create mode 100644 rust/src/cost_scanner/tests/codex_cache.rs create mode 100644 rust/src/cost_scanner/tests/codex_fork.rs create mode 100644 rust/src/cost_scanner/tests/codex_pending.rs create mode 100644 rust/src/cost_scanner/tests/codex_usage.rs create mode 100644 rust/src/cost_scanner/tests/totals.rs diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 4b134a7265..1915000848 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -5,3389 +5,12 @@ use crate::core::{CodexSessionLineage, CostUsagePricing}; use chrono::{FixedOffset, Local, NaiveTime, TimeZone}; use claude_pricing::FALLBACK_CLAUDE_MODEL; use std::io::Write; -use support::{app_scanner, cached_file, codex_scan_dirs, partition_dir}; - -/// Scanner Claude pricing without the per-scan memo, as an oracle. -struct ClaudePricing; - -impl ClaudePricing { - fn cost_usd_with_cache_ttl( - model: &str, - input: u64, - cache_create: u64, - cache_create_1h: u64, - cache_read: u64, - output: u64, - ) -> f64 { - let cache_create_1h = cache_create_1h.min(cache_create); - let cache_create_5m = cache_create.saturating_sub(cache_create_1h); - - // Standard buckets (input, cache-read, 5-minute cache-write, output), - // including any long-context tiering, come from the canonical table. - // Unknown/retired models fall back to Sonnet pricing. - #[allow( - clippy::cast_possible_truncation, - reason = "clamped to i32::MAX before casting" - )] - let clamp = |v: u64| v.min(i32::MAX as u64) as i32; - let base = CostUsagePricing::claude_cost_usd( - model, - clamp(input), - clamp(cache_read), - clamp(cache_create_5m), - clamp(output), - ) - .or_else(|| { - CostUsagePricing::claude_cost_usd( - FALLBACK_CLAUDE_MODEL, - clamp(input), - clamp(cache_read), - clamp(cache_create_5m), - clamp(output), - ) - }) - .unwrap_or(0.0); - - // Scanner-specific: one-hour cache writes bill at 2x the input rate. - let input_rate = CostUsagePricing::claude_input_cost_per_token(model) - .or_else(|| CostUsagePricing::claude_input_cost_per_token(FALLBACK_CLAUDE_MODEL)) - .unwrap_or(0.0); - - base + (cache_create_1h as f64) * input_rate * 2.0 - } -} - -/// Stream the de-duplicated, in-window usage records from one transcript -/// file into `on_record`, returning how many it consumed. -fn for_each_claude_usage_record( - path: &Path, - cutoff: &DateTime, - seen: &mut HashSet, - cancel: Option<&AtomicBool>, - on_record: F, -) -> usize -where - F: FnMut(&ClaudeUsageRecord), -{ - let mut pricing = ClaudeScanPricingResolver::default(); - let mut incomplete = ClaudeIncompleteTracker::default(); - scan_claude_file_with_pricing( - path, - cutoff, - seen, - cancel, - &mut pricing, - &mut incomplete, - on_record, - ) - .counted -} - -fn claude_usage_record_from_event(event: &ClaudeEvent) -> Option { - let mut pricing = ClaudeScanPricingResolver::default(); - claude_usage_record_from_event_with_pricing(event, &mut pricing) -} - -#[test] -fn test_unknown_model_falls_back_to_sonnet() { - // Unknown/retired Claude IDs fall back to Sonnet 4.6 base pricing - // ($3/1M input, $15/1M output). 100k tokens stay under the 200k tier. - let cost = - ClaudePricing::cost_usd_with_cache_ttl("claude-3-5-sonnet", 100_000, 0, 0, 0, 100_000); - // 100k * $3/M + 100k * $15/M = 0.30 + 1.50 = 1.80 - assert!((cost - 1.80).abs() < 0.001); -} - -#[test] -fn records_unknown_claude_model_while_using_fallback_cost() { - let event: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","timestamp":"2026-01-15T10:00:00Z","requestId":"req_unknown","message":{"id":"msg_unknown","model":"claude-retired-unknown","usage":{"input_tokens":100000,"output_tokens":100000}}}"#, - ) - .unwrap(); - let record = claude_usage_record_from_event(&event).expect("usage record"); - let mut summary = CostSummary::default(); - - add_claude_record_to_summary(&mut summary, &record); - - assert!(summary.total_cost_usd > 0.0); - assert!(summary.unknown_models.contains("claude-retired-unknown")); -} - -#[test] -fn claude_scan_pricing_resolver_reuses_positive_and_negative_resolution() { - let unknown = format!("claude-scan-unknown-{}", std::process::id()); - let mut resolver = ClaudeScanPricingResolver::default(); - - assert!(resolver.is_known("claude-sonnet-4-6")); - assert!(!resolver.is_known(&unknown)); - assert!(resolver.is_known("claude-sonnet-4-6")); - assert!(!resolver.is_known(&unknown)); - assert_eq!(resolver.normalization_cache_misses, 2); - assert_eq!(resolver.resolution_cache_misses, 2); - assert_eq!(resolver.resolutions.len(), 2); - - let mut cost_resolver = ClaudeScanPricingResolver::default(); - let resolved_unknown = cost_resolver.cost_usd_with_cache_ttl(&unknown, 100, 20, 10, 30, 40); - let fallback = - ClaudePricing::cost_usd_with_cache_ttl(FALLBACK_CLAUDE_MODEL, 100, 20, 10, 30, 40); - assert!((resolved_unknown - fallback).abs() < f64::EPSILON); -} - -#[test] -fn claude_scan_pricing_resolver_preserves_tiered_and_cache_ttl_pricing() { - let mut resolver = ClaudeScanPricingResolver::default(); - let cases = [ - ("claude-sonnet-4-6", 240_000, 0, 0, 0, 0), - ("claude-fable-5", 100, 30, 20, 20, 5), - ]; - - for (model, input, cache_create, cache_create_1h, cache_read, output) in cases { - let actual = resolver.cost_usd_with_cache_ttl( - model, - input, - cache_create, - cache_create_1h, - cache_read, - output, - ); - let expected = ClaudePricing::cost_usd_with_cache_ttl( - model, - input, - cache_create, - cache_create_1h, - cache_read, - output, - ); - assert!((actual - expected).abs() < f64::EPSILON, "{model}"); - } -} - -#[test] -fn claude_scan_resolver_applies_gpt_proxy_long_context_boundary() { - let snapshot = crate::core::ModelsDevPricingSnapshot::from_catalog_json_for_tests( - r#"{ - "openai": {"models": {"gpt-5.6-sol": {"id": "gpt-5.6-sol", "cost": { - "input": 2, "output": 4, "cache_read": 0.25, "cache_write": 3, - "context_over_200k": {"input": 7, "output": 11, "cache_read": 0.5, "cache_write": 9} - }}}} - }"#, - ) - .expect("pricing fixture"); - let mut resolver = ClaudeScanPricingResolver::with_snapshot(snapshot); - - let short = resolver.cost_usd_with_cache_ttl("gpt-5.6-sol", 262_000, 0, 0, 10_000, 13); - let long = resolver.cost_usd_with_cache_ttl("gpt-5.6-sol", 262_001, 0, 0, 10_000, 13); - - assert!((short - 0.526552).abs() < 1e-12); - assert!((long - 1.83915).abs() < 1e-12); -} - -#[test] -fn claude_scan_pricing_resolver_bounds_normalization_memo() { - let mut resolver = ClaudeScanPricingResolver::default(); - for index in 0..(ClaudeScanPricingResolver::MEMO_ENTRY_LIMIT + 8) { - let model = format!("claude-memo-{index}"); - assert_eq!(resolver.normalize(&model), model); - } - assert_eq!( - resolver.normalized_models.len(), - ClaudeScanPricingResolver::MEMO_ENTRY_LIMIT - ); - - let misses = resolver.normalization_cache_misses; - assert_eq!(resolver.normalize("claude-memo-0"), "claude-memo-0"); - assert_eq!(resolver.normalization_cache_misses, misses); - assert_eq!(resolver.normalize("claude-memo-1024"), "claude-memo-1024"); - assert_eq!(resolver.normalization_cache_misses, misses + 1); -} - -#[test] -fn claude_fable_5_prices_five_minute_and_one_hour_cache_writes() { - // (input, cache write, 1h share of it, cache read, output, expected USD). - for (input, cache_create, cache_create_1h, cache_read, output, expected) in [ - ( - 100, - 10, - 0, - 20, - 5, - (100.0 / 1_000_000.0) * 10.00 - + (10.0 / 1_000_000.0) * 12.50 - + (20.0 / 1_000_000.0) * 1.00 - + (5.0 / 1_000_000.0) * 50.00, - ), - ( - 100, - 30, - 20, - 20, - 5, - (100.0 / 1_000_000.0) * 10.00 - + (10.0 / 1_000_000.0) * 12.50 - + (20.0 / 1_000_000.0) * 20.00 - + (20.0 / 1_000_000.0) * 1.00 - + (5.0 / 1_000_000.0) * 50.00, - ), - ] { - let cost = ClaudePricing::cost_usd_with_cache_ttl( - "claude-fable-5", - input, - cache_create, - cache_create_1h, - cache_read, - output, - ); - assert!((cost - expected).abs() < f64::EPSILON, "{cache_create_1h}"); - } -} - -#[test] -fn claude_scan_pricing_follows_the_canonical_table() { - // Sonnet 4.6 honors the 200k tier: 200k @ $3/M + 40k @ $6/M = 0.84 (the - // scanner's old inline table applied a flat $3/M = 0.72). Opus 4.5-4.8 - // bill $5 in + $25 out; opus-4-8 must resolve through the canonical table. - // Legacy Opus 4.0/4.1 stay at $15 + $75 (retired IDs absent from the - // table, e.g. `claude-3-opus-...`, fall back to Sonnet). Haiku 4.5 bills - // $1 + $5, not the Haiku 3 rate. - for (model, input, output, expected) in [ - ("claude-sonnet-4-6", 240_000, 0, 0.84), - ("claude-opus-4-5", 1_000_000, 1_000_000, 30.00), - ("claude-opus-4-6", 1_000_000, 1_000_000, 30.00), - ("claude-opus-4-7", 1_000_000, 1_000_000, 30.00), - ("claude-opus-4-8", 1_000_000, 1_000_000, 30.00), - ("claude-opus-4-20250514", 1_000_000, 1_000_000, 90.00), - ("claude-opus-4-1", 1_000_000, 1_000_000, 90.00), - ("claude-haiku-4-5", 1_000_000, 1_000_000, 6.00), - ] { - let cost = ClaudePricing::cost_usd_with_cache_ttl(model, input, 0, 0, 0, output); - assert!( - (cost - expected).abs() < 0.001, - "{model} should bill ${expected}, got {cost}" - ); - } -} - -#[test] -fn parses_current_codex_payload_token_count_events() { - let path = std::env::temp_dir().join(format!( - "codexbar-current-codex-token-count-{}.jsonl", - std::process::id() - )); - // Use a recent timestamp so the event stays inside the scanner's - // 30-day window no matter when the test runs. A hardcoded date - // silently ages out of the window and makes this test fail with 0 - // sessions once it is more than 30 days in the past. - let recent = (Utc::now() - Duration::hours(1)) - .format("%Y-%m-%dT%H:%M:%S%.3fZ") - .to_string(); - let mut file = File::create(&path).unwrap(); - writeln!( - file, - r#"{{"timestamp":"{ts}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":125,"cached_input_tokens":30,"output_tokens":15}}}}}}}}"#, - ts = recent - ) - .unwrap(); - let scanner = CostScanner::new(30); - let mut summary = CostSummary::default(); - let today = Local::now().date_naive(); - let range = CostUsageDayRange::new(codex_period_start(today, 30), today); - let mut cache = CostUsageCache::default(); - let mut stats = CostScanStats::default(); - scanner.parse_codex_file(&path, &range, &mut summary, &mut cache, None, &mut stats); - - assert_eq!(summary.sessions_count, 1); - assert_eq!(summary.input_tokens, 125); - assert_eq!(summary.cached_tokens, 30); - assert_eq!(summary.output_tokens, 15); - assert_eq!( - summary - .by_model_tokens - .get("gpt-5") - .map(ModelTokenCounts::total), - Some(140) - ); - assert!(scan_codex_file_cost(&path) > 0.0); - // Best-effort test cleanup; the file may already be gone. - let _removed = std::fs::remove_file(&path); -} - -#[test] -fn scans_gpt6_astra_usage_with_cached_and_reasoning_tokens() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let today = Local::now().date_naive(); - let day = today.format("%Y-%m-%d").to_string(); - let day_dir = partition_dir(&sessions, today); - std::fs::create_dir_all(&day_dir).unwrap(); - let line = serde_json::json!({ - "timestamp": Local::now().to_rfc3339(), - "type": "event_msg", - "payload": { - "type": "token_count", - "info": { - "model": "gpt-6-astra", - "total_token_usage": { - "input_tokens": 1000, - "cached_input_tokens": 300, - "output_tokens": 100, - "reasoning_output_tokens": 7 - } - } - } - }); - std::fs::write(day_dir.join("astra.jsonl"), format!("{line}\n")).unwrap(); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - - assert_eq!(summary.input_tokens, 1000); - assert_eq!(summary.cached_tokens, 300); - assert_eq!(summary.output_tokens, 100); - assert_eq!(summary.reasoning_tokens, Some(7)); - // Codex token-count rows expose cache reads, not cache writes. The 700 - // non-cached input tokens therefore use Astra's standard input rate. - assert!((summary.total_cost_usd - 0.0123).abs() < 1e-12); - assert_eq!(cache.days[&day]["gpt-6-astra"], vec![1000, 300, 100, 7]); -} - -#[test] -fn derives_claude_dedup_key_from_message_and_request_ids() { - assert_eq!( - claude_usage_dedup_key(Some("msg_1"), Some("req_1"), None), - Some(ClaudeUsageDedupKey::Request { - message_id: Some("msg_1".to_string()), - request_id: "req_1".to_string(), - }) - ); - assert_eq!( - claude_usage_dedup_key(None, Some(" req_1 "), None), - Some(ClaudeUsageDedupKey::Request { - message_id: None, - request_id: "req_1".to_string(), - }) - ); - assert_eq!( - claude_usage_dedup_key(Some("msg_1"), None, Some("session_1")), - Some(ClaudeUsageDedupKey::Session { - session_id: "session_1".to_string(), - message_id: "msg_1".to_string(), - }) - ); - assert_eq!(claude_usage_dedup_key(Some("msg_1"), None, None), None); - assert_eq!( - claude_usage_dedup_key(None, Some("req_1"), Some("session_1")), - Some(ClaudeUsageDedupKey::Request { - message_id: None, - request_id: "req_1".to_string(), - }) - ); - assert_eq!( - claude_usage_dedup_key(Some("msg_1"), Some(" "), Some("session_1")), - Some(ClaudeUsageDedupKey::Session { - session_id: "session_1".to_string(), - message_id: "msg_1".to_string(), - }) - ); - assert_eq!( - claude_usage_dedup_key(Some(" "), None, Some("session_1")), - None - ); - assert_eq!(claude_usage_dedup_key(Some("msg_1"), None, Some(" ")), None); -} - -#[test] -fn session_id_falls_back_from_blank_direct_id_to_metadata() { - let event: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","sessionId":" ","metadata":{"session_id":" metadata-session "},"message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":10}}}"#, - ) - .unwrap(); - - assert_eq!(event.session_id(), Some("metadata-session")); -} - -#[test] -fn session_id_falls_back_from_blank_direct_and_metadata_ids_to_nested_metadata() { - let event: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","sessionId":" ","metadata":{"sessionId":"\t","metadata":{"session_id":" nested-session "}},"message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":10}}}"#, - ) - .unwrap(); - - assert_eq!(event.session_id(), Some("nested-session")); -} - -#[test] -fn session_aware_claude_dedup_keeps_distinct_sessions_separate() { - let first: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","sessionId":"session_a","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":10}}}"#, - ) - .unwrap(); - let second: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","sessionId":"session_b","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":10}}}"#, - ) - .unwrap(); - let first_record = claude_usage_record_from_event(&first).expect("first usage record"); - let second_record = claude_usage_record_from_event(&second).expect("second usage record"); - let cutoff = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc); - let mut seen = HashSet::new(); - - assert!(should_count_claude_record( - &first_record, - &cutoff, - &mut seen - )); - assert!(should_count_claude_record( - &second_record, - &cutoff, - &mut seen - )); - assert!(!should_count_claude_record( - &first_record, - &cutoff, - &mut seen - )); -} - -#[test] -fn counts_claude_usage_once_across_duplicate_records() { - // The same API response can be replayed into several transcript files - // (session resume, sidechains); it must only be counted once. - let event: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","timestamp":"2026-01-15T10:00:00Z","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":100,"output_tokens":50,"cache_creation_input_tokens":10,"cache_read_input_tokens":20}}}"#, - ) - .unwrap(); - - let record = claude_usage_record_from_event(&event).expect("usage record"); - assert_eq!(record.model, "claude-sonnet-4-6"); - assert_eq!(record.input, 100); - assert_eq!(record.output, 50); - assert_eq!(record.cache_create, 10); - assert_eq!(record.cache_read, 20); - assert!(record.cost.is_some_and(|cost| cost > 0.0)); - - let cutoff = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc); - let mut seen = HashSet::new(); - assert!(should_count_claude_record(&record, &cutoff, &mut seen)); - assert!(!should_count_claude_record(&record, &cutoff, &mut seen)); -} - -#[test] -fn rejects_claude_records_before_cutoff() { - let event: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","timestamp":"2025-12-01T10:00:00Z","requestId":"req_old","message":{"id":"msg_old","model":"claude-sonnet-4-6","usage":{"input_tokens":1,"output_tokens":1}}}"#, - ) - .unwrap(); - let record = claude_usage_record_from_event(&event).expect("usage record"); - let cutoff = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc); - let mut seen = HashSet::new(); - assert!(!should_count_claude_record(&record, &cutoff, &mut seen)); -} - -#[test] -fn ignores_claude_events_without_countable_usage() { - // Non-assistant events carry no billable usage. - let event: ClaudeEvent = - serde_json::from_str(r#"{"type":"user","message":{"usage":{"input_tokens":5}}}"#).unwrap(); - assert!(claude_usage_record_from_event(&event).is_none()); - - // Zero-token usage blocks (e.g. synthetic messages) are not sessions. - let event: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","message":{"id":"msg_zero","model":"claude-sonnet-4-6","usage":{"input_tokens":0,"output_tokens":0}}}"#, - ) - .unwrap(); - assert!(claude_usage_record_from_event(&event).is_none()); -} - -#[test] -fn excludes_preliminary_proxy_estimates_but_keeps_cache_aware_rows() { - let preliminary: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","message":{"id":"msg_preliminary","model":"gpt-5.6-sol","stop_reason":null,"usage":{"input_tokens":1000}}}"#, - ) - .unwrap(); - assert!(claude_usage_record_from_event(&preliminary).is_none()); - - let completed: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","message":{"id":"msg_completed","model":"gpt-5.6-sol","stop_reason":"end_turn","usage":{"input_tokens":1000}}}"#, - ) - .unwrap(); - assert!(claude_usage_record_from_event(&completed).is_some()); - - let cache_aware: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","message":{"id":"msg_cache_aware","model":"gpt-5.6-sol","stop_reason":null,"usage":{"input_tokens":1000,"cache_read_input_tokens":1}}}"#, - ) - .unwrap(); - assert!(claude_usage_record_from_event(&cache_aware).is_some()); -} - -#[test] -fn claude_scan_counts_unreconciled_incomplete_requests_per_day_and_model() { - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("transcript.jsonl"); - let now = Utc::now() - Duration::hours(1); - let ts = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - let day = cost_bucket_zone().date(now).format("%Y-%m-%d").to_string(); - let row = |request: &str, message: &str, stop: &str, usage: &str| { - format!( - r#"{{"type":"assistant","timestamp":"{ts}","requestId":"{request}","message":{{"id":"{message}","model":"claude-sonnet-4-6","stop_reason":{stop},"usage":{usage}}}}}"# - ) - }; - let body = [ - // Superseded: a completed row with the same key exists. - row("req_done", "msg_done", "null", r#"{"input_tokens":1000}"#), - row( - "req_done", - "msg_done", - r#""end_turn""#, - r#"{"input_tokens":1000,"output_tokens":50}"#, - ), - // Never completed, duplicated preliminary rows count once. - row("req_open", "msg_open", "null", r#"{"input_tokens":500}"#), - row("req_open", "msg_open", "null", r#"{"input_tokens":500}"#), - ] - .join("\n"); - std::fs::write(&path, body).unwrap(); - - let cutoff = Utc::now() - Duration::days(1); - let mut seen = HashSet::new(); - let mut pricing = ClaudeScanPricingResolver::default(); - let mut tracker = ClaudeIncompleteTracker::default(); - let mut summary = CostSummary::default(); - let result = scan_claude_file_with_pricing( - &path, - &cutoff, - &mut seen, - None, - &mut pricing, - &mut tracker, - |record| assert!(add_claude_record_to_summary(&mut summary, record)), - ); - assert_eq!(result.counted, 1); - assert!( - !result.is_complete(), - "preliminary rows keep coverage unknown" - ); - - let report = tracker.resolve(&seen); - report.apply_to(&mut summary); - assert_eq!(summary.incomplete_request_count, 1); - assert_eq!( - summary.incomplete_by_model.get("claude-sonnet-4-6"), - Some(&1) - ); - assert_eq!(report.by_day.get(&day), Some(&1)); - // Only the completed row contributes tokens. - assert_eq!(summary.input_tokens, 1000); - assert_eq!(summary.output_tokens, 50); -} - -#[test] -fn malformed_claude_history_stays_unknown_while_valid_empty_history_is_known_zero() { - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("transcript.jsonl"); - let cutoff = Utc::now() - Duration::days(1); - - std::fs::write(&path, b"\n").unwrap(); - let mut empty_seen = HashSet::new(); - let mut empty_pricing = ClaudeScanPricingResolver::default(); - let empty_result = scan_claude_file_with_pricing( - &path, - &cutoff, - &mut empty_seen, - None, - &mut empty_pricing, - &mut ClaudeIncompleteTracker::default(), - |_| {}, - ); - let mut empty_summary = CostSummary::default(); - finalize_claude_summary(&mut empty_summary, true, empty_result, false); - assert!(empty_summary.history_coverage_established); - assert!(empty_summary.known_zero); - - std::fs::write(&path, b"{malformed\n").unwrap(); - let mut malformed_seen = HashSet::new(); - let mut malformed_pricing = ClaudeScanPricingResolver::default(); - let malformed_result = scan_claude_file_with_pricing( - &path, - &cutoff, - &mut malformed_seen, - None, - &mut malformed_pricing, - &mut ClaudeIncompleteTracker::default(), - |_| {}, - ); - assert_eq!(malformed_result.malformed_lines, 1); - let mut malformed_summary = CostSummary::default(); - finalize_claude_summary(&mut malformed_summary, true, malformed_result, false); - assert!(!malformed_summary.history_coverage_established); - assert!(!malformed_summary.known_zero); -} - -/// One file's daily token scan, counting each record that cannot be -/// aggregated as the token history path does. -fn scan_claude_file_for_daily_tokens( - path: &Path, - cutoff: &DateTime, - seen: &mut HashSet, - pricing: &mut ClaudeScanPricingResolver, - daily_tokens: &mut HashMap, -) -> ClaudeFileScanResult { - let mut aggregation_failures = 0u32; - // Token history ignores incomplete-request markers. - let mut incomplete = ClaudeIncompleteTracker::default(); - let mut result = scan_claude_file_with_pricing( - path, - cutoff, - seen, - None, - pricing, - &mut incomplete, - |record| { - if record.timestamp.is_none() - || !add_claude_record_to_daily_tokens(daily_tokens, record) - { - aggregation_failures = aggregation_failures.saturating_add(1); - } - }, - ); - result.aggregation_failures = result - .aggregation_failures - .saturating_add(aggregation_failures); - result -} - -#[test] -fn claude_daily_token_coverage_requires_a_complete_valid_scan() { - let root = tempfile::tempdir().unwrap(); - let cutoff = Utc::now() - Duration::days(1); - let valid_path = root.path().join("valid.jsonl"); - let timestamp = Utc::now() - Duration::hours(1); - let today = timestamp - .with_timezone(&Local) - .date_naive() - .format("%Y-%m-%d") - .to_string(); - std::fs::write( - &valid_path, - format!( - "{}\n", - claude_transcript_line( - ×tamp.to_rfc3339(), - "requestId", - "req_valid", - "msg_valid" - ) - ), - ) - .unwrap(); - - let mut valid_tokens = HashMap::from([(today.clone(), 0)]); - let valid_result = scan_claude_file_for_daily_tokens( - &valid_path, - &cutoff, - &mut HashSet::new(), - &mut ClaudeScanPricingResolver::default(), - &mut valid_tokens, - ); - assert!(valid_result.is_complete()); - let mut covered_days = HashSet::new(); - mark_claude_daily_token_coverage(&mut covered_days, &valid_tokens, valid_result); - assert!(covered_days.contains(&today)); - - let assert_uncovered = |path: &Path| { - let mut daily_tokens = HashMap::from([(today.clone(), 0)]); - let result = scan_claude_file_for_daily_tokens( - path, - &cutoff, - &mut HashSet::new(), - &mut ClaudeScanPricingResolver::default(), - &mut daily_tokens, - ); - assert!(!result.is_complete()); - let mut covered_days = HashSet::from(["stale-coverage".to_string()]); - mark_claude_daily_token_coverage(&mut covered_days, &daily_tokens, result); - assert!(covered_days.is_empty()); - result - }; - - let malformed_path = root.path().join("malformed.jsonl"); - std::fs::write(&malformed_path, b"{malformed\n").unwrap(); - assert_eq!(assert_uncovered(&malformed_path).malformed_lines, 1); - - let incomplete_path = root.path().join("incomplete.jsonl"); - std::fs::write( - &incomplete_path, - r#"{"type":"assistant","message":{"id":"msg_preliminary","model":"gpt-5.6-sol","stop_reason":null,"usage":{"input_tokens":1000}}}"#, - ) - .unwrap(); - assert_eq!(assert_uncovered(&incomplete_path).incomplete_requests, 1); - - let missing_timestamp_path = root.path().join("missing-timestamp.jsonl"); - std::fs::write( - &missing_timestamp_path, - r#"{"type":"assistant","requestId":"req_no_timestamp","message":{"id":"msg_no_timestamp","model":"claude-sonnet-4-6","usage":{"input_tokens":1000,"output_tokens":500}}}"#, - ) - .unwrap(); - assert_eq!( - assert_uncovered(&missing_timestamp_path).aggregation_failures, - 1 - ); - - let unreadable_path = root.path().join("missing.jsonl"); - assert_eq!(assert_uncovered(&unreadable_path).read_failures, 1); - - let scanner = CostScanner::new(1); - let missing_directory = root.path().join("missing-directory"); - let traversal_read_failures = - scanner.walk_claude_files(&missing_directory, &cutoff, None, &mut |_| {}); - assert_eq!(traversal_read_failures, 1); - let mut covered_days = HashSet::from([today]); - mark_claude_daily_token_coverage( - &mut covered_days, - &valid_tokens, - ClaudeFileScanResult { - read_failures: traversal_read_failures, - ..ClaudeFileScanResult::default() - }, - ); - assert!(covered_days.is_empty()); -} - -#[test] -fn public_claude_daily_token_dispatch_reports_incomplete_fixture_scans() { - const CHILD_MARKER: &str = "CODEXBAR_CLAUDE_DAILY_TOKEN_TEST_CHILD"; - const CHILD_DONE: &str = "isolated Claude daily-history fixture verified"; - if std::env::var_os(CHILD_MARKER).is_some() { - let config_dir = std::env::var_os("CLAUDE_CONFIG_DIR") - .map(PathBuf::from) - .expect("child receives isolated Claude config directory"); - let projects_dir = config_dir.join("projects"); - let project_dir = projects_dir.join("fixture-project"); - let (complete_history, incomplete) = get_daily_token_history("claude", 1); - assert!(!incomplete, "valid fixture scan should establish coverage"); - assert!(complete_history.iter().any(|(_, tokens)| *tokens > 0)); - - std::fs::write(project_dir.join("malformed.jsonl"), b"{malformed\n").unwrap(); - let (partial_history, incomplete) = get_daily_token_history("claude", 1); - assert!( - incomplete, - "malformed fixture should leave coverage incomplete" - ); - assert_eq!(partial_history, complete_history); - println!("{CHILD_DONE}"); - return; - } - - let config_dir = tempfile::tempdir().unwrap(); - let project_dir = config_dir.path().join("projects").join("fixture-project"); - std::fs::create_dir_all(&project_dir).unwrap(); - let timestamp = Utc::now().to_rfc3339(); - std::fs::write( - project_dir.join("valid.jsonl"), - format!( - "{}\n", - claude_transcript_line(×tamp, "requestId", "req_public", "msg_public") - ), - ) - .unwrap(); - - let test_thread = std::thread::current(); - let test_name = test_thread.name().expect("test harness names this thread"); - let output = std::process::Command::new(std::env::current_exe().unwrap()) - .args(["--exact", test_name, "--nocapture", "--test-threads=1"]) - .env(CHILD_MARKER, "1") - .env("CLAUDE_CONFIG_DIR", config_dir.path()) - .output() - .expect("spawn isolated exact-test child"); - assert!( - output.status.success() && String::from_utf8_lossy(&output.stdout).contains(CHILD_DONE), - "fixture child failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); -} - -#[test] -fn classifies_vertex_ai_claude_metadata_without_changing_anthropic_rows() { - let cases = [ - ( - r#"{"type":"assistant","requestId":"req_vrtx_123","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":1}}}"#, - true, - ), - ( - r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_vrtx_123","model":"claude-sonnet-4-6","usage":{"input_tokens":1}}}"#, - true, - ), - ( - r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6@20260217","usage":{"input_tokens":1}}}"#, - true, - ), - ( - r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","metadata":{"provider":"Google-Vertex-AI"},"usage":{"input_tokens":1}}}"#, - true, - ), - ( - r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","content":[{"context":{"gcp_project":false}}],"usage":{"input_tokens":1}}}"#, - true, - ), - ( - r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":1}} ,"metadata":{"provider":"anthropic"}}"#, - false, - ), - ( - r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":1}} ,"metadata":{"provider":"gcp"}}"#, - false, - ), - ( - r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","content":[{"text":"vertex"}],"usage":{"input_tokens":1}}}"#, - false, - ), - ( - r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"Claude-sonnet-4-6@20260217","usage":{"input_tokens":1}}}"#, - false, - ), - ]; - - for (json, expected) in cases { - let event: ClaudeEvent = serde_json::from_str(json).unwrap(); - assert_eq!(event.is_vertex_ai_usage_entry(), expected, "{json}"); - } -} - -#[test] -fn shared_claude_reader_excludes_vertex_rows_but_keeps_anthropic_usage() { - let path = std::env::temp_dir().join(format!( - "codexbar-claude-vertex-filter-{}.jsonl", - std::process::id() - )); - let timestamp = (Utc::now() - Duration::hours(1)).to_rfc3339(); - let anthropic = format!( - r#"{{"type":"assistant","timestamp":"{timestamp}","requestId":"req_anthropic","message":{{"id":"msg_anthropic","model":"claude-sonnet-4-6","usage":{{"input_tokens":10,"output_tokens":5}}}}}}"# - ); - let vertex = serde_json::json!({ - "type": "assistant", - "timestamp": timestamp, - "requestId": "req_vrtx_123", - "message": { - "id": "msg_vrtx_123", - "model": "claude-sonnet-4-6", - "usage": {"input_tokens": u64::MAX, "output_tokens": u64::MAX} - } - }) - .to_string(); - std::fs::write(&path, format!("{anthropic}\n{vertex}\n")).unwrap(); - - let cutoff = Utc::now() - Duration::days(30); - let mut seen = HashSet::new(); - let mut records = Vec::new(); - let counted = for_each_claude_usage_record(&path, &cutoff, &mut seen, None, |record| { - records.push((record.input, record.output)) - }); - - assert_eq!(counted, 1); - assert_eq!(records, vec![(10, 5)]); - let _removed = std::fs::remove_file(&path); -} - -#[test] -fn oversized_claude_history_preserves_independent_components_and_fails_closed() { - let first: ClaudeEvent = serde_json::from_str(&format!( - r#"{{"type":"assistant","timestamp":"2026-09-20T12:00:00Z","requestId":"req_overflow_1","message":{{"id":"msg_overflow_1","model":"claude-sonnet-4-6","usage":{{"input_tokens":{},"output_tokens":2}}}}}}"#, - u64::MAX - )) - .unwrap(); - let second: ClaudeEvent = serde_json::from_str( - r#"{"type":"assistant","timestamp":"2026-09-20T12:01:00Z","requestId":"req_overflow_2","message":{"id":"msg_overflow_2","model":"claude-sonnet-4-6","usage":{"input_tokens":1,"output_tokens":3}}}"#, - ) - .unwrap(); - let first = claude_usage_record_from_event(&first).expect("first usage row"); - let second = claude_usage_record_from_event(&second).expect("second usage row"); - let mut summary = CostSummary::default(); - - assert!(add_claude_record_to_summary(&mut summary, &first)); - assert!(!add_claude_record_to_summary(&mut summary, &second)); - assert_eq!(summary.input_tokens, u64::MAX); - assert_eq!(summary.output_tokens, 5); - assert!(summary.total_cost_usd.is_finite()); - - finalize_claude_summary( - &mut summary, - true, - ClaudeFileScanResult { - counted: 2, - aggregation_failures: 1, - ..ClaudeFileScanResult::default() - }, - false, - ); - assert!(!summary.history_coverage_established); - assert!(!summary.known_zero); -} - -#[test] -fn oversized_single_claude_row_keeps_cost_but_marks_combined_quota_tokens_unknown() { - let event: ClaudeEvent = serde_json::from_str(&format!( - r#"{{"type":"assistant","timestamp":"2026-09-20T12:00:00Z","requestId":"req_combined_overflow","message":{{"id":"msg_combined_overflow","model":"claude-sonnet-4-6","usage":{{"input_tokens":{},"output_tokens":1}}}}}}"#, - u64::MAX - )) - .unwrap(); - let record = claude_usage_record_from_event(&event).expect("usage row"); - let quota = quota_history_record_from_usage(&record).expect("timestamped quota row"); - - assert!(record.cost.is_some_and(f64::is_finite)); - assert_eq!(quota.tokens, None); - assert!(!quota.tokens_are_complete); - assert!(quota.cost_usd.is_some_and(f64::is_finite)); - assert!(quota.cost_is_complete); -} - -#[test] -fn nonfinite_claude_price_is_unknown_instead_of_zero() { - let snapshot = crate::core::ModelsDevPricingSnapshot::from_catalog_json_for_tests( - r#"{ - "anthropic": {"models": {"claude-test-extreme-price": { - "id": "claude-test-extreme-price", "cost": {"input": 1e308, "output": 1} - }}} - }"#, - ) - .expect("pricing fixture"); - let mut pricing = ClaudeScanPricingResolver::with_snapshot(snapshot); - let event: ClaudeEvent = serde_json::from_str(&format!( - r#"{{"type":"assistant","timestamp":"2026-09-20T12:00:00Z","requestId":"req_nonfinite","message":{{"id":"msg_nonfinite","model":"claude-test-extreme-price","usage":{{"input_tokens":{},"output_tokens":1}}}}}}"#, - u64::MAX - )) - .unwrap(); - let record = - claude_usage_record_from_event_with_pricing(&event, &mut pricing).expect("usage row"); - let mut summary = CostSummary::default(); - - assert_eq!(record.cost, None); - assert!(!add_claude_record_to_summary(&mut summary, &record)); - assert_eq!(summary.input_tokens, u64::MAX); - assert_eq!(summary.output_tokens, 1); - assert_eq!(summary.total_cost_usd, 0.0); - assert!(!summary.known_zero); -} - -fn claude_transcript_line( - timestamp: &str, - request_key: &str, - request_id: &str, - message_id: &str, -) -> String { - format!( - r#"{{"type":"assistant","timestamp":"{timestamp}","{request_key}":"{request_id}","message":{{"id":"{message_id}","model":"claude-sonnet-4-6","usage":{{"input_tokens":1000,"output_tokens":500}}}}}}"# - ) -} - -#[test] -fn daily_history_dedups_across_files_and_buckets_by_local_day() { - // End-to-end regression for the daily buckets: two transcript files, - // two different days, plus a replay of the day-one record in the - // second file (snake_case request_id, as another writer would emit). - let dir = std::env::temp_dir(); - let file_a = dir.join(format!( - "codexbar-claude-daily-a-{}.jsonl", - std::process::id() - )); - let file_b = dir.join(format!( - "codexbar-claude-daily-b-{}.jsonl", - std::process::id() - )); - - // >24h apart guarantees two distinct local calendar days. - let day_one = Utc::now() - Duration::hours(30); - let day_two = Utc::now() - Duration::hours(2); - let ts_one = day_one.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - let ts_two = day_two.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - - std::fs::write( - &file_a, - format!( - "{}\n{}\n", - claude_transcript_line(&ts_one, "requestId", "req_1", "msg_1"), - claude_transcript_line(&ts_two, "requestId", "req_2", "msg_2"), - ), - ) - .unwrap(); - std::fs::write( - &file_b, - format!( - "{}\n", - claude_transcript_line(&ts_one, "request_id", "req_1", "msg_1"), - ), - ) - .unwrap(); - - let day_key = |ts: &DateTime| { - ts.with_timezone(&Local) - .date_naive() - .format("%Y-%m-%d") - .to_string() - }; - let mut daily_costs = HashMap::new(); - daily_costs.insert(day_key(&day_one), None); - daily_costs.insert(day_key(&day_two), None); - let mut unknown_cost_dates = HashSet::new(); - - let cutoff = Utc::now() - Duration::days(30); - let mut seen = HashSet::new(); - for path in [&file_a, &file_b] { - for_each_claude_usage_record(path, &cutoff, &mut seen, None, |record| { - add_claude_record_to_daily_costs(&mut daily_costs, &mut unknown_cost_dates, record); - }); - } - - let day_one_cost = daily_costs[&day_key(&day_one)].expect("day one cost"); - let day_two_cost = daily_costs[&day_key(&day_two)].expect("day two cost"); - assert!(day_one_cost > 0.0, "day one should carry real cost"); - // Identical usage on both days: equal buckets proves the file-b - // replay was de-duplicated (a leak would double day one). - assert!( - (day_one_cost - day_two_cost).abs() < f64::EPSILON, - "each day should hold exactly one record's cost, got {day_one_cost} vs {day_two_cost}" - ); - - // Best-effort test cleanup; the files may already be gone. - let _removed_a = std::fs::remove_file(&file_a); - let _removed_b = std::fs::remove_file(&file_b); -} - -fn claude_daily_cost_record(timestamp: DateTime, cost: Option) -> ClaudeUsageRecord { - ClaudeUsageRecord { - model: "claude-test".to_string(), - pricing_known: cost.is_some(), - timestamp: Some(timestamp), - dedup_key: None, - input: 1, - output: 1, - cache_create: 0, - cache_read: 0, - cost, - } -} - -#[test] -fn unknown_claude_cost_date_cannot_be_restored_by_later_priced_record() { - let timestamp = Utc::now(); - let day = timestamp - .with_timezone(&Local) - .date_naive() - .format("%Y-%m-%d") - .to_string(); - let mut daily_costs = HashMap::from([(day.clone(), None)]); - let mut unknown_cost_dates = HashSet::new(); - - assert!(add_claude_record_to_daily_costs( - &mut daily_costs, - &mut unknown_cost_dates, - &claude_daily_cost_record(timestamp, Some(0.75)), - )); - assert!(!add_claude_record_to_daily_costs( - &mut daily_costs, - &mut unknown_cost_dates, - &claude_daily_cost_record(timestamp, None), - )); - assert!(!add_claude_record_to_daily_costs( - &mut daily_costs, - &mut unknown_cost_dates, - &claude_daily_cost_record(timestamp, Some(1.25)), - )); - - assert_eq!(daily_costs[&day], None); - assert!(unknown_cost_dates.contains(&day)); -} - -#[test] -fn claude_daily_zero_fill_preserves_unknown_dates_and_fills_untouched_dates() { - let unknown_day = "2026-09-22".to_string(); - let untouched_day = "2026-09-23".to_string(); - let mut daily_costs = - HashMap::from([(unknown_day.clone(), None), (untouched_day.clone(), None)]); - let unknown_cost_dates = HashSet::from([unknown_day.clone()]); - - zero_fill_uninitialized_claude_daily_costs(&mut daily_costs, &unknown_cost_dates); - - assert_eq!(daily_costs[&unknown_day], None); - assert_eq!(daily_costs[&untouched_day], Some(0.0)); -} - -#[test] -fn claude_scan_counts_final_incomplete_jsonl_line() { - let path = - std::env::temp_dir().join(format!("codexbar-claude-tail-{}.jsonl", std::process::id())); - let ts = (Utc::now() - Duration::hours(1)) - .format("%Y-%m-%dT%H:%M:%S%.3fZ") - .to_string(); - // No trailing newline — the last (only) record must still be counted. - let body = claude_transcript_line(&ts, "requestId", "req_tail", "msg_tail"); - std::fs::write(&path, body.as_bytes()).unwrap(); - - let cutoff = Utc::now() - Duration::days(1); - let mut seen = HashSet::new(); - let counted = for_each_claude_usage_record(&path, &cutoff, &mut seen, None, |_| {}); - assert_eq!(counted, 1, "incomplete final JSONL line must be processed"); - // Best-effort test cleanup; the file may already be gone. - let _removed = std::fs::remove_file(&path); -} - -/// An event time for a fresh Codex session fixture: an hour ago, kept on today's local date. -/// -/// The scanner files each event under its local date, and the session fixtures live in today's -/// date folder. A plain `now - 1h` lands on yesterday in the first hour after local midnight -/// (00:00-01:00Z on the UTC CI runner), so tests that read today's bucket or rely on the day -/// folder scan order failed in that hour. -fn recent_codex_fixture_time() -> DateTime { - recent_fixture_time_at(Local::now()) -} - -/// `now - 1h`, or the start of `now`'s local day when that hour reaches back into yesterday. -fn recent_fixture_time_at(now: DateTime) -> DateTime { - let hour_ago = now.clone() - Duration::hours(1); - if hour_ago.date_naive() == now.date_naive() { - return hour_ago.with_timezone(&Utc); - } - now.timezone() - .from_local_datetime(&now.date_naive().and_time(NaiveTime::MIN)) - .earliest() - .unwrap_or(now) - .with_timezone(&Utc) -} - -#[test] -fn recent_codex_fixture_time_stays_on_the_local_day() { - let utc_plus_7 = FixedOffset::east_opt(7 * 3600).unwrap(); - let at = |hour, minute| { - utc_plus_7 - .with_ymd_and_hms(2026, 10, 1, hour, minute, 0) - .unwrap() - }; - assert_eq!(recent_fixture_time_at(at(8, 30)), at(7, 30)); - assert_eq!(recent_fixture_time_at(at(1, 0)), at(0, 0)); - assert_eq!(recent_fixture_time_at(at(0, 40)), at(0, 0)); - assert_eq!(recent_fixture_time_at(at(0, 0)), at(0, 0)); - - let ci_run = Utc.with_ymd_and_hms(2026, 10, 1, 0, 45, 0).unwrap(); - assert_eq!( - recent_fixture_time_at(ci_run), - Utc.with_ymd_and_hms(2026, 10, 1, 0, 0, 0).unwrap() - ); -} - -fn write_codex_session_fixture(sessions_root: &Path, name: &str, input_tokens: u64) -> PathBuf { - let event_time = recent_codex_fixture_time(); - let today = event_time.with_timezone(&Local).date_naive(); - let day_dir = partition_dir(sessions_root, today); - std::fs::create_dir_all(&day_dir).unwrap(); - let path = day_dir.join(name); - let ts = event_time.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - let body = format!( - r#"{{"timestamp":"{ts}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":{input_tokens},"cached_input_tokens":0,"output_tokens":5}}}}}}}} -"# - ); - std::fs::write(&path, body).unwrap(); - path -} - -fn write_codex_session_fixture_with_inputs( - sessions_root: &Path, - name: &str, - input_tokens: &[u64], -) -> PathBuf { - let base = recent_codex_fixture_time(); - let today = base.with_timezone(&Local).date_naive(); - let day_dir = partition_dir(sessions_root, today); - std::fs::create_dir_all(&day_dir).unwrap(); - let mut body = String::new(); - for (index, input) in input_tokens.iter().enumerate() { - let timestamp = (base - + Duration::seconds(i64::try_from(index).expect("fixture index fits i64"))) - .format("%Y-%m-%dT%H:%M:%S%.3fZ") - .to_string(); - body.push_str(&format!( - r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":{input},"cached_input_tokens":0,"output_tokens":5}}}}}}}} -"# - )); - } - let path = day_dir.join(name); - std::fs::write(&path, body).unwrap(); - path -} - -fn cached_usage_with_packed(day: &str, model: &str, packed: Vec) -> CostUsageFileUsage { - CostUsageFileUsage { - parsed_bytes: Some(1), - ..test_file_usage( - 1, - HashMap::from([( - day.to_string(), - HashMap::from([(model.to_string(), packed)]), - )]), - ) - } -} - -#[test] -fn rebuild_cache_days_preserves_known_reasoning() { - let day = Local::now().format("%Y-%m-%d").to_string(); - let mut cache = CostUsageCache { - files: HashMap::from([ - ( - "a".to_string(), - cached_usage_with_packed(&day, "gpt-5", vec![10, 0, 4, 3]), - ), - ( - "b".to_string(), - cached_usage_with_packed(&day, "gpt-5", vec![5, 0, 2, 1]), - ), - ]), - ..CostUsageCache::default() - }; - - rebuild_cache_days(&mut cache); - - assert_eq!(cache.days[&day]["gpt-5"], vec![15, 0, 6, 4]); -} - -#[test] -fn rebuild_cache_days_reasoning_unknown_is_order_independent() { - let run = |first: Vec, second: Vec| { - let day = Local::now().format("%Y-%m-%d").to_string(); - let mut cache = CostUsageCache { - files: HashMap::from([ - ( - "a".to_string(), - cached_usage_with_packed(&day, "gpt-5", first), - ), - ( - "b".to_string(), - cached_usage_with_packed(&day, "gpt-5", second), - ), - ]), - ..CostUsageCache::default() - }; - - rebuild_cache_days(&mut cache); - cache.days[&day]["gpt-5"].clone() - }; - - assert_eq!(run(vec![10, 0, 4, 3], vec![5, 0, 2]), vec![15, 0, 6]); - assert_eq!(run(vec![5, 0, 2], vec![10, 0, 4, 3]), vec![15, 0, 6]); -} - -#[test] -fn rebuild_cache_days_zero_row_does_not_poison_reasoning() { - let day = Local::now().format("%Y-%m-%d").to_string(); - let mut cache = CostUsageCache { - files: HashMap::from([ - ( - "a".to_string(), - cached_usage_with_packed(&day, "gpt-5", vec![10, 0, 4, 3]), - ), - ( - "b".to_string(), - cached_usage_with_packed(&day, "gpt-5", vec![0, 0, 0]), - ), - ]), - ..CostUsageCache::default() - }; - - rebuild_cache_days(&mut cache); - - assert_eq!(cache.days[&day]["gpt-5"], vec![10, 0, 4, 3]); -} - -#[test] -fn rebuild_cache_days_aggregates_multiple_files_above_i32_max() { - let day = Local::now().format("%Y-%m-%d").to_string(); - let mut cache = CostUsageCache { - files: HashMap::from([ - ( - "a".to_string(), - cached_usage_with_packed(&day, "gpt-5", vec![1_500_000_000, 1_400_000_000, 100]), - ), - ( - "b".to_string(), - cached_usage_with_packed(&day, "gpt-5", vec![1_500_000_000, 1_400_000_000, 100]), - ), - ]), - ..CostUsageCache::default() - }; - - rebuild_cache_days(&mut cache); - - assert_eq!( - cache.days[&day]["gpt-5"], - vec![3_000_000_000, 2_800_000_000, 200] - ); -} - -#[test] -fn retained_report_sums_multiple_days_above_i32_max() { - let day_a = "2026-09-08"; - let day_b = "2026-09-09"; - let mut cache = CostUsageCache { - files: HashMap::from([ - ( - "a".to_string(), - cached_usage_with_packed( - day_a, - "gpt-5.6-sol", - vec![1_500_000_000, 1_400_000_000, 1_000_000], - ), - ), - ( - "b".to_string(), - cached_usage_with_packed( - day_b, - "gpt-5.6-sol", - vec![1_500_000_000, 1_400_000_000, 1_000_000], - ), - ), - ]), - ..CostUsageCache::default() - }; - rebuild_cache_days(&mut cache); - - let report = JsonlScanner::cached_cost_report_from_days(&cache); - assert_eq!(report.input_tokens, 3_000_000_000); - assert_eq!(report.cached_tokens, 2_800_000_000); - assert_eq!(report.output_tokens, 2_000_000); - - let start = chrono::NaiveDate::from_ymd_opt(2026, 9, 8).unwrap(); - let end = chrono::NaiveDate::from_ymd_opt(2026, 9, 9).unwrap(); - let summary = summary_from_cached_report(&report, start, end); - assert_eq!(summary.input_tokens, 3_000_000_000); - assert_eq!(summary.cached_tokens, 2_800_000_000); - assert_eq!(summary.output_tokens, 2_000_000); - assert_eq!(summary.sessions_count, 2); -} - -#[test] -fn reasoning_survives_scan_rebuild_and_cache_reload() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - // The event timestamp (now − 1h) decides the parsed day key, so derive the - // fixture day from that same instant: at local 00:00–01:00 now − 1h falls - // on the previous local day and the row would land there, not on today. - let event_time = Utc::now() - Duration::hours(1); - let today = event_time.with_timezone(&Local).date_naive(); - let day = today.format("%Y-%m-%d").to_string(); - let day_dir = partition_dir(&sessions, today); - std::fs::create_dir_all(&day_dir).unwrap(); - let timestamp = event_time.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - let reasoning_line = serde_json::json!({ - "timestamp": timestamp, - "type": "event_msg", - "payload": { - "type": "token_count", - "info": { - "model": "gpt-5", - "total_token_usage": { - "input_tokens": 100, - "cached_input_tokens": 0, - "output_tokens": 20, - "reasoning_output_tokens": 7 - } - } - } - }); - std::fs::write( - day_dir.join("reasoning.jsonl"), - format!("{reasoning_line}\n"), - ) - .unwrap(); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(summary.output_tokens, 20); - assert_eq!(summary.reasoning_tokens, Some(7)); - let row = &cache.days[&day]["gpt-5"]; - assert!(row.len() >= 4); - assert_eq!(row[3], 7); - - let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - let loaded_row = &loaded.days[&day]["gpt-5"]; - assert!(loaded_row.len() >= 4); - assert_eq!(loaded_row[3], 7); - assert_eq!( - JsonlScanner::cached_cost_report_from_days(&loaded).reasoning_tokens, - Some(7) - ); - - let legacy_root = tempfile::tempdir().unwrap(); - let legacy_sessions = legacy_root.path().join("sessions"); - let legacy_cache_root = legacy_root.path().join("cache"); - let legacy_day_dir = partition_dir(&legacy_sessions, today); - std::fs::create_dir_all(&legacy_day_dir).unwrap(); - let legacy_line = serde_json::json!({ - "timestamp": timestamp, - "type": "event_msg", - "payload": { - "type": "token_count", - "info": { - "model": "gpt-5", - "total_token_usage": { - "input_tokens": 100, - "cached_input_tokens": 0, - "output_tokens": 20 - } - } - } - }); - std::fs::write( - legacy_day_dir.join("legacy.jsonl"), - format!("{legacy_line}\n"), - ) - .unwrap(); - - let legacy_scanner = app_scanner(7, &legacy_cache_root, &legacy_sessions); - let (legacy_summary, _, legacy_cache) = legacy_scanner.scan_codex_detailed_with_cache(None); - assert_eq!(legacy_summary.output_tokens, 20); - assert_eq!(legacy_summary.reasoning_tokens, None); - assert_eq!(legacy_cache.days[&day]["gpt-5"], vec![100, 0, 20]); - assert!( - (summary.total_cost_usd - legacy_summary.total_cost_usd).abs() < 1e-12, - "reasoning metadata must not change cost" - ); -} - -fn write_codex_fork_session_fixture( - sessions_root: &Path, - name: &str, - session_id: &str, - parent_id: Option<&str>, - fork_timestamp: DateTime, - token_start: DateTime, - totals: &[i64], -) -> PathBuf { - let today = Local::now().date_naive(); - let day_dir = partition_dir(sessions_root, today); - std::fs::create_dir_all(&day_dir).unwrap(); - - let mut body = format!( - "{{\"type\":\"session_meta\",\"timestamp\":\"{}\",\"payload\":{{\"session_id\":\"{}\"", - fork_timestamp.to_rfc3339(), - session_id - ); - if let Some(parent_id) = parent_id { - body.push_str(&format!(",\"forked_from_id\":\"{parent_id}\"")); - } - body.push_str("}}\n"); - - for (index, total) in totals.iter().enumerate() { - let timestamp = (token_start - + Duration::seconds(i64::try_from(index).expect("fixture index fits i64"))) - .to_rfc3339(); - let line = serde_json::json!({ - "timestamp": timestamp, - "type": "event_msg", - "payload": { - "type": "token_count", - "info": { - "model": "gpt-5", - "total_token_usage": { - "input_tokens": total, - "cached_input_tokens": 0, - "output_tokens": 5 - } - } - } - }); - body.push_str(&line.to_string()); - body.push('\n'); - } - - let path = day_dir.join(name); - std::fs::write(&path, body).unwrap(); - path -} - -fn cached_input_total(usage: &CostUsageFileUsage) -> i64 { - usage - .days - .values() - .flat_map(|models| models.values()) - .map(|tokens| tokens.first().copied().unwrap_or_default()) - .sum() -} - -#[test] -fn ordinary_non_fork_session_keeps_cumulative_accounting() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture_with_inputs(&sessions, "ordinary.jsonl", &[100, 140]); - - let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = false; - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - - assert_eq!(summary.input_tokens, 140); - let usage = cached_file(&cache, &path); - assert_eq!(cached_input_total(usage), 140); - assert_eq!(usage.codex_forked_from_id, None); - assert!(!usage.codex_unresolved_fork_parent); -} - -#[test] -fn fork_child_counts_only_growth_above_parent_baseline() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let base = Utc::now() - Duration::hours(1); - let fork = base + Duration::seconds(2); - let parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "parent-id", - None, - base, - base, - &[1_000_000], - ); - let child = write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "child-id", - Some("parent-id"), - fork, - base + Duration::seconds(3), - &[1_000_000, 1_000_140], - ); - - let now = std::time::SystemTime::now(); - File::options() - .write(true) - .open(&parent) - .unwrap() - .set_modified(now - std::time::Duration::from_secs(10)) - .unwrap(); - File::options() - .write(true) - .open(&child) - .unwrap() - .set_modified(now - std::time::Duration::from_secs(5)) - .unwrap(); - - let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = false; - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - - assert_eq!(summary.input_tokens, 1_000_140); - assert_eq!(summary.sessions_count, 2); - let child_usage = cached_file(&cache, &child); - assert_eq!(cached_input_total(child_usage), 140); - assert!(!child_usage.codex_unresolved_fork_parent); - assert!(cache.codex_pending_paths.is_empty()); - assert!( - cache - .files - .contains_key(&parent.to_string_lossy().to_string()) - ); -} - -#[test] -fn replaced_fork_child_does_not_reuse_cached_identity() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let base = Utc::now() - Duration::hours(1); - let fork = base + Duration::seconds(2); - let _parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "parent-id", - None, - base, - base, - &[1_000_000], - ); - let child = write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "child-id", - Some("parent-id"), - fork, - base + Duration::seconds(3), - &[1_000_000, 1_000_140], - ); - - let scanner = CostScanner::new(7) - .with_options({ - let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = false; - options - }) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let child_key = child.to_string_lossy().to_string(); - let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); - let first_child = first_cache.files.get(&child_key).unwrap(); - assert_eq!(first_child.codex_session_id.as_deref(), Some("child-id")); - assert_eq!( - first_child.codex_forked_from_id.as_deref(), - Some("parent-id") - ); - - let old_size = std::fs::metadata(&child).unwrap().len(); - write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "replacement-id", - None, - base + Duration::seconds(4), - base + Duration::seconds(5), - &[77], - ); - let new_size = std::fs::metadata(&child).unwrap().len(); - assert_ne!(old_size, new_size); - - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - let child_usage = cache.files.get(&child_key).unwrap(); - assert_eq!( - child_usage.codex_session_id.as_deref(), - Some("replacement-id") - ); - assert_eq!(child_usage.codex_forked_from_id, None); - assert!(!child_usage.codex_unresolved_fork_parent); - assert_eq!(cached_input_total(child_usage), 77); - assert_eq!(summary.input_tokens, 1_000_077); -} - -#[test] -fn missing_fork_parent_fails_closed_and_persists_pending() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let base = Utc::now() - Duration::hours(1); - let child = write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "child-id", - Some("missing-parent"), - base + Duration::seconds(1), - base + Duration::seconds(2), - &[1_000_000, 1_000_140], - ); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - - let child_key = child.to_string_lossy().to_string(); - let child_usage = cache.files.get(&child_key).unwrap(); - assert_eq!(summary.input_tokens, 0); - assert_eq!(summary.sessions_count, 0); - assert!(child_usage.days.is_empty()); - assert!(child_usage.codex_unresolved_fork_parent); - assert!(cache.codex_pending_paths.contains(&child_key)); - assert!(!summary.history_coverage_established); - assert!(!summary.known_zero); -} - -#[test] -fn fork_child_resolves_after_parent_is_cached() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let base = Utc::now() - Duration::hours(1); - let fork = base + Duration::seconds(2); - let child = write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "child-id", - Some("late-parent"), - fork, - base + Duration::seconds(3), - &[1_000_000, 1_000_140], - ); - let scanner = CostScanner::new(7) - .with_options({ - let mut options = CostScanOptions::app_driven(); - options.prefer_newest_codex_sessions_first = false; - options - }) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - - let (first, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(first.input_tokens, 0); - assert!( - first_cache - .codex_pending_paths - .contains(&child.to_string_lossy().to_string()) - ); - - let _parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "late-parent", - None, - base, - base, - &[1_000_000], - ); - - let mut resolved = None; - for _ in 0..3 { - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - if !cache.codex_scan_incomplete { - resolved = Some((summary, cache)); - break; - } - } - let (summary, cache) = resolved.expect("later bounded pass resolves the child"); - let child_usage = cached_file(&cache, &child); - assert_eq!(summary.input_tokens, 1_000_140); - assert_eq!(cached_input_total(child_usage), 140); - assert!(!child_usage.codex_unresolved_fork_parent); - assert!(cache.codex_pending_paths.is_empty()); -} - -#[test] -fn parent_last_token_after_fork_keeps_child_unresolved() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let base = Utc::now() - Duration::hours(1); - let parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "parent-id", - None, - base + Duration::seconds(10), - base + Duration::seconds(10), - &[1_000_000], - ); - let child = write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "child-id", - Some("parent-id"), - base + Duration::seconds(1), - base + Duration::seconds(2), - &[1_000_000, 1_000_140], - ); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - - let child_usage = cached_file(&cache, &child); - assert_eq!(summary.input_tokens, 1_000_000); - assert_eq!(cached_input_total(child_usage), 0); - assert!(child_usage.codex_unresolved_fork_parent); - assert!( - cache - .codex_pending_paths - .contains(&child.to_string_lossy().to_string()) - ); - assert!( - cache - .files - .contains_key(&parent.to_string_lossy().to_string()) - ); -} - -#[test] -fn deleted_unresolved_child_is_pruned_without_resurrection() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let base = Utc::now() - Duration::hours(1); - let child = write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "child-id", - Some("missing-parent"), - base + Duration::seconds(1), - base + Duration::seconds(2), - &[1_000_000, 1_000_140], - ); - let scanner = app_scanner(7, &cache_root, &sessions); - let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); - assert!( - first_cache - .codex_pending_paths - .contains(&child.to_string_lossy().to_string()) - ); - - std::fs::remove_file(&child).unwrap(); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - assert!(summary.history_coverage_established); - assert!(summary.known_zero); - assert!(cache.codex_pending_paths.is_empty()); - assert!( - !cache - .files - .contains_key(&child.to_string_lossy().to_string()) - ); -} - -#[test] -fn fork_baseline_reset_fails_closed_instead_of_billing_fresh_usage() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let base = Utc::now() - Duration::hours(1); - let _parent = write_codex_fork_session_fixture( - &sessions, - "parent.jsonl", - "parent-id", - None, - base, - base, - &[1_000_000], - ); - let child = write_codex_fork_session_fixture( - &sessions, - "child.jsonl", - "child-id", - Some("parent-id"), - base + Duration::seconds(1), - base + Duration::seconds(2), - &[1_000_000, 999_900, 1_000_140], - ); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - let child_usage = cached_file(&cache, &child); - - assert_eq!(summary.input_tokens, 1_000_000); - assert_eq!(cached_input_total(child_usage), 0); - assert!(child_usage.codex_unresolved_fork_parent); - assert!(!summary.history_coverage_established); -} - -#[test] -fn cost_scan_second_pass_skips_unchanged_files_via_cache() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - write_codex_session_fixture(&sessions, "a.jsonl", 100); - write_codex_session_fixture(&sessions, "b.jsonl", 200); - - let scanner = app_scanner(7, &cache_root, &sessions); - - let (summary1, stats1) = scanner.scan_codex_detailed(None); - assert_eq!(stats1.files_parsed, 2, "first pass parses both files"); - assert_eq!(stats1.files_skipped, 0); - assert_eq!(stats1.codex_metadata_read_paths.len(), 2); - assert_eq!(stats1.codex_history_read_paths.len(), 2); - assert_eq!(stats1.codex_read_receipt.metadata_reads, 2); - assert_eq!(stats1.codex_read_receipt.history_reads, 2); - assert!(summary1.total_cost_usd > 0.0); - assert_eq!(summary1.sessions_count, 2); - - // Second pass with default debounce still inspects files but skips re-parse. - // Use app_driven so we exercise per-file mtime skip rather than whole-scan debounce. - CodexLineagePlanner::reset_graph_build_count(); - let (summary2, stats2) = scanner.scan_codex_detailed(None); - assert_eq!(stats2.files_seen, 2); - assert_eq!(stats2.files_skipped, 2, "cache hit skips re-parse"); - assert_eq!(stats2.files_parsed, 0); - assert_eq!( - CodexLineagePlanner::graph_build_count(), - 0, - "warm root-only scan must bypass lineage graph construction" - ); - assert!(stats2.codex_metadata_read_paths.is_empty()); - assert!(stats2.codex_history_read_paths.is_empty()); - assert_eq!(stats2.codex_read_receipt, Default::default()); - assert_eq!(summary2.input_tokens, summary1.input_tokens); - assert!((summary2.total_cost_usd - summary1.total_cost_usd).abs() < 1e-9); - - // Force path already used above; confirm debounce short-circuit with default options. - let debounced = CostScanner::new(7) - .with_options(CostScanOptions::default()) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (summary3, stats3) = debounced.scan_codex_detailed(None); - assert!( - stats3.used_cache_debounce, - "default options debounce within 60s" - ); - assert_eq!(stats3.files_seen, 0); - assert_eq!(summary3.input_tokens, summary1.input_tokens); - - // app_driven after debounce still re-reads (skip via mtime, not full re-parse). - let forced = app_scanner(7, &cache_root, &sessions); - let (_, stats4) = forced.scan_codex_detailed(None); - assert!(!stats4.used_cache_debounce); - assert_eq!(stats4.files_skipped, 2); - assert_eq!(stats4.files_parsed, 0); - assert!(stats4.codex_history_read_paths.is_empty()); -} - -#[test] -fn codex_lazy_history_receipt_reads_only_changed_file_and_matches_fresh_parse() { - let (root, sessions, cache_root) = codex_scan_dirs(); - let first_path = write_codex_session_fixture_with_inputs(&sessions, "first.jsonl", &[100]); - let second_path = write_codex_session_fixture_with_inputs(&sessions, "second.jsonl", &[200]); - let scanner = app_scanner(7, &cache_root, &sessions); - - let (initial, _, _) = scanner.scan_codex_detailed_with_cache(None); - let (unchanged, unchanged_stats, _) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(unchanged.input_tokens, initial.input_tokens); - assert!(unchanged_stats.codex_metadata_read_paths.is_empty()); - assert!(unchanged_stats.codex_history_read_paths.is_empty()); - assert_eq!(unchanged_stats.codex_read_receipt, Default::default()); - - use std::io::Write as _; - let timestamp = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - let extra = format!( - r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":150,"cached_input_tokens":0,"output_tokens":5}}}}}}}} -"# - ); - std::fs::OpenOptions::new() - .append(true) - .open(&first_path) - .unwrap() - .write_all(extra.as_bytes()) - .unwrap(); - - let (incremental, incremental_stats, _) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!( - incremental_stats.codex_metadata_read_paths, - vec![first_path.to_string_lossy().to_string()] - ); - assert_eq!( - incremental_stats.codex_history_read_paths, - vec![first_path.to_string_lossy().to_string()] - ); - assert_eq!(incremental_stats.codex_read_receipt.metadata_reads, 1); - assert_eq!(incremental_stats.codex_read_receipt.history_reads, 1); - assert_eq!(incremental.input_tokens, 350); - - let fresh = app_scanner(7, root.path().join("fresh-cache"), &sessions); - let (full, full_stats) = fresh.scan_codex_detailed(None); - assert_eq!(full_stats.codex_history_read_paths.len(), 2); - assert_eq!(incremental.input_tokens, full.input_tokens); - assert_eq!(incremental.output_tokens, full.output_tokens); - assert_eq!(incremental.cached_tokens, full.cached_tokens); - assert_eq!(incremental.by_model_tokens, full.by_model_tokens); - assert!((incremental.total_cost_usd - full.total_cost_usd).abs() < 1e-12); - assert!(second_path.exists()); -} - -#[test] -fn codex_source_recovery_keeps_appended_duplicate_unpriced_after_cache_reload() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture_with_inputs(&sessions, "recovery.jsonl", &[100]); - let scanner = app_scanner(7, &cache_root, &sessions); - - let (_, _, mut first_cache) = scanner.scan_codex_detailed_with_cache(None); - let path_key = path.to_string_lossy().to_string(); - let source_rows = first_cache - .codex_source_rows - .get_mut(&path_key) - .expect("source rows persisted"); - assert_eq!(source_rows.rows.len(), 1); - // Cached evidence that disagrees with the source: a Priority row on a - // model with a Fast lane, so the recovered row keeps a `-priority` key - // (a Priority row without a Fast lane would price at its Standard base). - source_rows.rows[0].pricing.pricing_model = Some("gpt-5.5".to_string()); - source_rows.rows[0].pricing.pricing_mode = Some("priority".to_string()); - first_cache.last_scan_unix_ms = 1; - JsonlScanner::save_cache(ProviderId::Codex, &mut first_cache, Some(&cache_root)); - - // Half an hour after the fixture's row, so both rows share one local day. - let appended_time = recent_codex_fixture_time() + Duration::minutes(30); - let timestamp = appended_time.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - let appended = format!( - r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":200,"cached_input_tokens":0,"output_tokens":10}}}}}}}}"# - ) + "\n"; - std::fs::OpenOptions::new() - .append(true) - .open(&path) - .unwrap() - .write_all(appended.as_bytes()) - .unwrap(); - - let (_, _, second_cache) = scanner.scan_codex_detailed_with_cache(None); - let usage = second_cache.files.get(&path_key).expect("file cache"); - let day = appended_time - .with_timezone(&Local) - .format("%Y-%m-%d") - .to_string(); - assert_eq!( - usage.days[&day]["gpt-5.5-priority"], - vec![100, 0, 5], - "gpt-5.5 turns also price at the Priority rate" - ); - assert_eq!( - usage.days[&day][CostUsagePricing::CODEX_UNATTRIBUTED_MODEL], - vec![100, 0, 5] - ); -} - -#[test] -fn codex_file_identity_invalidates_same_path_cache_without_eager_history_read() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "replacement.jsonl", 100); - let scanner = app_scanner(7, &cache_root, &sessions); - let (_, _, _) = scanner.scan_codex_detailed_with_cache(None); - let old_mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); - let rotated = path.with_extension("old"); - std::fs::rename(&path, &rotated).unwrap(); - let replacement = write_codex_session_fixture(&sessions, "replacement.jsonl", 200); - // Windows requires a handle with write-attribute access for set_modified; - // keep the replacement's mtime equal to the original without opening it - // read-only. The file contents have the same length, so path/mtime/size - // remain unchanged while the file identity changes. - std::fs::OpenOptions::new() - .write(true) - .open(&replacement) - .unwrap() - .set_modified(old_mtime) - .unwrap(); - - let (summary, stats) = scanner.scan_codex_detailed(None); - assert_eq!(summary.input_tokens, 200); - assert_eq!( - stats.codex_history_read_paths, - vec![replacement.to_string_lossy().to_string()] - ); - assert_eq!(stats.codex_read_receipt.history_reads, 1); -} - -#[test] -fn cancelled_fresh_cache_hit_is_not_authoritative() { - let root = tempfile::tempdir().unwrap(); - let cache_root = root.path().join("cache"); - let today = Local::now().date_naive().format("%Y-%m-%d").to_string(); - let usage = HashMap::from([( - today.clone(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![100, 0, 10])]), - )]); - let mut cache = CostUsageCache { - last_scan_unix_ms: unix_now_ms(), - files: HashMap::from([( - "cached.jsonl".to_string(), - CostUsageFileUsage { - parsed_bytes: Some(100), - last_model: Some("gpt-5.6-sol".to_string()), - codex_token_timestamps_monotonic: Some(true), - ..test_file_usage(100, usage.clone()) - }, - )]), - days: usage, - ..Default::default() - }; - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - - let cancel = AtomicBool::new(true); - let scanner = CostScanner::new(7) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![root.path().join("sessions")]); - let (summary, stats) = scanner.scan_codex_detailed(Some(&cancel)); - - assert!( - stats.used_cache_debounce, - "fresh cache should use debounce path" - ); - assert_eq!(summary.sessions_count, 1, "cached usage is still visible"); - assert!( - !summary.history_coverage_established, - "cancelled cache publication must not claim complete history" - ); - assert!(!summary.known_zero); -} - -#[test] -fn cost_scan_cancel_stops_between_files() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - write_codex_session_fixture(&sessions, "a.jsonl", 100); - write_codex_session_fixture(&sessions, "b.jsonl", 200); - write_codex_session_fixture(&sessions, "c.jsonl", 300); - - let cancel = AtomicBool::new(true); - let scanner = app_scanner(7, cache_root, &sessions); - let (summary, stats) = scanner.scan_codex_detailed(Some(&cancel)); - assert_eq!(stats.files_seen, 0, "cancel before first file stops walk"); - assert_eq!(summary.sessions_count, 0); -} - -#[test] -fn cost_scan_reconciles_deleted_file_to_known_zero() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (first, _) = scanner.scan_codex_detailed(None); - assert_eq!(first.sessions_count, 1); - assert!(first.total_cost_usd > 0.0); - - std::fs::remove_file(&path).unwrap(); - let (second, _) = scanner.scan_codex_detailed(None); - - assert_eq!(second.sessions_count, 0); - assert_eq!(second.total_cost_usd, 0.0); - assert!(second.history_coverage_established); - assert!(second.known_zero); - let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!(cache.files.is_empty(), "deleted JSONL row must be removed"); - assert!(cache.days.is_empty(), "stale daily totals must disappear"); -} - -#[test] -fn cost_scan_reconciliation_preserves_sibling_totals_once() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let deleted = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); - let sibling = write_codex_session_fixture(&sessions, "sibling.jsonl", 200); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (first, _) = scanner.scan_codex_detailed(None); - assert_eq!(first.sessions_count, 2); - - std::fs::remove_file(&deleted).unwrap(); - let (second, _) = scanner.scan_codex_detailed(None); - - assert_eq!(second.sessions_count, 1); - assert_eq!(second.input_tokens, 200); - assert_eq!(second.output_tokens, 5); - let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert_eq!(cache.files.len(), 1); - assert!( - !cache - .files - .contains_key(&deleted.to_string_lossy().to_string()) - ); - assert!( - cache - .files - .contains_key(&sibling.to_string_lossy().to_string()) - ); -} - -#[test] -fn cancelled_scan_after_deletion_preserves_stale_cache_row() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (first, _) = scanner.scan_codex_detailed(None); - assert_eq!(first.sessions_count, 1); - std::fs::remove_file(&path).unwrap(); - - let cancel = AtomicBool::new(true); - let (cancelled, stats) = scanner.scan_codex_detailed(Some(&cancel)); - - assert_eq!(stats.files_seen, 0); - assert_eq!(cancelled.sessions_count, 0); - assert!(!cancelled.history_coverage_established); - assert!(!cancelled.known_zero); - let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!( - cache - .files - .contains_key(&path.to_string_lossy().to_string()) - ); -} - -#[test] -fn cost_scan_resumes_appended_bytes() { - let (root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "grow.jsonl", 50); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (s1, st1) = scanner.scan_codex_detailed(None); - assert_eq!(st1.files_parsed, 1); - assert_eq!(st1.token_timestamp_comparisons, 0); - assert_eq!(s1.input_tokens, 50); - - // Append another cumulative token_count event (100 total => +50 delta). - let ts = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); - let extra = format!( - r#"{{"timestamp":"{ts}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":100,"cached_input_tokens":0,"output_tokens":10}}}}}}}} -"# - ); - use std::io::Write as _; - let mut f = std::fs::OpenOptions::new() - .append(true) - .open(&path) - .unwrap(); - f.write_all(extra.as_bytes()).unwrap(); - drop(f); - - // Bump mtime/size visibly on some FS by rewriting metadata via reopen. - let (s2, st2) = scanner.scan_codex_detailed(None); - assert_eq!(st2.files_resumed, 1, "grown file resumes from offset"); - assert_eq!(st2.files_parsed, 0); - assert_eq!( - st2.token_timestamp_comparisons, 1, - "resume validates only the cached-prefix boundary and appended event" - ); - assert_eq!(s2.input_tokens, 100); - - // The append-only path must publish the same aggregate as a fresh - // full parse; the optimization is allowed to change work, not data. - let full_scanner = app_scanner(7, root.path().join("fresh-cache"), &sessions); - let (full, full_stats) = full_scanner.scan_codex_detailed(None); - assert_eq!(full_stats.files_parsed, 1); - assert_eq!(s2.input_tokens, full.input_tokens); - assert_eq!(s2.cached_tokens, full.cached_tokens); - assert_eq!(s2.output_tokens, full.output_tokens); - assert_eq!(s2.sessions_count, full.sessions_count); - assert_eq!(s2.by_model_tokens, full.by_model_tokens); - assert_eq!(s2.by_model.len(), full.by_model.len()); - for (model, resumed_cost) in &s2.by_model { - let full_cost = full.by_model.get(model).copied().expect("full model row"); - assert!((resumed_cost - full_cost).abs() < 1e-12); - } - assert!((s2.total_cost_usd - full.total_cost_usd).abs() < 1e-12); - - let cached = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - let cached_file = cached - .files - .get(&path.to_string_lossy().to_string()) - .expect("resumed file cache entry"); - assert_eq!(cached_file.codex_token_timestamps_monotonic, Some(true)); - assert!(cached_file.codex_last_token_timestamp.is_some()); -} - -#[test] -fn codex_partial_rescan_replaces_changed_session_after_cache_reopen() { - // Windows parity for upstream 0.60.2 cost persistence: a rewritten - // session must replace the cached file aggregate even when the first - // refresh only consumes a bounded prefix and the next refresh reloads the - // cache from disk. - let (root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture_with_inputs(&sessions, "changed.jsonl", &[100, 200]); - let old_metadata = std::fs::metadata(&path).unwrap(); - let old_size = old_metadata.len(); - let old_mtime = old_metadata.modified().unwrap(); - - let initial_scanner = app_scanner(7, &cache_root, &sessions); - let (initial, initial_stats) = initial_scanner.scan_codex_detailed(None); - assert_eq!(initial_stats.files_parsed, 1); - assert_eq!(initial.input_tokens, 200); - assert!(initial.history_coverage_established); - - // Keep the path, identity, and byte length stable while changing both - // token snapshots. The mtime change proves that the cached aggregate is - // invalidated before the bounded rescan begins. - write_codex_session_fixture_with_inputs(&sessions, "changed.jsonl", &[300, 400]); - std::fs::OpenOptions::new() - .write(true) - .open(&path) - .unwrap() - .set_modified(old_mtime + std::time::Duration::from_secs(2)) - .unwrap(); - let rewritten_metadata = std::fs::metadata(&path).unwrap(); - assert_eq!(rewritten_metadata.len(), old_size); - assert_ne!(rewritten_metadata.modified().unwrap(), old_mtime); - - let first_line_bytes = i64::try_from( - std::fs::read(&path) - .unwrap() - .split(|byte| *byte == b'\n') - .next() - .unwrap() - .len(), - ) - .expect("fixture line length fits i64") - + 1; - let mut bounded_options = CostScanOptions::app_driven(); - bounded_options.codex_max_session_file_bytes = first_line_bytes; - bounded_options.codex_max_scan_bytes_per_refresh = first_line_bytes; - - let bounded_scanner = CostScanner::new(7) - .with_options(bounded_options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (partial, partial_stats, partial_cache) = - bounded_scanner.scan_codex_detailed_with_cache(None); - assert!(partial_stats.files_parsed >= 1); - assert!(partial_cache.codex_scan_incomplete); - assert!(!partial.history_coverage_established); - assert_eq!( - partial_cache - .previous_report - .as_ref() - .map(|report| report.input_tokens), - Some(200), - "the last validated report remains visible during catch-up" - ); - let partial_file = partial_cache - .files - .get(&path.to_string_lossy().to_string()) - .expect("partially rescanned file cache entry"); - assert_eq!(partial_file.parsed_bytes, Some(first_line_bytes)); - - // A new scanner instance models a process/cache reopen. The persisted - // cursor must resume the rewritten file and finish at the new total. - let reopened_scanner = CostScanner::new(7) - .with_options(bounded_options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (resumed, resumed_stats, resumed_cache) = - reopened_scanner.scan_codex_detailed_with_cache(None); - assert_eq!(resumed_stats.files_resumed, 1); - assert_eq!(resumed_stats.files_parsed, 0); - assert!(!resumed_cache.codex_scan_incomplete); - assert!(resumed.history_coverage_established); - assert_eq!(resumed.input_tokens, 400); - - // Resumption may change the amount of work, but it must publish the same - // cost aggregate as a clean full parse of the rewritten session. - let fresh_scanner = app_scanner(7, root.path().join("fresh-cache"), &sessions); - let (fresh, fresh_stats) = fresh_scanner.scan_codex_detailed(None); - assert_eq!(fresh_stats.files_parsed, 1); - assert_eq!(resumed.input_tokens, fresh.input_tokens); - assert_eq!(resumed.cached_tokens, fresh.cached_tokens); - assert_eq!(resumed.output_tokens, fresh.output_tokens); - assert_eq!(resumed.sessions_count, fresh.sessions_count); - assert_eq!(resumed.by_model_tokens, fresh.by_model_tokens); - assert_eq!(resumed.by_model.len(), fresh.by_model.len()); - for (model, resumed_cost) in &resumed.by_model { - let fresh_cost = fresh.by_model.get(model).copied().expect("fresh model row"); - assert!((resumed_cost - fresh_cost).abs() < 1e-12); - } - assert!((resumed.total_cost_usd - fresh.total_cost_usd).abs() < 1e-12); -} - -#[test] -fn bounded_growing_rollout_freezes_target_and_resumes_a_retained_tail() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = - write_codex_session_fixture_with_inputs(&sessions, "growing.jsonl", &[100, 200, 300]); - let initial_size = i64::try_from(std::fs::metadata(&path).unwrap().len()) - .expect("fixture file length fits i64"); - let first_line_bytes = i64::try_from( - std::fs::read(&path) - .unwrap() - .split(|byte| *byte == b'\n') - .next() - .unwrap() - .len(), - ) - .expect("fixture line length fits i64") - + 1; - - let mut options = CostScanOptions::app_driven(); - options.codex_max_session_file_bytes = first_line_bytes; - options.codex_max_scan_bytes_per_refresh = first_line_bytes; - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - - let (first, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(first.input_tokens, 100); - let first_usage = cached_file(&first_cache, &path); - assert_eq!(first_usage.codex_scan_target_size, Some(initial_size)); - assert_eq!(first_usage.parsed_bytes, Some(first_line_bytes)); - assert!(first_cache.codex_scan_incomplete); - - let timestamp = (Utc::now() - Duration::minutes(10)) - .format("%Y-%m-%dT%H:%M:%S%.3fZ") - .to_string(); - let append_line = |total: u64| { - format!( - r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":{total},"cached_input_tokens":0,"output_tokens":{}}}}}}}}} -"#, - total / 10 - ) - }; - let mut next_total = 400_u64; - let mut bounded_summary = first; - let mut bounded_cache = first_cache; - for _ in 0..8 { - let line = append_line(next_total); - next_total += 100; - let mut file = std::fs::OpenOptions::new() - .append(true) - .open(&path) - .unwrap(); - file.write_all(line.as_bytes()).unwrap(); - drop(file); - - (bounded_summary, _, bounded_cache) = scanner.scan_codex_detailed_with_cache(None); - let usage = cached_file(&bounded_cache, &path); - assert_eq!(usage.codex_scan_target_size, Some(initial_size)); - assert!(usage.parsed_bytes.unwrap_or_default() <= initial_size); - if usage.parsed_bytes == Some(initial_size) { - break; - } - } - - assert_eq!(bounded_summary.input_tokens, 300); - let bounded_usage = cached_file(&bounded_cache, &path); - assert_eq!(bounded_usage.parsed_bytes, Some(initial_size)); - assert_eq!(bounded_usage.codex_scan_target_size, Some(initial_size)); - assert!( - bounded_cache.codex_scan_incomplete, - "the appended tail stays queued" - ); - - for _ in 0..32 { - if !bounded_cache.codex_scan_incomplete { - break; - } - (bounded_summary, _, bounded_cache) = scanner.scan_codex_detailed_with_cache(None); - } - assert!(!bounded_cache.codex_scan_incomplete); - let stable_summary = bounded_summary.clone(); - let stable_size = i64::try_from(std::fs::metadata(&path).unwrap().len()) - .expect("fixture file length fits i64"); - - let partial_line = append_line(next_total); - let split = partial_line.len() / 2; - let mut file = std::fs::OpenOptions::new() - .append(true) - .open(&path) - .unwrap(); - file.write_all(&partial_line.as_bytes()[..split]).unwrap(); - drop(file); - let (partial_summary, _, partial_cache) = scanner.scan_codex_detailed_with_cache(None); - let partial_usage = cached_file(&partial_cache, &path); - assert_eq!(partial_summary.input_tokens, stable_summary.input_tokens); - assert_eq!(partial_usage.parsed_bytes, Some(stable_size)); - assert_eq!(partial_usage.codex_scan_target_size, Some(stable_size)); - assert!(partial_cache.codex_scan_incomplete); - - let mut file = std::fs::OpenOptions::new() - .append(true) - .open(&path) - .unwrap(); - file.write_all(&partial_line.as_bytes()[split..]).unwrap(); - drop(file); - next_total += 100; - let (resumed_summary, _, resumed_cache) = scanner.scan_codex_detailed_with_cache(None); - assert!(!resumed_cache.codex_scan_incomplete); - assert_eq!(resumed_summary.input_tokens, next_total - 100); - let resumed_usage = cached_file(&resumed_cache, &path); - assert_eq!( - resumed_usage.parsed_bytes, - Some( - i64::try_from(std::fs::metadata(&path).unwrap().len()) - .expect("fixture file length fits i64"), - ) - ); - assert_eq!( - resumed_usage.codex_scan_target_size, - resumed_usage.parsed_bytes - ); -} - -#[test] -fn cost_scan_midline_rewrite_forces_full_parse_not_resume() { - // F2 (upstream 0.48.0 #2648): when a file is rewritten/truncated so the - // cached resume offset is now mid-line (byte before offset is not \n), - // the scanner must fall through to a full re-parse from offset 0 rather - // than resuming from the stale mid-line offset. - let (_root, sessions, cache_root) = codex_scan_dirs(); - let _path = write_codex_session_fixture(&sessions, "a.jsonl", 100); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (s1, st1) = scanner.scan_codex_detailed(None); - assert_eq!(st1.files_parsed, 1); - assert_eq!(s1.input_tokens, 100); - - // Rewrite the file with a shorter body at the same path so the cached - // parsed_bytes offset now points mid-line in the new content. - let today = Local::now().date_naive(); - let day_dir = partition_dir(&sessions, today); - let ts = (Utc::now() - Duration::minutes(30)) - .format("%Y-%m-%dT%H:%M:%S%.3fZ") - .to_string(); - // Shorter content with different token count — the cached offset will - // be past EOF or mid-line in this new content. - let body = format!( - r#"{{"timestamp":"{ts}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":50,"cached_input_tokens":0,"output_tokens":5}}}}}}}} -"# - ); - std::fs::write(day_dir.join("a.jsonl"), body).unwrap(); - - let (s2, st2) = scanner.scan_codex_detailed(None); - // The scanner must full-parse (not resume) because the cached offset - // no longer sits on a line boundary in the rewritten content. - assert!( - st2.files_parsed >= 1 || st2.files_resumed == 0, - "midline rewrite forces full parse, not resume (parsed={}, resumed={})", - st2.files_parsed, - st2.files_resumed - ); - assert_eq!(s2.input_tokens, 50, "full parse picks up new token count"); -} - -#[test] -fn previous_report_clears_after_successful_full_scan() { - // F8 (upstream 0.48.0): a completed full scan clears previous_report so - // the refreshing indicator does not stay permanently on. - let (_root, sessions, cache_root) = codex_scan_dirs(); - write_codex_session_fixture(&sessions, "a.jsonl", 100); - - let scanner = app_scanner(7, &cache_root, &sessions); - - // First scan: builds cache fresh; no previous_report expected. - let (summary1, _) = scanner.scan_codex_detailed(None); - assert!(summary1.history_coverage_established); - let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!( - cache.previous_report.is_none(), - "first scan clears previous_report" - ); - - // Inject a previous_report to simulate trim-set catch-up. - let mut cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - cache.previous_report = Some(crate::core::CachedCostReport { - total_cost_usd: 0.0, - input_tokens: 0, - cached_tokens: 0, - output_tokens: 0, - reasoning_tokens: None, - sessions_count: 0, - updated_at: None, - partial: false, - }); - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - - // Verify the cache now has previous_report set. - let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!( - cache.previous_report.is_some(), - "injected previous_report persists" - ); - - // Full scan with app_driven clears previous_report on success. - let (summary2, _) = scanner.scan_codex_detailed(None); - assert!( - summary2.history_coverage_established, - "after full scan coverage is established" - ); - - let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!( - cache.previous_report.is_none(), - "full scan clears previous_report" - ); -} - -// ── Upstream 0.50.1 #2932: known-zero history ──────────────────────────── - -#[test] -fn known_zero_is_set_when_scan_completes_with_no_sessions() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - std::fs::create_dir_all(&sessions).unwrap(); - - let scanner = app_scanner(7, &cache_root, &sessions); - - let (summary, _) = scanner.scan_codex_detailed(None); - assert!(summary.history_coverage_established, "scan completed"); - assert_eq!(summary.sessions_count, 0, "no sessions"); - assert!(summary.known_zero, "completed scan with zero = known-zero"); -} - -#[test] -fn known_zero_is_not_set_when_scan_has_results() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - write_codex_session_fixture(&sessions, "a.jsonl", 100); - - let scanner = app_scanner(7, &cache_root, &sessions); - - let (summary, _) = scanner.scan_codex_detailed(None); - assert!(summary.history_coverage_established); - assert_eq!(summary.sessions_count, 1); - assert!(!summary.known_zero, "scan with results is not known-zero"); -} - -#[test] -fn tiny_candidate_limit_prefers_newest_dirty_file_and_persists_older_pending() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let older = write_codex_session_fixture(&sessions, "a-older.jsonl", 100); - let newer = write_codex_session_fixture(&sessions, "z-newer.jsonl", 200); - - let mut options = CostScanOptions::app_driven(); - options.codex_candidate_limit = 1; - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - - let (first, first_stats, first_cache) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(first_stats.files_parsed, 1); - assert_eq!( - first.input_tokens, 200, - "newest dirty file is processed first" - ); - assert!(first_cache.codex_scan_incomplete); - assert_eq!( - first_cache.codex_pending_paths, - vec![older.to_string_lossy().to_string()] - ); - assert!( - !first_cache - .codex_pending_paths - .contains(&newer.to_string_lossy().to_string()) - ); - - let (second, _, second_cache) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(second.input_tokens, 300); - assert!(!second_cache.codex_scan_incomplete); - assert!(second_cache.codex_pending_paths.is_empty()); -} - -#[test] -fn tiny_byte_limit_resumes_and_drains_to_unbounded_totals() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let cache_root = root.path().join("bounded-cache"); - let path = write_codex_session_fixture_with_inputs(&sessions, "multi.jsonl", &[100, 200, 300]); - let first_line_bytes = i64::try_from( - std::fs::read(&path) - .unwrap() - .split(|byte| *byte == b'\n') - .next() - .unwrap() - .len(), - ) - .expect("fixture line length fits i64") - + 1; - - let mut bounded_options = CostScanOptions::app_driven(); - bounded_options.codex_max_session_file_bytes = first_line_bytes; - bounded_options.codex_max_scan_bytes_per_refresh = first_line_bytes; - let bounded = CostScanner::new(7) - .with_options(bounded_options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - - let (first, first_stats, first_cache) = bounded.scan_codex_detailed_with_cache(None); - assert_eq!(first_stats.codex_bytes_read, first_line_bytes as u64); - assert!(first_stats.files_deferred > 0); - assert!(first_cache.codex_scan_incomplete); - assert!( - first_cache - .files - .get(&path.to_string_lossy().to_string()) - .expect("partial cache entry") - .parsed_bytes - .unwrap_or(0) - < i64::try_from(std::fs::metadata(&path).unwrap().len()) - .expect("fixture file length fits i64") - ); - assert!(!first.history_coverage_established); - - let mut final_bounded = None; - let mut saw_resume = false; - for _ in 0..8 { - let (summary, stats, cache) = bounded.scan_codex_detailed_with_cache(None); - saw_resume |= stats.files_resumed > 0; - if !cache.codex_scan_incomplete { - final_bounded = Some(summary); - break; - } - } - let final_bounded = final_bounded.expect("bounded passes drain"); - assert!(saw_resume, "later passes resume the cached prefix"); - - let full = app_scanner(7, root.path().join("full-cache"), &sessions); - let (full_summary, _, full_cache) = full.scan_codex_detailed_with_cache(None); - assert!(!full_cache.codex_scan_incomplete); - assert_eq!(final_bounded.input_tokens, full_summary.input_tokens); - assert_eq!(final_bounded.cached_tokens, full_summary.cached_tokens); - assert_eq!(final_bounded.output_tokens, full_summary.output_tokens); - assert_eq!(final_bounded.sessions_count, full_summary.sessions_count); - assert_eq!(final_bounded.by_model_tokens, full_summary.by_model_tokens); - assert!((final_bounded.total_cost_usd - full_summary.total_cost_usd).abs() < 1e-12); -} - -#[test] -fn incomplete_summary_preserves_previous_report_and_marks_it_non_authoritative() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - write_codex_session_fixture_with_inputs(&sessions, "multi.jsonl", &[100, 200]); - - let report = CachedCostReport { - total_cost_usd: 42.5, - input_tokens: 11, - cached_tokens: 2, - output_tokens: 3, - reasoning_tokens: Some(7), - sessions_count: 7, - updated_at: Some("2026-09-06T00:00:00Z".to_string()), - partial: false, - }; - let mut cache = CostUsageCache { - previous_report: Some(report.clone()), - codex_scan_incomplete: true, - ..Default::default() - }; - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - - let scanner = app_scanner(7, &cache_root, &sessions); - let cancel = AtomicBool::new(true); - let (summary, _, saved) = scanner.scan_codex_detailed_with_cache(Some(&cancel)); - - assert_eq!(summary.total_cost_usd, report.total_cost_usd); - assert_eq!(summary.input_tokens, report.input_tokens as u64); - assert_eq!(summary.cached_tokens, report.cached_tokens as u64); - assert_eq!(summary.output_tokens, report.output_tokens as u64); - assert_eq!(summary.reasoning_tokens, Some(7)); - assert_eq!(summary.sessions_count, report.sessions_count as u32); - assert!(!summary.history_coverage_established); - assert!(!summary.known_zero); - assert!(summary.model_pricing_completeness.is_partial()); - assert_eq!( - saved.previous_report.map(|saved| saved.total_cost_usd), - Some(42.5) - ); - assert!(saved.codex_scan_incomplete); -} - -#[test] -fn failed_catch_up_pause_preserves_cursor_and_report_until_explicit_refresh() { - let (root, sessions, cache_root) = codex_scan_dirs(); - let pending = write_codex_session_fixture(&sessions, "pending.jsonl", 100); - let report = CachedCostReport { - total_cost_usd: 42.5, - input_tokens: 11, - cached_tokens: 2, - output_tokens: 3, - reasoning_tokens: Some(7), - sessions_count: 7, - updated_at: Some("2026-09-06T00:00:00Z".to_string()), - partial: false, - }; - let mut cache = CostUsageCache { - previous_report: Some(report.clone()), - codex_pending_paths: vec![pending.to_string_lossy().to_string()], - codex_scan_incomplete: true, - ..Default::default() - }; - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - - let failed = CostScanner::new(7) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![root.path().join("temporarily-unavailable")]); - let (failed_summary, _, failed_cache) = failed.scan_codex_detailed_with_cache(None); - assert_eq!( - failed_cache.codex_scan_pause_reason, - Some(CodexScanPauseReason::Error( - "Codex session source unavailable".to_string() - )) - ); - assert_eq!(failed_cache.codex_pending_paths, cache.codex_pending_paths); - assert_eq!( - failed_cache - .previous_report - .as_ref() - .map(|saved| saved.total_cost_usd), - Some(report.total_cost_usd) - ); - assert_eq!(failed_summary.total_cost_usd, report.total_cost_usd); - - let background = CostScanner::new(7) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (background_summary, background_stats, background_cache) = - background.scan_codex_detailed_with_cache(None); - assert_eq!(background_stats.files_parsed, 0); - assert_eq!(background_summary.total_cost_usd, report.total_cost_usd); - assert_eq!( - background_cache.codex_pending_paths, - failed_cache.codex_pending_paths - ); - assert_eq!( - background_cache.codex_scan_pause_reason, - failed_cache.codex_scan_pause_reason - ); - - let explicit = app_scanner(7, &cache_root, &sessions); - let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); - assert!(resumed_cache.codex_scan_pause_reason.is_none()); - assert!(!resumed_cache.codex_scan_incomplete); - assert!(resumed_cache.codex_pending_paths.is_empty()); - assert!(resumed_summary.history_coverage_established); - assert_eq!(resumed_summary.input_tokens, 100); - assert!(resumed_cache.previous_report.is_none()); -} - -#[test] -fn missing_unobserved_sessions_root_does_not_pause_validated_cache() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let missing = root.path().join("optional-sessions"); - let cache_root = root.path().join("cache"); - write_codex_session_fixture(&sessions, "observed.jsonl", 100); - - let initial = app_scanner(7, &cache_root, &sessions); - let (initial_summary, _) = initial.scan_codex_detailed(None); - assert!(initial_summary.history_coverage_established); - - let mut cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - cache.last_scan_unix_ms = 1; - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - - let background = CostScanner::new(7) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions, missing]); - let (summary, _, saved) = background.scan_codex_detailed_with_cache(None); - - assert!(summary.history_coverage_established); - assert_eq!(summary.input_tokens, 100); - assert!(!saved.codex_scan_incomplete); - assert!(saved.codex_scan_pause_reason.is_none()); -} - -#[test] -fn trace_pruning_preserves_cursor_and_validated_history_until_refresh() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "pruned.jsonl", 100); - - let initial = app_scanner(7, &cache_root, &sessions); - let (initial_summary, _, initial_cache) = initial.scan_codex_detailed_with_cache(None); - assert_eq!(initial_summary.input_tokens, 100); - assert!(!initial_cache.codex_scan_incomplete); - - // Keep the next pass outside the scanner debounce while simulating Codex - // retention pruning the same trace file down to a smaller valid payload. - let mut initial_cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - initial_cache.last_scan_unix_ms = 1; - JsonlScanner::save_cache(ProviderId::Codex, &mut initial_cache, Some(&cache_root)); - let _ = write_codex_session_fixture(&sessions, "pruned.jsonl", 1); - - let background = CostScanner::new(7) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (paused_summary, _, paused_cache) = background.scan_codex_detailed_with_cache(None); - assert_eq!(paused_summary.input_tokens, 100); - assert_eq!( - paused_cache.codex_scan_pause_reason, - Some(CodexScanPauseReason::NoProgress) - ); - assert_eq!( - paused_cache.codex_pending_paths, - vec![path.to_string_lossy().to_string()] - ); - assert!(paused_cache.previous_report.is_some()); - - let explicit = app_scanner(7, &cache_root, &sessions); - let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); - assert_eq!(resumed_summary.input_tokens, 1); - assert!(resumed_cache.codex_scan_pause_reason.is_none()); - assert!(resumed_cache.codex_pending_paths.is_empty()); - assert!(resumed_cache.previous_report.is_none()); -} - -fn expected_codex_scan_start(days: u32) -> String { - let today = Local::now().date_naive(); - let report_start = today - Duration::days(i64::from(days.saturating_sub(1))); - CostUsageDayRange::new(report_start, today).scan_since_key -} - -fn pending_codex_options(force: bool) -> CostScanOptions { - let mut options = if force { - CostScanOptions::app_driven() - } else { - CostScanOptions::default() - }; - options.codex_candidate_limit = 1; - options -} - -#[test] -fn pending_codex_scan_30_to_7_keeps_wide_start_and_narrow_report() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - for index in 0..3 { - write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); - } - - let first = CostScanner::new(30) - .with_options(pending_codex_options(true)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); - assert!(first_cache.codex_scan_incomplete); - - let second = CostScanner::new(7) - .with_options(pending_codex_options(false)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - let (summary, _, second_cache) = second.scan_codex_detailed_with_cache(None); - let today = Local::now().date_naive(); - assert!(second_cache.codex_scan_incomplete); - assert_eq!( - second_cache.codex_pending_scan_since_key.as_deref(), - Some(expected_codex_scan_start(30).as_str()) - ); - assert_eq!(summary.period_start, Some(today - Duration::days(6))); - assert_eq!(summary.period_end, Some(today)); -} - -#[test] -fn pending_codex_scan_7_to_30_expands_to_earliest_start() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - for index in 0..3 { - write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); - } - - let first = CostScanner::new(7) - .with_options(pending_codex_options(true)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); - assert!(first_cache.codex_scan_incomplete); - - let second = CostScanner::new(30) - .with_options(pending_codex_options(false)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - let (_, _, second_cache) = second.scan_codex_detailed_with_cache(None); - assert!(second_cache.codex_scan_incomplete); - assert_eq!( - second_cache.codex_pending_scan_since_key.as_deref(), - Some(expected_codex_scan_start(30).as_str()) - ); -} - -#[test] -fn pending_codex_scan_repeated_narrow_wide_alternation_is_monotonic() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - for index in 0..8 { - write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); - } - - let first = CostScanner::new(30) - .with_options(pending_codex_options(true)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); - assert!(first_cache.codex_scan_incomplete); - - for days in [7, 30, 7, 30, 7] { - let scanner = CostScanner::new(days) - .with_options(pending_codex_options(false)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); - assert!(cache.codex_scan_incomplete); - assert_eq!( - cache.codex_pending_scan_since_key.as_deref(), - Some(expected_codex_scan_start(30).as_str()) - ); - } -} - -#[test] -fn pending_codex_scan_incompatible_root_timezone_or_end_resets_start() { - for incompatibility in ["root", "timezone", "end"] { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - let alternate_sessions = root.path().join("alternate-sessions"); - let cache_root = root.path().join("cache"); - for index in 0..3 { - write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); - write_codex_session_fixture( - &alternate_sessions, - &format!("alternate-{index}.jsonl"), - 200 + index, - ); - } - - let first = CostScanner::new(30) - .with_options(pending_codex_options(true)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); - assert!(first_cache.codex_scan_incomplete); - - let mut persisted = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - match incompatibility { - "root" => {} - "timezone" => { - persisted.codex_pending_scan_timezone = Some("not-the-local-zone".to_string()) - } - "end" => persisted.codex_pending_scan_until_key = Some("2099-01-01".to_string()), - _ => unreachable!(), - } - JsonlScanner::save_cache(ProviderId::Codex, &mut persisted, Some(&cache_root)); - - let roots = if incompatibility == "root" { - vec![alternate_sessions] - } else { - vec![sessions] - }; - let second = CostScanner::new(7) - .with_options(pending_codex_options(false)) - .with_cache_root(&cache_root) - .with_sessions_dirs(roots); - let (_, _, second_cache) = second.scan_codex_detailed_with_cache(None); - assert!(second_cache.codex_scan_incomplete); - assert_eq!( - second_cache.codex_pending_scan_since_key.as_deref(), - Some(expected_codex_scan_start(7).as_str()), - "{incompatibility} context must reset the pending start" - ); - } -} - -#[test] -fn pending_codex_scan_force_rescan_resets_start() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - for index in 0..3 { - write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); - } - - let first = CostScanner::new(30) - .with_options(pending_codex_options(true)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); - assert!(first_cache.codex_scan_incomplete); - - let forced = CostScanner::new(7) - .with_options(pending_codex_options(true)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - let (_, _, forced_cache) = forced.scan_codex_detailed_with_cache(None); - assert!(forced_cache.codex_scan_incomplete); - assert_eq!( - forced_cache.codex_pending_scan_since_key.as_deref(), - Some(expected_codex_scan_start(7).as_str()) - ); -} - -#[test] -fn pending_codex_scan_start_survives_restart() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - for index in 0..3 { - write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); - } - - let first = CostScanner::new(30) - .with_options(pending_codex_options(true)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); - assert_eq!( - first_cache.codex_pending_scan_since_key.as_deref(), - Some(expected_codex_scan_start(30).as_str()) - ); - - let persisted = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert_eq!( - persisted.codex_pending_scan_since_key.as_deref(), - Some(expected_codex_scan_start(30).as_str()) - ); - - let restarted = CostScanner::new(7) - .with_options(pending_codex_options(false)) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - let (_, _, restarted_cache) = restarted.scan_codex_detailed_with_cache(None); - assert_eq!( - restarted_cache.codex_pending_scan_since_key.as_deref(), - Some(expected_codex_scan_start(30).as_str()) - ); -} - -#[test] -fn disappeared_trace_path_waits_for_explicit_validation() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "disappeared.jsonl", 100); - - let initial = app_scanner(7, &cache_root, &sessions); - let (initial_summary, _, _) = initial.scan_codex_detailed_with_cache(None); - assert_eq!(initial_summary.input_tokens, 100); - - let mut cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - cache.last_scan_unix_ms = 1; - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - std::fs::remove_file(&path).unwrap(); - - let background = CostScanner::new(7) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]); - let (paused_summary, _, paused_cache) = background.scan_codex_detailed_with_cache(None); - assert_eq!(paused_summary.input_tokens, 100); - assert_eq!( - paused_cache.codex_scan_pause_reason, - Some(CodexScanPauseReason::NoProgress) - ); - assert_eq!( - paused_cache.codex_pending_paths, - vec![path.to_string_lossy().to_string()] - ); - - let explicit = app_scanner(7, &cache_root, &sessions); - let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); - assert_eq!(resumed_summary.sessions_count, 0); - assert!(resumed_summary.history_coverage_established); - assert!(resumed_cache.codex_scan_pause_reason.is_none()); - assert!(resumed_cache.codex_pending_paths.is_empty()); - assert!(resumed_cache.previous_report.is_none()); -} - -#[test] -fn paused_catch_up_round_trips_without_retrying_in_background() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let pending = write_codex_session_fixture(&sessions, "pending.jsonl", 100); - let report = CachedCostReport { - total_cost_usd: 9.25, - input_tokens: 21, - cached_tokens: 1, - output_tokens: 5, - reasoning_tokens: None, - sessions_count: 2, - updated_at: Some("2026-09-06T00:00:00Z".to_string()), - partial: true, - }; - let mut cache = CostUsageCache { - previous_report: Some(report.clone()), - codex_pending_paths: vec![pending.to_string_lossy().to_string()], - codex_scan_incomplete: true, - codex_scan_pause_reason: Some(CodexScanPauseReason::NoProgress), - ..Default::default() - }; - - let encoded = serde_json::to_string(&cache).unwrap(); - let decoded: CostUsageCache = serde_json::from_str(&encoded).unwrap(); - assert_eq!( - decoded.codex_scan_pause_reason, - cache.codex_scan_pause_reason - ); - - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - let scanner = CostScanner::new(7) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - let (summary, stats, saved) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(stats.files_parsed, 0); - assert_eq!(summary.total_cost_usd, report.total_cost_usd); - assert_eq!(saved.codex_pending_paths, cache.codex_pending_paths); - assert_eq!( - saved - .previous_report - .as_ref() - .map(|saved| saved.total_cost_usd), - Some(report.total_cost_usd) - ); - assert_eq!(saved.codex_scan_pause_reason, cache.codex_scan_pause_reason); -} - -#[test] -fn pending_and_incomplete_round_trip_through_cache_json() { - let cache = CostUsageCache { - codex_pending_paths: vec!["C:\\sessions\\pending.jsonl".to_string()], - codex_scan_incomplete: true, - ..Default::default() - }; - - let encoded = serde_json::to_string(&cache).unwrap(); - let decoded: CostUsageCache = serde_json::from_str(&encoded).unwrap(); - assert_eq!(decoded.codex_pending_paths, cache.codex_pending_paths); - assert!(decoded.codex_scan_incomplete); -} - -#[test] -fn deleted_pending_path_is_pruned_after_complete_discovery() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture_with_inputs(&sessions, "partial.jsonl", &[100, 200]); - let first_line_bytes = i64::try_from( - std::fs::read(&path) - .unwrap() - .split(|byte| *byte == b'\n') - .next() - .unwrap() - .len(), - ) - .expect("fixture line length fits i64") - + 1; - let mut options = CostScanOptions::app_driven(); - options.codex_max_session_file_bytes = first_line_bytes; - options.codex_max_scan_bytes_per_refresh = first_line_bytes; - let scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions]); - let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); - assert!(first_cache.codex_scan_incomplete); - assert_eq!(first_cache.codex_pending_paths.len(), 1); - - std::fs::remove_file(&path).unwrap(); - let (summary, _, second_cache) = scanner.scan_codex_detailed_with_cache(None); - assert_eq!(summary.sessions_count, 0); - assert!(summary.history_coverage_established); - assert!(second_cache.codex_pending_paths.is_empty()); - assert!(!second_cache.codex_scan_incomplete); - assert!( - !second_cache - .files - .contains_key(&path.to_string_lossy().to_string()) - ); - assert!(second_cache.days.is_empty()); -} - -#[test] -fn legacy_cache_json_defaults_bounded_scan_state() { - let legacy = r#"{"last_scan_unix_ms":0,"files":{},"days":{}}"#; - let cache: CostUsageCache = serde_json::from_str(legacy).unwrap(); - assert!(cache.codex_pending_paths.is_empty()); - assert!(!cache.codex_scan_incomplete); -} - -#[test] -fn complete_empty_codex_fragment_persists_in_cache() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "empty.jsonl", 100); - std::fs::write(&path, b"\n").unwrap(); - let key = path.to_string_lossy().to_string(); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); - - assert_eq!(summary.input_tokens, 0); - let entry = cache.files.get(&key).expect("empty fragment is cached"); - assert!(entry.days.is_empty()); - assert_eq!(entry.parsed_bytes, Some(1)); - assert_eq!(entry.codex_scan_target_size, Some(1)); - - let persisted = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!(persisted.files.contains_key(&key)); -} - -#[test] -fn complete_empty_codex_fragment_reparses_from_start_after_growth() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "empty.jsonl", 100); - std::fs::write(&path, b"\n").unwrap(); - let key = path.to_string_lossy().to_string(); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); - let first = first_cache.files.get(&key).expect("initial empty fragment"); - assert_eq!(first.parsed_bytes, Some(1)); - assert_eq!(first.codex_scan_target_size, Some(1)); - - write_codex_session_fixture(&sessions, "empty.jsonl", 100); - let (grown_summary, stats, grown_cache) = scanner.scan_codex_detailed_with_cache(None); - - assert_eq!(grown_summary.input_tokens, 100); - assert_eq!(stats.files_resumed, 0); - assert!(!grown_cache.files[&key].days.is_empty()); -} - -#[test] -fn incomplete_or_buffered_empty_codex_fragment_is_not_marked_complete() { - let (_root, sessions, cache_root) = codex_scan_dirs(); - let path = write_codex_session_fixture(&sessions, "incomplete.jsonl", 100); - std::fs::write(&path, br#"{"timestamp":"2026-09-07T00:00:00Z""#).unwrap(); - let key = path.to_string_lossy().to_string(); - - let scanner = app_scanner(7, &cache_root, &sessions); - let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); - - let entry = cache - .files - .get(&key) - .expect("incomplete fragment is tracked"); - assert!(entry.days.is_empty()); - assert_ne!(entry.parsed_bytes, Some(entry.size)); - assert!(cache.codex_scan_incomplete); - assert!(cache.codex_pending_paths.contains(&key)); - - let buffered_root = tempfile::tempdir().unwrap(); - let buffered_sessions = buffered_root.path().join("sessions"); - let buffered_cache_root = buffered_root.path().join("cache"); - let buffered_path = write_codex_session_fixture(&buffered_sessions, "buffered.jsonl", 100); - std::fs::write(&buffered_path, b"\nnot-yet-read").unwrap(); - let buffered_key = buffered_path.to_string_lossy().to_string(); - let mut options = CostScanOptions::app_driven(); - options.codex_max_session_file_bytes = 1; - options.codex_max_scan_bytes_per_refresh = 1; - let buffered_scanner = CostScanner::new(7) - .with_options(options) - .with_cache_root(&buffered_cache_root) - .with_sessions_dirs(vec![buffered_sessions]); - let (_, _, buffered_cache) = buffered_scanner.scan_codex_detailed_with_cache(None); - - let buffered_entry = buffered_cache - .files - .get(&buffered_key) - .expect("buffered fragment is tracked"); - assert!(buffered_entry.days.is_empty()); - assert_ne!(buffered_entry.parsed_bytes, Some(buffered_entry.size)); - assert!(buffered_cache.codex_scan_incomplete); - assert!(buffered_cache.codex_pending_paths.contains(&buffered_key)); -} - -// Regression (PR #611 review): Codex `input_tokens` already contains cached -// input, Claude cache buckets are separate. Every total must follow the -// provider's rule, and the model, daily and window totals must agree. -#[test] -fn provider_totals_add_cache_only_when_it_is_separate_from_input() { - let counts = ModelTokenCounts { - input_tokens: 100, - output_tokens: 5, - cached_tokens: 90, - reasoning_tokens: None, - }; - assert_eq!(counts.total_for_provider("codex"), 105); - assert_eq!(counts.total_for_provider("claude"), 195); - - let summary = CostSummary { - input_tokens: 100, - output_tokens: 5, - cached_tokens: 90, - ..CostSummary::default() - }; - assert_eq!(summary.total_tokens_for_provider("codex"), 105); - assert_eq!(summary.total_tokens_for_provider("claude"), 195); -} - -#[test] -fn codex_day_total_excludes_cached_input_and_matches_model_totals() { - let day_key = "2026-08-18".to_string(); - let day = CostUsageDayRange::parse_day_key(&day_key).expect("day"); - let range = CostUsageDayRange::new(day, day); - let mut one_day = HashMap::new(); - one_day.insert( - day_key, - HashMap::from([("gpt-5".to_string(), vec![1_000, 900, 50])]), - ); - let mut scratch = CostSummary::default(); - add_codex_days_map_to_summary(&mut scratch, &one_day, &range); - - assert_eq!(scratch.input_tokens, 1_000); - assert_eq!(scratch.cached_tokens, 900); - // The value `get_daily_token_history("codex")` stores for the day. - let day_total = scratch.total_tokens_for_provider("codex"); - assert_eq!(day_total, 1_050, "cached input is already inside input"); - let model_total: u64 = scratch - .by_model_tokens - .values() - .map(|counts| counts.total_for_provider("codex")) - .sum(); - assert_eq!(model_total, day_total); -} - -#[test] -fn claude_day_total_includes_cache_and_matches_summary_and_model_totals() { - use chrono::TimeZone as _; - let timestamp = Local - .with_ymd_and_hms(2026, 8, 18, 12, 0, 0) - .single() - .expect("local time") - .with_timezone(&Utc); - let record = ClaudeUsageRecord { - model: "claude-sonnet-4-5".to_string(), - pricing_known: true, - timestamp: Some(timestamp), - dedup_key: None, - input: 10, - output: 20, - cache_create: 300, - cache_read: 4_000, - cost: Some(0.0), - }; - let mut summary = CostSummary::default(); - assert!(add_claude_record_to_summary(&mut summary, &record)); - let mut daily = HashMap::from([("2026-08-18".to_string(), 0u64)]); - assert!(add_claude_record_to_daily_tokens(&mut daily, &record)); - - let window_total = summary.total_tokens_for_provider("claude"); - assert_eq!(window_total, 4_330); - assert_eq!(daily["2026-08-18"], window_total); - let model_total: u64 = summary - .by_model_tokens - .values() - .map(|counts| counts.total_for_provider("claude")) - .sum(); - assert_eq!(model_total, window_total); -} +use support::{ + app_scanner, cached_file, cached_input_total, cached_usage_with_packed, + claude_usage_record_from_event, codex_scan_dirs, for_each_claude_usage_record, partition_dir, + recent_codex_fixture_time, recent_fixture_time_at, write_codex_fork_session_fixture, + write_codex_session_fixture, write_codex_session_fixture_with_inputs, +}; #[path = "tests/claude_swap.rs"] mod claude_swap; @@ -3423,54 +46,29 @@ mod period; #[path = "tests/archived.rs"] mod archived; -/// #755: a background scan paused with `no_progress` and no retained report -/// used to sum every cached day, so Today and 30d showed the same totals. -#[test] -fn paused_scan_without_previous_report_reports_only_the_requested_days() { - let root = tempfile::tempdir().unwrap(); - let sessions = root.path().join("sessions"); - std::fs::create_dir_all(&sessions).unwrap(); - let cache_root = root.path().join("cache"); - let today = Local::now().date_naive(); - let older = (today - chrono::Duration::days(10)) - .format("%Y-%m-%d") - .to_string(); - let today = today.format("%Y-%m-%d").to_string(); - let mut cache = CostUsageCache { - files: HashMap::from([ - ( - "today".to_string(), - cached_usage_with_packed(&today, "gpt-5.6-sol", vec![1_000, 400, 10]), - ), - ( - "older".to_string(), - cached_usage_with_packed(&older, "gpt-5.6-sol", vec![5_000, 2_000, 50]), - ), - ]), - // A queued file whose fork parent never resolved keeps the current - // window unestablished, as in the reported cache. - codex_pending_paths: vec![root.path().join("gone.jsonl").to_string_lossy().to_string()], - codex_scan_incomplete: true, - codex_scan_pause_reason: Some(CodexScanPauseReason::NoProgress), - ..Default::default() - }; - rebuild_cache_days(&mut cache); - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); +#[path = "tests/claude_rates.rs"] +mod claude_rates; + +#[path = "tests/codex_usage.rs"] +mod codex_usage; + +#[path = "tests/claude_records.rs"] +mod claude_records; + +#[path = "tests/claude_daily.rs"] +mod claude_daily; + +#[path = "tests/codex_fork.rs"] +mod codex_fork; + +#[path = "tests/codex_cache.rs"] +mod codex_cache; + +#[path = "tests/codex_bounded.rs"] +mod codex_bounded; - let scan = |days| { - CostScanner::new(days) - .with_cache_root(&cache_root) - .with_sessions_dirs(vec![sessions.clone()]) - .scan_codex_detailed_with_cache(None) - }; - let (today_summary, today_stats, _) = scan(1); - let (month_summary, month_stats, _) = scan(30); +#[path = "tests/codex_pending.rs"] +mod codex_pending; - assert_eq!(today_stats.files_parsed, 0); - assert_eq!(month_stats.files_parsed, 0); - assert_eq!(today_summary.input_tokens, 1_000); - assert_eq!(today_summary.output_tokens, 10); - assert_eq!(month_summary.input_tokens, 6_000); - assert_eq!(month_summary.output_tokens, 60); - assert!(today_summary.total_cost_usd < month_summary.total_cost_usd); -} +#[path = "tests/totals.rs"] +mod totals; diff --git a/rust/src/cost_scanner/tests/claude_daily.rs b/rust/src/cost_scanner/tests/claude_daily.rs new file mode 100644 index 0000000000..289f619962 --- /dev/null +++ b/rust/src/cost_scanner/tests/claude_daily.rs @@ -0,0 +1,511 @@ +//! Claude daily tokens and cost history, Vertex rows, oversized and non-finite rows. + +use super::*; + +/// One file's daily token scan, counting each record that cannot be +/// aggregated as the token history path does. +fn scan_claude_file_for_daily_tokens( + path: &Path, + cutoff: &DateTime, + seen: &mut HashSet, + pricing: &mut ClaudeScanPricingResolver, + daily_tokens: &mut HashMap, +) -> ClaudeFileScanResult { + let mut aggregation_failures = 0u32; + // Token history ignores incomplete-request markers. + let mut incomplete = ClaudeIncompleteTracker::default(); + let mut result = scan_claude_file_with_pricing( + path, + cutoff, + seen, + None, + pricing, + &mut incomplete, + |record| { + if record.timestamp.is_none() + || !add_claude_record_to_daily_tokens(daily_tokens, record) + { + aggregation_failures = aggregation_failures.saturating_add(1); + } + }, + ); + result.aggregation_failures = result + .aggregation_failures + .saturating_add(aggregation_failures); + result +} + +#[test] +fn claude_daily_token_coverage_requires_a_complete_valid_scan() { + let root = tempfile::tempdir().unwrap(); + let cutoff = Utc::now() - Duration::days(1); + let valid_path = root.path().join("valid.jsonl"); + let timestamp = Utc::now() - Duration::hours(1); + let today = timestamp + .with_timezone(&Local) + .date_naive() + .format("%Y-%m-%d") + .to_string(); + std::fs::write( + &valid_path, + format!( + "{}\n", + claude_transcript_line( + ×tamp.to_rfc3339(), + "requestId", + "req_valid", + "msg_valid" + ) + ), + ) + .unwrap(); + + let mut valid_tokens = HashMap::from([(today.clone(), 0)]); + let valid_result = scan_claude_file_for_daily_tokens( + &valid_path, + &cutoff, + &mut HashSet::new(), + &mut ClaudeScanPricingResolver::default(), + &mut valid_tokens, + ); + assert!(valid_result.is_complete()); + let mut covered_days = HashSet::new(); + mark_claude_daily_token_coverage(&mut covered_days, &valid_tokens, valid_result); + assert!(covered_days.contains(&today)); + + let assert_uncovered = |path: &Path| { + let mut daily_tokens = HashMap::from([(today.clone(), 0)]); + let result = scan_claude_file_for_daily_tokens( + path, + &cutoff, + &mut HashSet::new(), + &mut ClaudeScanPricingResolver::default(), + &mut daily_tokens, + ); + assert!(!result.is_complete()); + let mut covered_days = HashSet::from(["stale-coverage".to_string()]); + mark_claude_daily_token_coverage(&mut covered_days, &daily_tokens, result); + assert!(covered_days.is_empty()); + result + }; + + let malformed_path = root.path().join("malformed.jsonl"); + std::fs::write(&malformed_path, b"{malformed\n").unwrap(); + assert_eq!(assert_uncovered(&malformed_path).malformed_lines, 1); + + let incomplete_path = root.path().join("incomplete.jsonl"); + std::fs::write( + &incomplete_path, + r#"{"type":"assistant","message":{"id":"msg_preliminary","model":"gpt-5.6-sol","stop_reason":null,"usage":{"input_tokens":1000}}}"#, + ) + .unwrap(); + assert_eq!(assert_uncovered(&incomplete_path).incomplete_requests, 1); + + let missing_timestamp_path = root.path().join("missing-timestamp.jsonl"); + std::fs::write( + &missing_timestamp_path, + r#"{"type":"assistant","requestId":"req_no_timestamp","message":{"id":"msg_no_timestamp","model":"claude-sonnet-4-6","usage":{"input_tokens":1000,"output_tokens":500}}}"#, + ) + .unwrap(); + assert_eq!( + assert_uncovered(&missing_timestamp_path).aggregation_failures, + 1 + ); + + let unreadable_path = root.path().join("missing.jsonl"); + assert_eq!(assert_uncovered(&unreadable_path).read_failures, 1); + + let scanner = CostScanner::new(1); + let missing_directory = root.path().join("missing-directory"); + let traversal_read_failures = + scanner.walk_claude_files(&missing_directory, &cutoff, None, &mut |_| {}); + assert_eq!(traversal_read_failures, 1); + let mut covered_days = HashSet::from([today]); + mark_claude_daily_token_coverage( + &mut covered_days, + &valid_tokens, + ClaudeFileScanResult { + read_failures: traversal_read_failures, + ..ClaudeFileScanResult::default() + }, + ); + assert!(covered_days.is_empty()); +} + +#[test] +fn public_claude_daily_token_dispatch_reports_incomplete_fixture_scans() { + const CHILD_MARKER: &str = "CODEXBAR_CLAUDE_DAILY_TOKEN_TEST_CHILD"; + const CHILD_DONE: &str = "isolated Claude daily-history fixture verified"; + if std::env::var_os(CHILD_MARKER).is_some() { + let config_dir = std::env::var_os("CLAUDE_CONFIG_DIR") + .map(PathBuf::from) + .expect("child receives isolated Claude config directory"); + let projects_dir = config_dir.join("projects"); + let project_dir = projects_dir.join("fixture-project"); + let (complete_history, incomplete) = get_daily_token_history("claude", 1); + assert!(!incomplete, "valid fixture scan should establish coverage"); + assert!(complete_history.iter().any(|(_, tokens)| *tokens > 0)); + + std::fs::write(project_dir.join("malformed.jsonl"), b"{malformed\n").unwrap(); + let (partial_history, incomplete) = get_daily_token_history("claude", 1); + assert!( + incomplete, + "malformed fixture should leave coverage incomplete" + ); + assert_eq!(partial_history, complete_history); + println!("{CHILD_DONE}"); + return; + } + + let config_dir = tempfile::tempdir().unwrap(); + let project_dir = config_dir.path().join("projects").join("fixture-project"); + std::fs::create_dir_all(&project_dir).unwrap(); + let timestamp = Utc::now().to_rfc3339(); + std::fs::write( + project_dir.join("valid.jsonl"), + format!( + "{}\n", + claude_transcript_line(×tamp, "requestId", "req_public", "msg_public") + ), + ) + .unwrap(); + + let test_thread = std::thread::current(); + let test_name = test_thread.name().expect("test harness names this thread"); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", test_name, "--nocapture", "--test-threads=1"]) + .env(CHILD_MARKER, "1") + .env("CLAUDE_CONFIG_DIR", config_dir.path()) + .output() + .expect("spawn isolated exact-test child"); + assert!( + output.status.success() && String::from_utf8_lossy(&output.stdout).contains(CHILD_DONE), + "fixture child failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn classifies_vertex_ai_claude_metadata_without_changing_anthropic_rows() { + let cases = [ + ( + r#"{"type":"assistant","requestId":"req_vrtx_123","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":1}}}"#, + true, + ), + ( + r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_vrtx_123","model":"claude-sonnet-4-6","usage":{"input_tokens":1}}}"#, + true, + ), + ( + r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6@20260217","usage":{"input_tokens":1}}}"#, + true, + ), + ( + r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","metadata":{"provider":"Google-Vertex-AI"},"usage":{"input_tokens":1}}}"#, + true, + ), + ( + r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","content":[{"context":{"gcp_project":false}}],"usage":{"input_tokens":1}}}"#, + true, + ), + ( + r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":1}} ,"metadata":{"provider":"anthropic"}}"#, + false, + ), + ( + r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":1}} ,"metadata":{"provider":"gcp"}}"#, + false, + ), + ( + r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","content":[{"text":"vertex"}],"usage":{"input_tokens":1}}}"#, + false, + ), + ( + r#"{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"Claude-sonnet-4-6@20260217","usage":{"input_tokens":1}}}"#, + false, + ), + ]; + + for (json, expected) in cases { + let event: ClaudeEvent = serde_json::from_str(json).unwrap(); + assert_eq!(event.is_vertex_ai_usage_entry(), expected, "{json}"); + } +} + +#[test] +fn shared_claude_reader_excludes_vertex_rows_but_keeps_anthropic_usage() { + let path = std::env::temp_dir().join(format!( + "codexbar-claude-vertex-filter-{}.jsonl", + std::process::id() + )); + let timestamp = (Utc::now() - Duration::hours(1)).to_rfc3339(); + let anthropic = format!( + r#"{{"type":"assistant","timestamp":"{timestamp}","requestId":"req_anthropic","message":{{"id":"msg_anthropic","model":"claude-sonnet-4-6","usage":{{"input_tokens":10,"output_tokens":5}}}}}}"# + ); + let vertex = serde_json::json!({ + "type": "assistant", + "timestamp": timestamp, + "requestId": "req_vrtx_123", + "message": { + "id": "msg_vrtx_123", + "model": "claude-sonnet-4-6", + "usage": {"input_tokens": u64::MAX, "output_tokens": u64::MAX} + } + }) + .to_string(); + std::fs::write(&path, format!("{anthropic}\n{vertex}\n")).unwrap(); + + let cutoff = Utc::now() - Duration::days(30); + let mut seen = HashSet::new(); + let mut records = Vec::new(); + let counted = for_each_claude_usage_record(&path, &cutoff, &mut seen, None, |record| { + records.push((record.input, record.output)) + }); + + assert_eq!(counted, 1); + assert_eq!(records, vec![(10, 5)]); + let _removed = std::fs::remove_file(&path); +} + +#[test] +fn oversized_claude_history_preserves_independent_components_and_fails_closed() { + let first: ClaudeEvent = serde_json::from_str(&format!( + r#"{{"type":"assistant","timestamp":"2026-09-20T12:00:00Z","requestId":"req_overflow_1","message":{{"id":"msg_overflow_1","model":"claude-sonnet-4-6","usage":{{"input_tokens":{},"output_tokens":2}}}}}}"#, + u64::MAX + )) + .unwrap(); + let second: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","timestamp":"2026-09-20T12:01:00Z","requestId":"req_overflow_2","message":{"id":"msg_overflow_2","model":"claude-sonnet-4-6","usage":{"input_tokens":1,"output_tokens":3}}}"#, + ) + .unwrap(); + let first = claude_usage_record_from_event(&first).expect("first usage row"); + let second = claude_usage_record_from_event(&second).expect("second usage row"); + let mut summary = CostSummary::default(); + + assert!(add_claude_record_to_summary(&mut summary, &first)); + assert!(!add_claude_record_to_summary(&mut summary, &second)); + assert_eq!(summary.input_tokens, u64::MAX); + assert_eq!(summary.output_tokens, 5); + assert!(summary.total_cost_usd.is_finite()); + + finalize_claude_summary( + &mut summary, + true, + ClaudeFileScanResult { + counted: 2, + aggregation_failures: 1, + ..ClaudeFileScanResult::default() + }, + false, + ); + assert!(!summary.history_coverage_established); + assert!(!summary.known_zero); +} + +#[test] +fn oversized_single_claude_row_keeps_cost_but_marks_combined_quota_tokens_unknown() { + let event: ClaudeEvent = serde_json::from_str(&format!( + r#"{{"type":"assistant","timestamp":"2026-09-20T12:00:00Z","requestId":"req_combined_overflow","message":{{"id":"msg_combined_overflow","model":"claude-sonnet-4-6","usage":{{"input_tokens":{},"output_tokens":1}}}}}}"#, + u64::MAX + )) + .unwrap(); + let record = claude_usage_record_from_event(&event).expect("usage row"); + let quota = quota_history_record_from_usage(&record).expect("timestamped quota row"); + + assert!(record.cost.is_some_and(f64::is_finite)); + assert_eq!(quota.tokens, None); + assert!(!quota.tokens_are_complete); + assert!(quota.cost_usd.is_some_and(f64::is_finite)); + assert!(quota.cost_is_complete); +} + +#[test] +fn nonfinite_claude_price_is_unknown_instead_of_zero() { + let snapshot = crate::core::ModelsDevPricingSnapshot::from_catalog_json_for_tests( + r#"{ + "anthropic": {"models": {"claude-test-extreme-price": { + "id": "claude-test-extreme-price", "cost": {"input": 1e308, "output": 1} + }}} + }"#, + ) + .expect("pricing fixture"); + let mut pricing = ClaudeScanPricingResolver::with_snapshot(snapshot); + let event: ClaudeEvent = serde_json::from_str(&format!( + r#"{{"type":"assistant","timestamp":"2026-09-20T12:00:00Z","requestId":"req_nonfinite","message":{{"id":"msg_nonfinite","model":"claude-test-extreme-price","usage":{{"input_tokens":{},"output_tokens":1}}}}}}"#, + u64::MAX + )) + .unwrap(); + let record = + claude_usage_record_from_event_with_pricing(&event, &mut pricing).expect("usage row"); + let mut summary = CostSummary::default(); + + assert_eq!(record.cost, None); + assert!(!add_claude_record_to_summary(&mut summary, &record)); + assert_eq!(summary.input_tokens, u64::MAX); + assert_eq!(summary.output_tokens, 1); + assert_eq!(summary.total_cost_usd, 0.0); + assert!(!summary.known_zero); +} + +fn claude_transcript_line( + timestamp: &str, + request_key: &str, + request_id: &str, + message_id: &str, +) -> String { + format!( + r#"{{"type":"assistant","timestamp":"{timestamp}","{request_key}":"{request_id}","message":{{"id":"{message_id}","model":"claude-sonnet-4-6","usage":{{"input_tokens":1000,"output_tokens":500}}}}}}"# + ) +} + +#[test] +fn daily_history_dedups_across_files_and_buckets_by_local_day() { + // End-to-end regression for the daily buckets: two transcript files, + // two different days, plus a replay of the day-one record in the + // second file (snake_case request_id, as another writer would emit). + let dir = std::env::temp_dir(); + let file_a = dir.join(format!( + "codexbar-claude-daily-a-{}.jsonl", + std::process::id() + )); + let file_b = dir.join(format!( + "codexbar-claude-daily-b-{}.jsonl", + std::process::id() + )); + + // >24h apart guarantees two distinct local calendar days. + let day_one = Utc::now() - Duration::hours(30); + let day_two = Utc::now() - Duration::hours(2); + let ts_one = day_one.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let ts_two = day_two.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + + std::fs::write( + &file_a, + format!( + "{}\n{}\n", + claude_transcript_line(&ts_one, "requestId", "req_1", "msg_1"), + claude_transcript_line(&ts_two, "requestId", "req_2", "msg_2"), + ), + ) + .unwrap(); + std::fs::write( + &file_b, + format!( + "{}\n", + claude_transcript_line(&ts_one, "request_id", "req_1", "msg_1"), + ), + ) + .unwrap(); + + let day_key = |ts: &DateTime| { + ts.with_timezone(&Local) + .date_naive() + .format("%Y-%m-%d") + .to_string() + }; + let mut daily_costs = HashMap::new(); + daily_costs.insert(day_key(&day_one), None); + daily_costs.insert(day_key(&day_two), None); + let mut unknown_cost_dates = HashSet::new(); + + let cutoff = Utc::now() - Duration::days(30); + let mut seen = HashSet::new(); + for path in [&file_a, &file_b] { + for_each_claude_usage_record(path, &cutoff, &mut seen, None, |record| { + add_claude_record_to_daily_costs(&mut daily_costs, &mut unknown_cost_dates, record); + }); + } + + let day_one_cost = daily_costs[&day_key(&day_one)].expect("day one cost"); + let day_two_cost = daily_costs[&day_key(&day_two)].expect("day two cost"); + assert!(day_one_cost > 0.0, "day one should carry real cost"); + // Identical usage on both days: equal buckets proves the file-b + // replay was de-duplicated (a leak would double day one). + assert!( + (day_one_cost - day_two_cost).abs() < f64::EPSILON, + "each day should hold exactly one record's cost, got {day_one_cost} vs {day_two_cost}" + ); + + // Best-effort test cleanup; the files may already be gone. + let _removed_a = std::fs::remove_file(&file_a); + let _removed_b = std::fs::remove_file(&file_b); +} + +fn claude_daily_cost_record(timestamp: DateTime, cost: Option) -> ClaudeUsageRecord { + ClaudeUsageRecord { + model: "claude-test".to_string(), + pricing_known: cost.is_some(), + timestamp: Some(timestamp), + dedup_key: None, + input: 1, + output: 1, + cache_create: 0, + cache_read: 0, + cost, + } +} + +#[test] +fn unknown_claude_cost_date_cannot_be_restored_by_later_priced_record() { + let timestamp = Utc::now(); + let day = timestamp + .with_timezone(&Local) + .date_naive() + .format("%Y-%m-%d") + .to_string(); + let mut daily_costs = HashMap::from([(day.clone(), None)]); + let mut unknown_cost_dates = HashSet::new(); + + assert!(add_claude_record_to_daily_costs( + &mut daily_costs, + &mut unknown_cost_dates, + &claude_daily_cost_record(timestamp, Some(0.75)), + )); + assert!(!add_claude_record_to_daily_costs( + &mut daily_costs, + &mut unknown_cost_dates, + &claude_daily_cost_record(timestamp, None), + )); + assert!(!add_claude_record_to_daily_costs( + &mut daily_costs, + &mut unknown_cost_dates, + &claude_daily_cost_record(timestamp, Some(1.25)), + )); + + assert_eq!(daily_costs[&day], None); + assert!(unknown_cost_dates.contains(&day)); +} + +#[test] +fn claude_daily_zero_fill_preserves_unknown_dates_and_fills_untouched_dates() { + let unknown_day = "2026-09-22".to_string(); + let untouched_day = "2026-09-23".to_string(); + let mut daily_costs = + HashMap::from([(unknown_day.clone(), None), (untouched_day.clone(), None)]); + let unknown_cost_dates = HashSet::from([unknown_day.clone()]); + + zero_fill_uninitialized_claude_daily_costs(&mut daily_costs, &unknown_cost_dates); + + assert_eq!(daily_costs[&unknown_day], None); + assert_eq!(daily_costs[&untouched_day], Some(0.0)); +} + +#[test] +fn claude_scan_counts_final_incomplete_jsonl_line() { + let path = + std::env::temp_dir().join(format!("codexbar-claude-tail-{}.jsonl", std::process::id())); + let ts = (Utc::now() - Duration::hours(1)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string(); + // No trailing newline — the last (only) record must still be counted. + let body = claude_transcript_line(&ts, "requestId", "req_tail", "msg_tail"); + std::fs::write(&path, body.as_bytes()).unwrap(); + + let cutoff = Utc::now() - Duration::days(1); + let mut seen = HashSet::new(); + let counted = for_each_claude_usage_record(&path, &cutoff, &mut seen, None, |_| {}); + assert_eq!(counted, 1, "incomplete final JSONL line must be processed"); + // Best-effort test cleanup; the file may already be gone. + let _removed = std::fs::remove_file(&path); +} diff --git a/rust/src/cost_scanner/tests/claude_rates.rs b/rust/src/cost_scanner/tests/claude_rates.rs new file mode 100644 index 0000000000..d3a7be5004 --- /dev/null +++ b/rust/src/cost_scanner/tests/claude_rates.rs @@ -0,0 +1,232 @@ +//! Claude pricing as the scanner applies it. + +use super::*; + +/// Scanner Claude pricing without the per-scan memo, as an oracle. +struct ClaudePricing; + +impl ClaudePricing { + fn cost_usd_with_cache_ttl( + model: &str, + input: u64, + cache_create: u64, + cache_create_1h: u64, + cache_read: u64, + output: u64, + ) -> f64 { + let cache_create_1h = cache_create_1h.min(cache_create); + let cache_create_5m = cache_create.saturating_sub(cache_create_1h); + + // Standard buckets (input, cache-read, 5-minute cache-write, output), + // including any long-context tiering, come from the canonical table. + // Unknown/retired models fall back to Sonnet pricing. + #[allow( + clippy::cast_possible_truncation, + reason = "clamped to i32::MAX before casting" + )] + let clamp = |v: u64| v.min(i32::MAX as u64) as i32; + let base = CostUsagePricing::claude_cost_usd( + model, + clamp(input), + clamp(cache_read), + clamp(cache_create_5m), + clamp(output), + ) + .or_else(|| { + CostUsagePricing::claude_cost_usd( + FALLBACK_CLAUDE_MODEL, + clamp(input), + clamp(cache_read), + clamp(cache_create_5m), + clamp(output), + ) + }) + .unwrap_or(0.0); + + // Scanner-specific: one-hour cache writes bill at 2x the input rate. + let input_rate = CostUsagePricing::claude_input_cost_per_token(model) + .or_else(|| CostUsagePricing::claude_input_cost_per_token(FALLBACK_CLAUDE_MODEL)) + .unwrap_or(0.0); + + base + (cache_create_1h as f64) * input_rate * 2.0 + } +} + +#[test] +fn test_unknown_model_falls_back_to_sonnet() { + // Unknown/retired Claude IDs fall back to Sonnet 4.6 base pricing + // ($3/1M input, $15/1M output). 100k tokens stay under the 200k tier. + let cost = + ClaudePricing::cost_usd_with_cache_ttl("claude-3-5-sonnet", 100_000, 0, 0, 0, 100_000); + // 100k * $3/M + 100k * $15/M = 0.30 + 1.50 = 1.80 + assert!((cost - 1.80).abs() < 0.001); +} + +#[test] +fn records_unknown_claude_model_while_using_fallback_cost() { + let event: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","timestamp":"2026-01-15T10:00:00Z","requestId":"req_unknown","message":{"id":"msg_unknown","model":"claude-retired-unknown","usage":{"input_tokens":100000,"output_tokens":100000}}}"#, + ) + .unwrap(); + let record = claude_usage_record_from_event(&event).expect("usage record"); + let mut summary = CostSummary::default(); + + add_claude_record_to_summary(&mut summary, &record); + + assert!(summary.total_cost_usd > 0.0); + assert!(summary.unknown_models.contains("claude-retired-unknown")); +} + +#[test] +fn claude_scan_pricing_resolver_reuses_positive_and_negative_resolution() { + let unknown = format!("claude-scan-unknown-{}", std::process::id()); + let mut resolver = ClaudeScanPricingResolver::default(); + + assert!(resolver.is_known("claude-sonnet-4-6")); + assert!(!resolver.is_known(&unknown)); + assert!(resolver.is_known("claude-sonnet-4-6")); + assert!(!resolver.is_known(&unknown)); + assert_eq!(resolver.normalization_cache_misses, 2); + assert_eq!(resolver.resolution_cache_misses, 2); + assert_eq!(resolver.resolutions.len(), 2); + + let mut cost_resolver = ClaudeScanPricingResolver::default(); + let resolved_unknown = cost_resolver.cost_usd_with_cache_ttl(&unknown, 100, 20, 10, 30, 40); + let fallback = + ClaudePricing::cost_usd_with_cache_ttl(FALLBACK_CLAUDE_MODEL, 100, 20, 10, 30, 40); + assert!((resolved_unknown - fallback).abs() < f64::EPSILON); +} + +#[test] +fn claude_scan_pricing_resolver_preserves_tiered_and_cache_ttl_pricing() { + let mut resolver = ClaudeScanPricingResolver::default(); + let cases = [ + ("claude-sonnet-4-6", 240_000, 0, 0, 0, 0), + ("claude-fable-5", 100, 30, 20, 20, 5), + ]; + + for (model, input, cache_create, cache_create_1h, cache_read, output) in cases { + let actual = resolver.cost_usd_with_cache_ttl( + model, + input, + cache_create, + cache_create_1h, + cache_read, + output, + ); + let expected = ClaudePricing::cost_usd_with_cache_ttl( + model, + input, + cache_create, + cache_create_1h, + cache_read, + output, + ); + assert!((actual - expected).abs() < f64::EPSILON, "{model}"); + } +} + +#[test] +fn claude_scan_resolver_applies_gpt_proxy_long_context_boundary() { + let snapshot = crate::core::ModelsDevPricingSnapshot::from_catalog_json_for_tests( + r#"{ + "openai": {"models": {"gpt-5.6-sol": {"id": "gpt-5.6-sol", "cost": { + "input": 2, "output": 4, "cache_read": 0.25, "cache_write": 3, + "context_over_200k": {"input": 7, "output": 11, "cache_read": 0.5, "cache_write": 9} + }}}} + }"#, + ) + .expect("pricing fixture"); + let mut resolver = ClaudeScanPricingResolver::with_snapshot(snapshot); + + let short = resolver.cost_usd_with_cache_ttl("gpt-5.6-sol", 262_000, 0, 0, 10_000, 13); + let long = resolver.cost_usd_with_cache_ttl("gpt-5.6-sol", 262_001, 0, 0, 10_000, 13); + + assert!((short - 0.526552).abs() < 1e-12); + assert!((long - 1.83915).abs() < 1e-12); +} + +#[test] +fn claude_scan_pricing_resolver_bounds_normalization_memo() { + let mut resolver = ClaudeScanPricingResolver::default(); + for index in 0..(ClaudeScanPricingResolver::MEMO_ENTRY_LIMIT + 8) { + let model = format!("claude-memo-{index}"); + assert_eq!(resolver.normalize(&model), model); + } + assert_eq!( + resolver.normalized_models.len(), + ClaudeScanPricingResolver::MEMO_ENTRY_LIMIT + ); + + let misses = resolver.normalization_cache_misses; + assert_eq!(resolver.normalize("claude-memo-0"), "claude-memo-0"); + assert_eq!(resolver.normalization_cache_misses, misses); + assert_eq!(resolver.normalize("claude-memo-1024"), "claude-memo-1024"); + assert_eq!(resolver.normalization_cache_misses, misses + 1); +} + +#[test] +fn claude_fable_5_prices_five_minute_and_one_hour_cache_writes() { + // (input, cache write, 1h share of it, cache read, output, expected USD). + for (input, cache_create, cache_create_1h, cache_read, output, expected) in [ + ( + 100, + 10, + 0, + 20, + 5, + (100.0 / 1_000_000.0) * 10.00 + + (10.0 / 1_000_000.0) * 12.50 + + (20.0 / 1_000_000.0) * 1.00 + + (5.0 / 1_000_000.0) * 50.00, + ), + ( + 100, + 30, + 20, + 20, + 5, + (100.0 / 1_000_000.0) * 10.00 + + (10.0 / 1_000_000.0) * 12.50 + + (20.0 / 1_000_000.0) * 20.00 + + (20.0 / 1_000_000.0) * 1.00 + + (5.0 / 1_000_000.0) * 50.00, + ), + ] { + let cost = ClaudePricing::cost_usd_with_cache_ttl( + "claude-fable-5", + input, + cache_create, + cache_create_1h, + cache_read, + output, + ); + assert!((cost - expected).abs() < f64::EPSILON, "{cache_create_1h}"); + } +} + +#[test] +fn claude_scan_pricing_follows_the_canonical_table() { + // Sonnet 4.6 honors the 200k tier: 200k @ $3/M + 40k @ $6/M = 0.84 (the + // scanner's old inline table applied a flat $3/M = 0.72). Opus 4.5-4.8 + // bill $5 in + $25 out; opus-4-8 must resolve through the canonical table. + // Legacy Opus 4.0/4.1 stay at $15 + $75 (retired IDs absent from the + // table, e.g. `claude-3-opus-...`, fall back to Sonnet). Haiku 4.5 bills + // $1 + $5, not the Haiku 3 rate. + for (model, input, output, expected) in [ + ("claude-sonnet-4-6", 240_000, 0, 0.84), + ("claude-opus-4-5", 1_000_000, 1_000_000, 30.00), + ("claude-opus-4-6", 1_000_000, 1_000_000, 30.00), + ("claude-opus-4-7", 1_000_000, 1_000_000, 30.00), + ("claude-opus-4-8", 1_000_000, 1_000_000, 30.00), + ("claude-opus-4-20250514", 1_000_000, 1_000_000, 90.00), + ("claude-opus-4-1", 1_000_000, 1_000_000, 90.00), + ("claude-haiku-4-5", 1_000_000, 1_000_000, 6.00), + ] { + let cost = ClaudePricing::cost_usd_with_cache_ttl(model, input, 0, 0, 0, output); + assert!( + (cost - expected).abs() < 0.001, + "{model} should bill ${expected}, got {cost}" + ); + } +} diff --git a/rust/src/cost_scanner/tests/claude_records.rs b/rust/src/cost_scanner/tests/claude_records.rs new file mode 100644 index 0000000000..1674c9b4b4 --- /dev/null +++ b/rust/src/cost_scanner/tests/claude_records.rs @@ -0,0 +1,280 @@ +//! Claude transcript records: dedup, cutoffs, session ids and incomplete rows. + +use super::*; + +#[test] +fn derives_claude_dedup_key_from_message_and_request_ids() { + assert_eq!( + claude_usage_dedup_key(Some("msg_1"), Some("req_1"), None), + Some(ClaudeUsageDedupKey::Request { + message_id: Some("msg_1".to_string()), + request_id: "req_1".to_string(), + }) + ); + assert_eq!( + claude_usage_dedup_key(None, Some(" req_1 "), None), + Some(ClaudeUsageDedupKey::Request { + message_id: None, + request_id: "req_1".to_string(), + }) + ); + assert_eq!( + claude_usage_dedup_key(Some("msg_1"), None, Some("session_1")), + Some(ClaudeUsageDedupKey::Session { + session_id: "session_1".to_string(), + message_id: "msg_1".to_string(), + }) + ); + assert_eq!(claude_usage_dedup_key(Some("msg_1"), None, None), None); + assert_eq!( + claude_usage_dedup_key(None, Some("req_1"), Some("session_1")), + Some(ClaudeUsageDedupKey::Request { + message_id: None, + request_id: "req_1".to_string(), + }) + ); + assert_eq!( + claude_usage_dedup_key(Some("msg_1"), Some(" "), Some("session_1")), + Some(ClaudeUsageDedupKey::Session { + session_id: "session_1".to_string(), + message_id: "msg_1".to_string(), + }) + ); + assert_eq!( + claude_usage_dedup_key(Some(" "), None, Some("session_1")), + None + ); + assert_eq!(claude_usage_dedup_key(Some("msg_1"), None, Some(" ")), None); +} + +#[test] +fn session_id_falls_back_from_blank_direct_id_to_metadata() { + let event: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","sessionId":" ","metadata":{"session_id":" metadata-session "},"message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":10}}}"#, + ) + .unwrap(); + + assert_eq!(event.session_id(), Some("metadata-session")); +} + +#[test] +fn session_id_falls_back_from_blank_direct_and_metadata_ids_to_nested_metadata() { + let event: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","sessionId":" ","metadata":{"sessionId":"\t","metadata":{"session_id":" nested-session "}},"message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":10}}}"#, + ) + .unwrap(); + + assert_eq!(event.session_id(), Some("nested-session")); +} + +#[test] +fn session_aware_claude_dedup_keeps_distinct_sessions_separate() { + let first: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","sessionId":"session_a","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":10}}}"#, + ) + .unwrap(); + let second: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","sessionId":"session_b","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":10}}}"#, + ) + .unwrap(); + let first_record = claude_usage_record_from_event(&first).expect("first usage record"); + let second_record = claude_usage_record_from_event(&second).expect("second usage record"); + let cutoff = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + let mut seen = HashSet::new(); + + assert!(should_count_claude_record( + &first_record, + &cutoff, + &mut seen + )); + assert!(should_count_claude_record( + &second_record, + &cutoff, + &mut seen + )); + assert!(!should_count_claude_record( + &first_record, + &cutoff, + &mut seen + )); +} + +#[test] +fn counts_claude_usage_once_across_duplicate_records() { + // The same API response can be replayed into several transcript files + // (session resume, sidechains); it must only be counted once. + let event: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","timestamp":"2026-01-15T10:00:00Z","requestId":"req_1","message":{"id":"msg_1","model":"claude-sonnet-4-6","usage":{"input_tokens":100,"output_tokens":50,"cache_creation_input_tokens":10,"cache_read_input_tokens":20}}}"#, + ) + .unwrap(); + + let record = claude_usage_record_from_event(&event).expect("usage record"); + assert_eq!(record.model, "claude-sonnet-4-6"); + assert_eq!(record.input, 100); + assert_eq!(record.output, 50); + assert_eq!(record.cache_create, 10); + assert_eq!(record.cache_read, 20); + assert!(record.cost.is_some_and(|cost| cost > 0.0)); + + let cutoff = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + let mut seen = HashSet::new(); + assert!(should_count_claude_record(&record, &cutoff, &mut seen)); + assert!(!should_count_claude_record(&record, &cutoff, &mut seen)); +} + +#[test] +fn rejects_claude_records_before_cutoff() { + let event: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","timestamp":"2025-12-01T10:00:00Z","requestId":"req_old","message":{"id":"msg_old","model":"claude-sonnet-4-6","usage":{"input_tokens":1,"output_tokens":1}}}"#, + ) + .unwrap(); + let record = claude_usage_record_from_event(&event).expect("usage record"); + let cutoff = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + let mut seen = HashSet::new(); + assert!(!should_count_claude_record(&record, &cutoff, &mut seen)); +} + +#[test] +fn ignores_claude_events_without_countable_usage() { + // Non-assistant events carry no billable usage. + let event: ClaudeEvent = + serde_json::from_str(r#"{"type":"user","message":{"usage":{"input_tokens":5}}}"#).unwrap(); + assert!(claude_usage_record_from_event(&event).is_none()); + + // Zero-token usage blocks (e.g. synthetic messages) are not sessions. + let event: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","message":{"id":"msg_zero","model":"claude-sonnet-4-6","usage":{"input_tokens":0,"output_tokens":0}}}"#, + ) + .unwrap(); + assert!(claude_usage_record_from_event(&event).is_none()); +} + +#[test] +fn excludes_preliminary_proxy_estimates_but_keeps_cache_aware_rows() { + let preliminary: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","message":{"id":"msg_preliminary","model":"gpt-5.6-sol","stop_reason":null,"usage":{"input_tokens":1000}}}"#, + ) + .unwrap(); + assert!(claude_usage_record_from_event(&preliminary).is_none()); + + let completed: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","message":{"id":"msg_completed","model":"gpt-5.6-sol","stop_reason":"end_turn","usage":{"input_tokens":1000}}}"#, + ) + .unwrap(); + assert!(claude_usage_record_from_event(&completed).is_some()); + + let cache_aware: ClaudeEvent = serde_json::from_str( + r#"{"type":"assistant","message":{"id":"msg_cache_aware","model":"gpt-5.6-sol","stop_reason":null,"usage":{"input_tokens":1000,"cache_read_input_tokens":1}}}"#, + ) + .unwrap(); + assert!(claude_usage_record_from_event(&cache_aware).is_some()); +} + +#[test] +fn claude_scan_counts_unreconciled_incomplete_requests_per_day_and_model() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("transcript.jsonl"); + let now = Utc::now() - Duration::hours(1); + let ts = now.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let day = cost_bucket_zone().date(now).format("%Y-%m-%d").to_string(); + let row = |request: &str, message: &str, stop: &str, usage: &str| { + format!( + r#"{{"type":"assistant","timestamp":"{ts}","requestId":"{request}","message":{{"id":"{message}","model":"claude-sonnet-4-6","stop_reason":{stop},"usage":{usage}}}}}"# + ) + }; + let body = [ + // Superseded: a completed row with the same key exists. + row("req_done", "msg_done", "null", r#"{"input_tokens":1000}"#), + row( + "req_done", + "msg_done", + r#""end_turn""#, + r#"{"input_tokens":1000,"output_tokens":50}"#, + ), + // Never completed, duplicated preliminary rows count once. + row("req_open", "msg_open", "null", r#"{"input_tokens":500}"#), + row("req_open", "msg_open", "null", r#"{"input_tokens":500}"#), + ] + .join("\n"); + std::fs::write(&path, body).unwrap(); + + let cutoff = Utc::now() - Duration::days(1); + let mut seen = HashSet::new(); + let mut pricing = ClaudeScanPricingResolver::default(); + let mut tracker = ClaudeIncompleteTracker::default(); + let mut summary = CostSummary::default(); + let result = scan_claude_file_with_pricing( + &path, + &cutoff, + &mut seen, + None, + &mut pricing, + &mut tracker, + |record| assert!(add_claude_record_to_summary(&mut summary, record)), + ); + assert_eq!(result.counted, 1); + assert!( + !result.is_complete(), + "preliminary rows keep coverage unknown" + ); + + let report = tracker.resolve(&seen); + report.apply_to(&mut summary); + assert_eq!(summary.incomplete_request_count, 1); + assert_eq!( + summary.incomplete_by_model.get("claude-sonnet-4-6"), + Some(&1) + ); + assert_eq!(report.by_day.get(&day), Some(&1)); + // Only the completed row contributes tokens. + assert_eq!(summary.input_tokens, 1000); + assert_eq!(summary.output_tokens, 50); +} + +#[test] +fn malformed_claude_history_stays_unknown_while_valid_empty_history_is_known_zero() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("transcript.jsonl"); + let cutoff = Utc::now() - Duration::days(1); + + std::fs::write(&path, b"\n").unwrap(); + let mut empty_seen = HashSet::new(); + let mut empty_pricing = ClaudeScanPricingResolver::default(); + let empty_result = scan_claude_file_with_pricing( + &path, + &cutoff, + &mut empty_seen, + None, + &mut empty_pricing, + &mut ClaudeIncompleteTracker::default(), + |_| {}, + ); + let mut empty_summary = CostSummary::default(); + finalize_claude_summary(&mut empty_summary, true, empty_result, false); + assert!(empty_summary.history_coverage_established); + assert!(empty_summary.known_zero); + + std::fs::write(&path, b"{malformed\n").unwrap(); + let mut malformed_seen = HashSet::new(); + let mut malformed_pricing = ClaudeScanPricingResolver::default(); + let malformed_result = scan_claude_file_with_pricing( + &path, + &cutoff, + &mut malformed_seen, + None, + &mut malformed_pricing, + &mut ClaudeIncompleteTracker::default(), + |_| {}, + ); + assert_eq!(malformed_result.malformed_lines, 1); + let mut malformed_summary = CostSummary::default(); + finalize_claude_summary(&mut malformed_summary, true, malformed_result, false); + assert!(!malformed_summary.history_coverage_established); + assert!(!malformed_summary.known_zero); +} diff --git a/rust/src/cost_scanner/tests/codex_bounded.rs b/rust/src/cost_scanner/tests/codex_bounded.rs new file mode 100644 index 0000000000..2961e424c9 --- /dev/null +++ b/rust/src/cost_scanner/tests/codex_bounded.rs @@ -0,0 +1,361 @@ +//! Bounded Codex scans: known-zero history, byte and file limits, paused reports. + +use super::*; + +// ── Upstream 0.50.1 #2932: known-zero history ──────────────────────────── + +#[test] +fn known_zero_is_set_when_scan_completes_with_no_sessions() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + std::fs::create_dir_all(&sessions).unwrap(); + + let scanner = app_scanner(7, &cache_root, &sessions); + + let (summary, _) = scanner.scan_codex_detailed(None); + assert!(summary.history_coverage_established, "scan completed"); + assert_eq!(summary.sessions_count, 0, "no sessions"); + assert!(summary.known_zero, "completed scan with zero = known-zero"); +} + +#[test] +fn known_zero_is_not_set_when_scan_has_results() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + write_codex_session_fixture(&sessions, "a.jsonl", 100); + + let scanner = app_scanner(7, &cache_root, &sessions); + + let (summary, _) = scanner.scan_codex_detailed(None); + assert!(summary.history_coverage_established); + assert_eq!(summary.sessions_count, 1); + assert!(!summary.known_zero, "scan with results is not known-zero"); +} + +#[test] +fn tiny_candidate_limit_prefers_newest_dirty_file_and_persists_older_pending() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let older = write_codex_session_fixture(&sessions, "a-older.jsonl", 100); + let newer = write_codex_session_fixture(&sessions, "z-newer.jsonl", 200); + + let mut options = CostScanOptions::app_driven(); + options.codex_candidate_limit = 1; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + + let (first, first_stats, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(first_stats.files_parsed, 1); + assert_eq!( + first.input_tokens, 200, + "newest dirty file is processed first" + ); + assert!(first_cache.codex_scan_incomplete); + assert_eq!( + first_cache.codex_pending_paths, + vec![older.to_string_lossy().to_string()] + ); + assert!( + !first_cache + .codex_pending_paths + .contains(&newer.to_string_lossy().to_string()) + ); + + let (second, _, second_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(second.input_tokens, 300); + assert!(!second_cache.codex_scan_incomplete); + assert!(second_cache.codex_pending_paths.is_empty()); +} + +#[test] +fn tiny_byte_limit_resumes_and_drains_to_unbounded_totals() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let cache_root = root.path().join("bounded-cache"); + let path = write_codex_session_fixture_with_inputs(&sessions, "multi.jsonl", &[100, 200, 300]); + let first_line_bytes = i64::try_from( + std::fs::read(&path) + .unwrap() + .split(|byte| *byte == b'\n') + .next() + .unwrap() + .len(), + ) + .expect("fixture line length fits i64") + + 1; + + let mut bounded_options = CostScanOptions::app_driven(); + bounded_options.codex_max_session_file_bytes = first_line_bytes; + bounded_options.codex_max_scan_bytes_per_refresh = first_line_bytes; + let bounded = CostScanner::new(7) + .with_options(bounded_options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + + let (first, first_stats, first_cache) = bounded.scan_codex_detailed_with_cache(None); + assert_eq!(first_stats.codex_bytes_read, first_line_bytes as u64); + assert!(first_stats.files_deferred > 0); + assert!(first_cache.codex_scan_incomplete); + assert!( + first_cache + .files + .get(&path.to_string_lossy().to_string()) + .expect("partial cache entry") + .parsed_bytes + .unwrap_or(0) + < i64::try_from(std::fs::metadata(&path).unwrap().len()) + .expect("fixture file length fits i64") + ); + assert!(!first.history_coverage_established); + + let mut final_bounded = None; + let mut saw_resume = false; + for _ in 0..8 { + let (summary, stats, cache) = bounded.scan_codex_detailed_with_cache(None); + saw_resume |= stats.files_resumed > 0; + if !cache.codex_scan_incomplete { + final_bounded = Some(summary); + break; + } + } + let final_bounded = final_bounded.expect("bounded passes drain"); + assert!(saw_resume, "later passes resume the cached prefix"); + + let full = app_scanner(7, root.path().join("full-cache"), &sessions); + let (full_summary, _, full_cache) = full.scan_codex_detailed_with_cache(None); + assert!(!full_cache.codex_scan_incomplete); + assert_eq!(final_bounded.input_tokens, full_summary.input_tokens); + assert_eq!(final_bounded.cached_tokens, full_summary.cached_tokens); + assert_eq!(final_bounded.output_tokens, full_summary.output_tokens); + assert_eq!(final_bounded.sessions_count, full_summary.sessions_count); + assert_eq!(final_bounded.by_model_tokens, full_summary.by_model_tokens); + assert!((final_bounded.total_cost_usd - full_summary.total_cost_usd).abs() < 1e-12); +} + +#[test] +fn incomplete_summary_preserves_previous_report_and_marks_it_non_authoritative() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + write_codex_session_fixture_with_inputs(&sessions, "multi.jsonl", &[100, 200]); + + let report = CachedCostReport { + total_cost_usd: 42.5, + input_tokens: 11, + cached_tokens: 2, + output_tokens: 3, + reasoning_tokens: Some(7), + sessions_count: 7, + updated_at: Some("2026-09-06T00:00:00Z".to_string()), + partial: false, + }; + let mut cache = CostUsageCache { + previous_report: Some(report.clone()), + codex_scan_incomplete: true, + ..Default::default() + }; + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + + let scanner = app_scanner(7, &cache_root, &sessions); + let cancel = AtomicBool::new(true); + let (summary, _, saved) = scanner.scan_codex_detailed_with_cache(Some(&cancel)); + + assert_eq!(summary.total_cost_usd, report.total_cost_usd); + assert_eq!(summary.input_tokens, report.input_tokens as u64); + assert_eq!(summary.cached_tokens, report.cached_tokens as u64); + assert_eq!(summary.output_tokens, report.output_tokens as u64); + assert_eq!(summary.reasoning_tokens, Some(7)); + assert_eq!(summary.sessions_count, report.sessions_count as u32); + assert!(!summary.history_coverage_established); + assert!(!summary.known_zero); + assert!(summary.model_pricing_completeness.is_partial()); + assert_eq!( + saved.previous_report.map(|saved| saved.total_cost_usd), + Some(42.5) + ); + assert!(saved.codex_scan_incomplete); +} + +#[test] +fn failed_catch_up_pause_preserves_cursor_and_report_until_explicit_refresh() { + let (root, sessions, cache_root) = codex_scan_dirs(); + let pending = write_codex_session_fixture(&sessions, "pending.jsonl", 100); + let report = CachedCostReport { + total_cost_usd: 42.5, + input_tokens: 11, + cached_tokens: 2, + output_tokens: 3, + reasoning_tokens: Some(7), + sessions_count: 7, + updated_at: Some("2026-09-06T00:00:00Z".to_string()), + partial: false, + }; + let mut cache = CostUsageCache { + previous_report: Some(report.clone()), + codex_pending_paths: vec![pending.to_string_lossy().to_string()], + codex_scan_incomplete: true, + ..Default::default() + }; + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + + let failed = CostScanner::new(7) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![root.path().join("temporarily-unavailable")]); + let (failed_summary, _, failed_cache) = failed.scan_codex_detailed_with_cache(None); + assert_eq!( + failed_cache.codex_scan_pause_reason, + Some(CodexScanPauseReason::Error( + "Codex session source unavailable".to_string() + )) + ); + assert_eq!(failed_cache.codex_pending_paths, cache.codex_pending_paths); + assert_eq!( + failed_cache + .previous_report + .as_ref() + .map(|saved| saved.total_cost_usd), + Some(report.total_cost_usd) + ); + assert_eq!(failed_summary.total_cost_usd, report.total_cost_usd); + + let background = CostScanner::new(7) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (background_summary, background_stats, background_cache) = + background.scan_codex_detailed_with_cache(None); + assert_eq!(background_stats.files_parsed, 0); + assert_eq!(background_summary.total_cost_usd, report.total_cost_usd); + assert_eq!( + background_cache.codex_pending_paths, + failed_cache.codex_pending_paths + ); + assert_eq!( + background_cache.codex_scan_pause_reason, + failed_cache.codex_scan_pause_reason + ); + + let explicit = app_scanner(7, &cache_root, &sessions); + let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); + assert!(resumed_cache.codex_scan_pause_reason.is_none()); + assert!(!resumed_cache.codex_scan_incomplete); + assert!(resumed_cache.codex_pending_paths.is_empty()); + assert!(resumed_summary.history_coverage_established); + assert_eq!(resumed_summary.input_tokens, 100); + assert!(resumed_cache.previous_report.is_none()); +} + +#[test] +fn missing_unobserved_sessions_root_does_not_pause_validated_cache() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let missing = root.path().join("optional-sessions"); + let cache_root = root.path().join("cache"); + write_codex_session_fixture(&sessions, "observed.jsonl", 100); + + let initial = app_scanner(7, &cache_root, &sessions); + let (initial_summary, _) = initial.scan_codex_detailed(None); + assert!(initial_summary.history_coverage_established); + + let mut cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + cache.last_scan_unix_ms = 1; + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + + let background = CostScanner::new(7) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions, missing]); + let (summary, _, saved) = background.scan_codex_detailed_with_cache(None); + + assert!(summary.history_coverage_established); + assert_eq!(summary.input_tokens, 100); + assert!(!saved.codex_scan_incomplete); + assert!(saved.codex_scan_pause_reason.is_none()); +} + +#[test] +fn trace_pruning_preserves_cursor_and_validated_history_until_refresh() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "pruned.jsonl", 100); + + let initial = app_scanner(7, &cache_root, &sessions); + let (initial_summary, _, initial_cache) = initial.scan_codex_detailed_with_cache(None); + assert_eq!(initial_summary.input_tokens, 100); + assert!(!initial_cache.codex_scan_incomplete); + + // Keep the next pass outside the scanner debounce while simulating Codex + // retention pruning the same trace file down to a smaller valid payload. + let mut initial_cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + initial_cache.last_scan_unix_ms = 1; + JsonlScanner::save_cache(ProviderId::Codex, &mut initial_cache, Some(&cache_root)); + let _ = write_codex_session_fixture(&sessions, "pruned.jsonl", 1); + + let background = CostScanner::new(7) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (paused_summary, _, paused_cache) = background.scan_codex_detailed_with_cache(None); + assert_eq!(paused_summary.input_tokens, 100); + assert_eq!( + paused_cache.codex_scan_pause_reason, + Some(CodexScanPauseReason::NoProgress) + ); + assert_eq!( + paused_cache.codex_pending_paths, + vec![path.to_string_lossy().to_string()] + ); + assert!(paused_cache.previous_report.is_some()); + + let explicit = app_scanner(7, &cache_root, &sessions); + let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); + assert_eq!(resumed_summary.input_tokens, 1); + assert!(resumed_cache.codex_scan_pause_reason.is_none()); + assert!(resumed_cache.codex_pending_paths.is_empty()); + assert!(resumed_cache.previous_report.is_none()); +} + +/// #755: a background scan paused with `no_progress` and no retained report +/// used to sum every cached day, so Today and 30d showed the same totals. +#[test] +fn paused_scan_without_previous_report_reports_only_the_requested_days() { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let cache_root = root.path().join("cache"); + let today = Local::now().date_naive(); + let older = (today - chrono::Duration::days(10)) + .format("%Y-%m-%d") + .to_string(); + let today = today.format("%Y-%m-%d").to_string(); + let mut cache = CostUsageCache { + files: HashMap::from([ + ( + "today".to_string(), + cached_usage_with_packed(&today, "gpt-5.6-sol", vec![1_000, 400, 10]), + ), + ( + "older".to_string(), + cached_usage_with_packed(&older, "gpt-5.6-sol", vec![5_000, 2_000, 50]), + ), + ]), + // A queued file whose fork parent never resolved keeps the current + // window unestablished, as in the reported cache. + codex_pending_paths: vec![root.path().join("gone.jsonl").to_string_lossy().to_string()], + codex_scan_incomplete: true, + codex_scan_pause_reason: Some(CodexScanPauseReason::NoProgress), + ..Default::default() + }; + rebuild_cache_days(&mut cache); + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + + let scan = |days| { + CostScanner::new(days) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]) + .scan_codex_detailed_with_cache(None) + }; + let (today_summary, today_stats, _) = scan(1); + let (month_summary, month_stats, _) = scan(30); + + assert_eq!(today_stats.files_parsed, 0); + assert_eq!(month_stats.files_parsed, 0); + assert_eq!(today_summary.input_tokens, 1_000); + assert_eq!(today_summary.output_tokens, 10); + assert_eq!(month_summary.input_tokens, 6_000); + assert_eq!(month_summary.output_tokens, 60); + assert!(today_summary.total_cost_usd < month_summary.total_cost_usd); +} diff --git a/rust/src/cost_scanner/tests/codex_cache.rs b/rust/src/cost_scanner/tests/codex_cache.rs new file mode 100644 index 0000000000..e86b7c02c7 --- /dev/null +++ b/rust/src/cost_scanner/tests/codex_cache.rs @@ -0,0 +1,707 @@ +//! Codex scan cache reuse, resume, cancel and reconciliation. + +use super::*; + +#[test] +fn cost_scan_second_pass_skips_unchanged_files_via_cache() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + write_codex_session_fixture(&sessions, "a.jsonl", 100); + write_codex_session_fixture(&sessions, "b.jsonl", 200); + + let scanner = app_scanner(7, &cache_root, &sessions); + + let (summary1, stats1) = scanner.scan_codex_detailed(None); + assert_eq!(stats1.files_parsed, 2, "first pass parses both files"); + assert_eq!(stats1.files_skipped, 0); + assert_eq!(stats1.codex_metadata_read_paths.len(), 2); + assert_eq!(stats1.codex_history_read_paths.len(), 2); + assert_eq!(stats1.codex_read_receipt.metadata_reads, 2); + assert_eq!(stats1.codex_read_receipt.history_reads, 2); + assert!(summary1.total_cost_usd > 0.0); + assert_eq!(summary1.sessions_count, 2); + + // Second pass with default debounce still inspects files but skips re-parse. + // Use app_driven so we exercise per-file mtime skip rather than whole-scan debounce. + CodexLineagePlanner::reset_graph_build_count(); + let (summary2, stats2) = scanner.scan_codex_detailed(None); + assert_eq!(stats2.files_seen, 2); + assert_eq!(stats2.files_skipped, 2, "cache hit skips re-parse"); + assert_eq!(stats2.files_parsed, 0); + assert_eq!( + CodexLineagePlanner::graph_build_count(), + 0, + "warm root-only scan must bypass lineage graph construction" + ); + assert!(stats2.codex_metadata_read_paths.is_empty()); + assert!(stats2.codex_history_read_paths.is_empty()); + assert_eq!(stats2.codex_read_receipt, Default::default()); + assert_eq!(summary2.input_tokens, summary1.input_tokens); + assert!((summary2.total_cost_usd - summary1.total_cost_usd).abs() < 1e-9); + + // Force path already used above; confirm debounce short-circuit with default options. + let debounced = CostScanner::new(7) + .with_options(CostScanOptions::default()) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (summary3, stats3) = debounced.scan_codex_detailed(None); + assert!( + stats3.used_cache_debounce, + "default options debounce within 60s" + ); + assert_eq!(stats3.files_seen, 0); + assert_eq!(summary3.input_tokens, summary1.input_tokens); + + // app_driven after debounce still re-reads (skip via mtime, not full re-parse). + let forced = app_scanner(7, &cache_root, &sessions); + let (_, stats4) = forced.scan_codex_detailed(None); + assert!(!stats4.used_cache_debounce); + assert_eq!(stats4.files_skipped, 2); + assert_eq!(stats4.files_parsed, 0); + assert!(stats4.codex_history_read_paths.is_empty()); +} + +#[test] +fn codex_lazy_history_receipt_reads_only_changed_file_and_matches_fresh_parse() { + let (root, sessions, cache_root) = codex_scan_dirs(); + let first_path = write_codex_session_fixture_with_inputs(&sessions, "first.jsonl", &[100]); + let second_path = write_codex_session_fixture_with_inputs(&sessions, "second.jsonl", &[200]); + let scanner = app_scanner(7, &cache_root, &sessions); + + let (initial, _, _) = scanner.scan_codex_detailed_with_cache(None); + let (unchanged, unchanged_stats, _) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(unchanged.input_tokens, initial.input_tokens); + assert!(unchanged_stats.codex_metadata_read_paths.is_empty()); + assert!(unchanged_stats.codex_history_read_paths.is_empty()); + assert_eq!(unchanged_stats.codex_read_receipt, Default::default()); + + use std::io::Write as _; + let timestamp = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let extra = format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":150,"cached_input_tokens":0,"output_tokens":5}}}}}}}} +"# + ); + std::fs::OpenOptions::new() + .append(true) + .open(&first_path) + .unwrap() + .write_all(extra.as_bytes()) + .unwrap(); + + let (incremental, incremental_stats, _) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!( + incremental_stats.codex_metadata_read_paths, + vec![first_path.to_string_lossy().to_string()] + ); + assert_eq!( + incremental_stats.codex_history_read_paths, + vec![first_path.to_string_lossy().to_string()] + ); + assert_eq!(incremental_stats.codex_read_receipt.metadata_reads, 1); + assert_eq!(incremental_stats.codex_read_receipt.history_reads, 1); + assert_eq!(incremental.input_tokens, 350); + + let fresh = app_scanner(7, root.path().join("fresh-cache"), &sessions); + let (full, full_stats) = fresh.scan_codex_detailed(None); + assert_eq!(full_stats.codex_history_read_paths.len(), 2); + assert_eq!(incremental.input_tokens, full.input_tokens); + assert_eq!(incremental.output_tokens, full.output_tokens); + assert_eq!(incremental.cached_tokens, full.cached_tokens); + assert_eq!(incremental.by_model_tokens, full.by_model_tokens); + assert!((incremental.total_cost_usd - full.total_cost_usd).abs() < 1e-12); + assert!(second_path.exists()); +} + +#[test] +fn codex_source_recovery_keeps_appended_duplicate_unpriced_after_cache_reload() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture_with_inputs(&sessions, "recovery.jsonl", &[100]); + let scanner = app_scanner(7, &cache_root, &sessions); + + let (_, _, mut first_cache) = scanner.scan_codex_detailed_with_cache(None); + let path_key = path.to_string_lossy().to_string(); + let source_rows = first_cache + .codex_source_rows + .get_mut(&path_key) + .expect("source rows persisted"); + assert_eq!(source_rows.rows.len(), 1); + // Cached evidence that disagrees with the source: a Priority row on a + // model with a Fast lane, so the recovered row keeps a `-priority` key + // (a Priority row without a Fast lane would price at its Standard base). + source_rows.rows[0].pricing.pricing_model = Some("gpt-5.5".to_string()); + source_rows.rows[0].pricing.pricing_mode = Some("priority".to_string()); + first_cache.last_scan_unix_ms = 1; + JsonlScanner::save_cache(ProviderId::Codex, &mut first_cache, Some(&cache_root)); + + // Half an hour after the fixture's row, so both rows share one local day. + let appended_time = recent_codex_fixture_time() + Duration::minutes(30); + let timestamp = appended_time.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let appended = format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":200,"cached_input_tokens":0,"output_tokens":10}}}}}}}}"# + ) + "\n"; + std::fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap() + .write_all(appended.as_bytes()) + .unwrap(); + + let (_, _, second_cache) = scanner.scan_codex_detailed_with_cache(None); + let usage = second_cache.files.get(&path_key).expect("file cache"); + let day = appended_time + .with_timezone(&Local) + .format("%Y-%m-%d") + .to_string(); + assert_eq!( + usage.days[&day]["gpt-5.5-priority"], + vec![100, 0, 5], + "gpt-5.5 turns also price at the Priority rate" + ); + assert_eq!( + usage.days[&day][CostUsagePricing::CODEX_UNATTRIBUTED_MODEL], + vec![100, 0, 5] + ); +} + +#[test] +fn codex_file_identity_invalidates_same_path_cache_without_eager_history_read() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "replacement.jsonl", 100); + let scanner = app_scanner(7, &cache_root, &sessions); + let (_, _, _) = scanner.scan_codex_detailed_with_cache(None); + let old_mtime = std::fs::metadata(&path).unwrap().modified().unwrap(); + let rotated = path.with_extension("old"); + std::fs::rename(&path, &rotated).unwrap(); + let replacement = write_codex_session_fixture(&sessions, "replacement.jsonl", 200); + // Windows requires a handle with write-attribute access for set_modified; + // keep the replacement's mtime equal to the original without opening it + // read-only. The file contents have the same length, so path/mtime/size + // remain unchanged while the file identity changes. + std::fs::OpenOptions::new() + .write(true) + .open(&replacement) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + + let (summary, stats) = scanner.scan_codex_detailed(None); + assert_eq!(summary.input_tokens, 200); + assert_eq!( + stats.codex_history_read_paths, + vec![replacement.to_string_lossy().to_string()] + ); + assert_eq!(stats.codex_read_receipt.history_reads, 1); +} + +#[test] +fn cancelled_fresh_cache_hit_is_not_authoritative() { + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path().join("cache"); + let today = Local::now().date_naive().format("%Y-%m-%d").to_string(); + let usage = HashMap::from([( + today.clone(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![100, 0, 10])]), + )]); + let mut cache = CostUsageCache { + last_scan_unix_ms: unix_now_ms(), + files: HashMap::from([( + "cached.jsonl".to_string(), + CostUsageFileUsage { + parsed_bytes: Some(100), + last_model: Some("gpt-5.6-sol".to_string()), + codex_token_timestamps_monotonic: Some(true), + ..test_file_usage(100, usage.clone()) + }, + )]), + days: usage, + ..Default::default() + }; + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + + let cancel = AtomicBool::new(true); + let scanner = CostScanner::new(7) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![root.path().join("sessions")]); + let (summary, stats) = scanner.scan_codex_detailed(Some(&cancel)); + + assert!( + stats.used_cache_debounce, + "fresh cache should use debounce path" + ); + assert_eq!(summary.sessions_count, 1, "cached usage is still visible"); + assert!( + !summary.history_coverage_established, + "cancelled cache publication must not claim complete history" + ); + assert!(!summary.known_zero); +} + +#[test] +fn cost_scan_cancel_stops_between_files() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + write_codex_session_fixture(&sessions, "a.jsonl", 100); + write_codex_session_fixture(&sessions, "b.jsonl", 200); + write_codex_session_fixture(&sessions, "c.jsonl", 300); + + let cancel = AtomicBool::new(true); + let scanner = app_scanner(7, cache_root, &sessions); + let (summary, stats) = scanner.scan_codex_detailed(Some(&cancel)); + assert_eq!(stats.files_seen, 0, "cancel before first file stops walk"); + assert_eq!(summary.sessions_count, 0); +} + +#[test] +fn cost_scan_reconciles_deleted_file_to_known_zero() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (first, _) = scanner.scan_codex_detailed(None); + assert_eq!(first.sessions_count, 1); + assert!(first.total_cost_usd > 0.0); + + std::fs::remove_file(&path).unwrap(); + let (second, _) = scanner.scan_codex_detailed(None); + + assert_eq!(second.sessions_count, 0); + assert_eq!(second.total_cost_usd, 0.0); + assert!(second.history_coverage_established); + assert!(second.known_zero); + let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!(cache.files.is_empty(), "deleted JSONL row must be removed"); + assert!(cache.days.is_empty(), "stale daily totals must disappear"); +} + +#[test] +fn cost_scan_reconciliation_preserves_sibling_totals_once() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let deleted = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); + let sibling = write_codex_session_fixture(&sessions, "sibling.jsonl", 200); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (first, _) = scanner.scan_codex_detailed(None); + assert_eq!(first.sessions_count, 2); + + std::fs::remove_file(&deleted).unwrap(); + let (second, _) = scanner.scan_codex_detailed(None); + + assert_eq!(second.sessions_count, 1); + assert_eq!(second.input_tokens, 200); + assert_eq!(second.output_tokens, 5); + let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert_eq!(cache.files.len(), 1); + assert!( + !cache + .files + .contains_key(&deleted.to_string_lossy().to_string()) + ); + assert!( + cache + .files + .contains_key(&sibling.to_string_lossy().to_string()) + ); +} + +#[test] +fn cancelled_scan_after_deletion_preserves_stale_cache_row() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "deleted.jsonl", 100); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (first, _) = scanner.scan_codex_detailed(None); + assert_eq!(first.sessions_count, 1); + std::fs::remove_file(&path).unwrap(); + + let cancel = AtomicBool::new(true); + let (cancelled, stats) = scanner.scan_codex_detailed(Some(&cancel)); + + assert_eq!(stats.files_seen, 0); + assert_eq!(cancelled.sessions_count, 0); + assert!(!cancelled.history_coverage_established); + assert!(!cancelled.known_zero); + let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!( + cache + .files + .contains_key(&path.to_string_lossy().to_string()) + ); +} + +#[test] +fn cost_scan_resumes_appended_bytes() { + let (root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "grow.jsonl", 50); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (s1, st1) = scanner.scan_codex_detailed(None); + assert_eq!(st1.files_parsed, 1); + assert_eq!(st1.token_timestamp_comparisons, 0); + assert_eq!(s1.input_tokens, 50); + + // Append another cumulative token_count event (100 total => +50 delta). + let ts = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let extra = format!( + r#"{{"timestamp":"{ts}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":100,"cached_input_tokens":0,"output_tokens":10}}}}}}}} +"# + ); + use std::io::Write as _; + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap(); + f.write_all(extra.as_bytes()).unwrap(); + drop(f); + + // Bump mtime/size visibly on some FS by rewriting metadata via reopen. + let (s2, st2) = scanner.scan_codex_detailed(None); + assert_eq!(st2.files_resumed, 1, "grown file resumes from offset"); + assert_eq!(st2.files_parsed, 0); + assert_eq!( + st2.token_timestamp_comparisons, 1, + "resume validates only the cached-prefix boundary and appended event" + ); + assert_eq!(s2.input_tokens, 100); + + // The append-only path must publish the same aggregate as a fresh + // full parse; the optimization is allowed to change work, not data. + let full_scanner = app_scanner(7, root.path().join("fresh-cache"), &sessions); + let (full, full_stats) = full_scanner.scan_codex_detailed(None); + assert_eq!(full_stats.files_parsed, 1); + assert_eq!(s2.input_tokens, full.input_tokens); + assert_eq!(s2.cached_tokens, full.cached_tokens); + assert_eq!(s2.output_tokens, full.output_tokens); + assert_eq!(s2.sessions_count, full.sessions_count); + assert_eq!(s2.by_model_tokens, full.by_model_tokens); + assert_eq!(s2.by_model.len(), full.by_model.len()); + for (model, resumed_cost) in &s2.by_model { + let full_cost = full.by_model.get(model).copied().expect("full model row"); + assert!((resumed_cost - full_cost).abs() < 1e-12); + } + assert!((s2.total_cost_usd - full.total_cost_usd).abs() < 1e-12); + + let cached = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + let cached_file = cached + .files + .get(&path.to_string_lossy().to_string()) + .expect("resumed file cache entry"); + assert_eq!(cached_file.codex_token_timestamps_monotonic, Some(true)); + assert!(cached_file.codex_last_token_timestamp.is_some()); +} + +#[test] +fn codex_partial_rescan_replaces_changed_session_after_cache_reopen() { + // Windows parity for upstream 0.60.2 cost persistence: a rewritten + // session must replace the cached file aggregate even when the first + // refresh only consumes a bounded prefix and the next refresh reloads the + // cache from disk. + let (root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture_with_inputs(&sessions, "changed.jsonl", &[100, 200]); + let old_metadata = std::fs::metadata(&path).unwrap(); + let old_size = old_metadata.len(); + let old_mtime = old_metadata.modified().unwrap(); + + let initial_scanner = app_scanner(7, &cache_root, &sessions); + let (initial, initial_stats) = initial_scanner.scan_codex_detailed(None); + assert_eq!(initial_stats.files_parsed, 1); + assert_eq!(initial.input_tokens, 200); + assert!(initial.history_coverage_established); + + // Keep the path, identity, and byte length stable while changing both + // token snapshots. The mtime change proves that the cached aggregate is + // invalidated before the bounded rescan begins. + write_codex_session_fixture_with_inputs(&sessions, "changed.jsonl", &[300, 400]); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(old_mtime + std::time::Duration::from_secs(2)) + .unwrap(); + let rewritten_metadata = std::fs::metadata(&path).unwrap(); + assert_eq!(rewritten_metadata.len(), old_size); + assert_ne!(rewritten_metadata.modified().unwrap(), old_mtime); + + let first_line_bytes = i64::try_from( + std::fs::read(&path) + .unwrap() + .split(|byte| *byte == b'\n') + .next() + .unwrap() + .len(), + ) + .expect("fixture line length fits i64") + + 1; + let mut bounded_options = CostScanOptions::app_driven(); + bounded_options.codex_max_session_file_bytes = first_line_bytes; + bounded_options.codex_max_scan_bytes_per_refresh = first_line_bytes; + + let bounded_scanner = CostScanner::new(7) + .with_options(bounded_options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (partial, partial_stats, partial_cache) = + bounded_scanner.scan_codex_detailed_with_cache(None); + assert!(partial_stats.files_parsed >= 1); + assert!(partial_cache.codex_scan_incomplete); + assert!(!partial.history_coverage_established); + assert_eq!( + partial_cache + .previous_report + .as_ref() + .map(|report| report.input_tokens), + Some(200), + "the last validated report remains visible during catch-up" + ); + let partial_file = partial_cache + .files + .get(&path.to_string_lossy().to_string()) + .expect("partially rescanned file cache entry"); + assert_eq!(partial_file.parsed_bytes, Some(first_line_bytes)); + + // A new scanner instance models a process/cache reopen. The persisted + // cursor must resume the rewritten file and finish at the new total. + let reopened_scanner = CostScanner::new(7) + .with_options(bounded_options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (resumed, resumed_stats, resumed_cache) = + reopened_scanner.scan_codex_detailed_with_cache(None); + assert_eq!(resumed_stats.files_resumed, 1); + assert_eq!(resumed_stats.files_parsed, 0); + assert!(!resumed_cache.codex_scan_incomplete); + assert!(resumed.history_coverage_established); + assert_eq!(resumed.input_tokens, 400); + + // Resumption may change the amount of work, but it must publish the same + // cost aggregate as a clean full parse of the rewritten session. + let fresh_scanner = app_scanner(7, root.path().join("fresh-cache"), &sessions); + let (fresh, fresh_stats) = fresh_scanner.scan_codex_detailed(None); + assert_eq!(fresh_stats.files_parsed, 1); + assert_eq!(resumed.input_tokens, fresh.input_tokens); + assert_eq!(resumed.cached_tokens, fresh.cached_tokens); + assert_eq!(resumed.output_tokens, fresh.output_tokens); + assert_eq!(resumed.sessions_count, fresh.sessions_count); + assert_eq!(resumed.by_model_tokens, fresh.by_model_tokens); + assert_eq!(resumed.by_model.len(), fresh.by_model.len()); + for (model, resumed_cost) in &resumed.by_model { + let fresh_cost = fresh.by_model.get(model).copied().expect("fresh model row"); + assert!((resumed_cost - fresh_cost).abs() < 1e-12); + } + assert!((resumed.total_cost_usd - fresh.total_cost_usd).abs() < 1e-12); +} + +#[test] +fn bounded_growing_rollout_freezes_target_and_resumes_a_retained_tail() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = + write_codex_session_fixture_with_inputs(&sessions, "growing.jsonl", &[100, 200, 300]); + let initial_size = i64::try_from(std::fs::metadata(&path).unwrap().len()) + .expect("fixture file length fits i64"); + let first_line_bytes = i64::try_from( + std::fs::read(&path) + .unwrap() + .split(|byte| *byte == b'\n') + .next() + .unwrap() + .len(), + ) + .expect("fixture line length fits i64") + + 1; + + let mut options = CostScanOptions::app_driven(); + options.codex_max_session_file_bytes = first_line_bytes; + options.codex_max_scan_bytes_per_refresh = first_line_bytes; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + + let (first, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(first.input_tokens, 100); + let first_usage = cached_file(&first_cache, &path); + assert_eq!(first_usage.codex_scan_target_size, Some(initial_size)); + assert_eq!(first_usage.parsed_bytes, Some(first_line_bytes)); + assert!(first_cache.codex_scan_incomplete); + + let timestamp = (Utc::now() - Duration::minutes(10)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string(); + let append_line = |total: u64| { + format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":{total},"cached_input_tokens":0,"output_tokens":{}}}}}}}}} +"#, + total / 10 + ) + }; + let mut next_total = 400_u64; + let mut bounded_summary = first; + let mut bounded_cache = first_cache; + for _ in 0..8 { + let line = append_line(next_total); + next_total += 100; + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap(); + file.write_all(line.as_bytes()).unwrap(); + drop(file); + + (bounded_summary, _, bounded_cache) = scanner.scan_codex_detailed_with_cache(None); + let usage = cached_file(&bounded_cache, &path); + assert_eq!(usage.codex_scan_target_size, Some(initial_size)); + assert!(usage.parsed_bytes.unwrap_or_default() <= initial_size); + if usage.parsed_bytes == Some(initial_size) { + break; + } + } + + assert_eq!(bounded_summary.input_tokens, 300); + let bounded_usage = cached_file(&bounded_cache, &path); + assert_eq!(bounded_usage.parsed_bytes, Some(initial_size)); + assert_eq!(bounded_usage.codex_scan_target_size, Some(initial_size)); + assert!( + bounded_cache.codex_scan_incomplete, + "the appended tail stays queued" + ); + + for _ in 0..32 { + if !bounded_cache.codex_scan_incomplete { + break; + } + (bounded_summary, _, bounded_cache) = scanner.scan_codex_detailed_with_cache(None); + } + assert!(!bounded_cache.codex_scan_incomplete); + let stable_summary = bounded_summary.clone(); + let stable_size = i64::try_from(std::fs::metadata(&path).unwrap().len()) + .expect("fixture file length fits i64"); + + let partial_line = append_line(next_total); + let split = partial_line.len() / 2; + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap(); + file.write_all(&partial_line.as_bytes()[..split]).unwrap(); + drop(file); + let (partial_summary, _, partial_cache) = scanner.scan_codex_detailed_with_cache(None); + let partial_usage = cached_file(&partial_cache, &path); + assert_eq!(partial_summary.input_tokens, stable_summary.input_tokens); + assert_eq!(partial_usage.parsed_bytes, Some(stable_size)); + assert_eq!(partial_usage.codex_scan_target_size, Some(stable_size)); + assert!(partial_cache.codex_scan_incomplete); + + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap(); + file.write_all(&partial_line.as_bytes()[split..]).unwrap(); + drop(file); + next_total += 100; + let (resumed_summary, _, resumed_cache) = scanner.scan_codex_detailed_with_cache(None); + assert!(!resumed_cache.codex_scan_incomplete); + assert_eq!(resumed_summary.input_tokens, next_total - 100); + let resumed_usage = cached_file(&resumed_cache, &path); + assert_eq!( + resumed_usage.parsed_bytes, + Some( + i64::try_from(std::fs::metadata(&path).unwrap().len()) + .expect("fixture file length fits i64"), + ) + ); + assert_eq!( + resumed_usage.codex_scan_target_size, + resumed_usage.parsed_bytes + ); +} + +#[test] +fn cost_scan_midline_rewrite_forces_full_parse_not_resume() { + // F2 (upstream 0.48.0 #2648): when a file is rewritten/truncated so the + // cached resume offset is now mid-line (byte before offset is not \n), + // the scanner must fall through to a full re-parse from offset 0 rather + // than resuming from the stale mid-line offset. + let (_root, sessions, cache_root) = codex_scan_dirs(); + let _path = write_codex_session_fixture(&sessions, "a.jsonl", 100); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (s1, st1) = scanner.scan_codex_detailed(None); + assert_eq!(st1.files_parsed, 1); + assert_eq!(s1.input_tokens, 100); + + // Rewrite the file with a shorter body at the same path so the cached + // parsed_bytes offset now points mid-line in the new content. + let today = Local::now().date_naive(); + let day_dir = partition_dir(&sessions, today); + let ts = (Utc::now() - Duration::minutes(30)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string(); + // Shorter content with different token count — the cached offset will + // be past EOF or mid-line in this new content. + let body = format!( + r#"{{"timestamp":"{ts}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":50,"cached_input_tokens":0,"output_tokens":5}}}}}}}} +"# + ); + std::fs::write(day_dir.join("a.jsonl"), body).unwrap(); + + let (s2, st2) = scanner.scan_codex_detailed(None); + // The scanner must full-parse (not resume) because the cached offset + // no longer sits on a line boundary in the rewritten content. + assert!( + st2.files_parsed >= 1 || st2.files_resumed == 0, + "midline rewrite forces full parse, not resume (parsed={}, resumed={})", + st2.files_parsed, + st2.files_resumed + ); + assert_eq!(s2.input_tokens, 50, "full parse picks up new token count"); +} + +#[test] +fn previous_report_clears_after_successful_full_scan() { + // F8 (upstream 0.48.0): a completed full scan clears previous_report so + // the refreshing indicator does not stay permanently on. + let (_root, sessions, cache_root) = codex_scan_dirs(); + write_codex_session_fixture(&sessions, "a.jsonl", 100); + + let scanner = app_scanner(7, &cache_root, &sessions); + + // First scan: builds cache fresh; no previous_report expected. + let (summary1, _) = scanner.scan_codex_detailed(None); + assert!(summary1.history_coverage_established); + let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!( + cache.previous_report.is_none(), + "first scan clears previous_report" + ); + + // Inject a previous_report to simulate trim-set catch-up. + let mut cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + cache.previous_report = Some(crate::core::CachedCostReport { + total_cost_usd: 0.0, + input_tokens: 0, + cached_tokens: 0, + output_tokens: 0, + reasoning_tokens: None, + sessions_count: 0, + updated_at: None, + partial: false, + }); + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + + // Verify the cache now has previous_report set. + let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!( + cache.previous_report.is_some(), + "injected previous_report persists" + ); + + // Full scan with app_driven clears previous_report on success. + let (summary2, _) = scanner.scan_codex_detailed(None); + assert!( + summary2.history_coverage_established, + "after full scan coverage is established" + ); + + let cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!( + cache.previous_report.is_none(), + "full scan clears previous_report" + ); +} diff --git a/rust/src/cost_scanner/tests/codex_fork.rs b/rust/src/cost_scanner/tests/codex_fork.rs new file mode 100644 index 0000000000..1c8317d83e --- /dev/null +++ b/rust/src/cost_scanner/tests/codex_fork.rs @@ -0,0 +1,341 @@ +//! Codex fork children billed above their parent baseline. + +use super::*; + +#[test] +fn ordinary_non_fork_session_keeps_cumulative_accounting() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture_with_inputs(&sessions, "ordinary.jsonl", &[100, 140]); + + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(summary.input_tokens, 140); + let usage = cached_file(&cache, &path); + assert_eq!(cached_input_total(usage), 140); + assert_eq!(usage.codex_forked_from_id, None); + assert!(!usage.codex_unresolved_fork_parent); +} + +#[test] +fn fork_child_counts_only_growth_above_parent_baseline() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let base = Utc::now() - Duration::hours(1); + let fork = base + Duration::seconds(2); + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000_000], + ); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("parent-id"), + fork, + base + Duration::seconds(3), + &[1_000_000, 1_000_140], + ); + + let now = std::time::SystemTime::now(); + File::options() + .write(true) + .open(&parent) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(10)) + .unwrap(); + File::options() + .write(true) + .open(&child) + .unwrap() + .set_modified(now - std::time::Duration::from_secs(5)) + .unwrap(); + + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(summary.input_tokens, 1_000_140); + assert_eq!(summary.sessions_count, 2); + let child_usage = cached_file(&cache, &child); + assert_eq!(cached_input_total(child_usage), 140); + assert!(!child_usage.codex_unresolved_fork_parent); + assert!(cache.codex_pending_paths.is_empty()); + assert!( + cache + .files + .contains_key(&parent.to_string_lossy().to_string()) + ); +} + +#[test] +fn replaced_fork_child_does_not_reuse_cached_identity() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let base = Utc::now() - Duration::hours(1); + let fork = base + Duration::seconds(2); + let _parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000_000], + ); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("parent-id"), + fork, + base + Duration::seconds(3), + &[1_000_000, 1_000_140], + ); + + let scanner = CostScanner::new(7) + .with_options({ + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + options + }) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let child_key = child.to_string_lossy().to_string(); + let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + let first_child = first_cache.files.get(&child_key).unwrap(); + assert_eq!(first_child.codex_session_id.as_deref(), Some("child-id")); + assert_eq!( + first_child.codex_forked_from_id.as_deref(), + Some("parent-id") + ); + + let old_size = std::fs::metadata(&child).unwrap().len(); + write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "replacement-id", + None, + base + Duration::seconds(4), + base + Duration::seconds(5), + &[77], + ); + let new_size = std::fs::metadata(&child).unwrap().len(); + assert_ne!(old_size, new_size); + + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + let child_usage = cache.files.get(&child_key).unwrap(); + assert_eq!( + child_usage.codex_session_id.as_deref(), + Some("replacement-id") + ); + assert_eq!(child_usage.codex_forked_from_id, None); + assert!(!child_usage.codex_unresolved_fork_parent); + assert_eq!(cached_input_total(child_usage), 77); + assert_eq!(summary.input_tokens, 1_000_077); +} + +#[test] +fn missing_fork_parent_fails_closed_and_persists_pending() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let base = Utc::now() - Duration::hours(1); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("missing-parent"), + base + Duration::seconds(1), + base + Duration::seconds(2), + &[1_000_000, 1_000_140], + ); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + let child_key = child.to_string_lossy().to_string(); + let child_usage = cache.files.get(&child_key).unwrap(); + assert_eq!(summary.input_tokens, 0); + assert_eq!(summary.sessions_count, 0); + assert!(child_usage.days.is_empty()); + assert!(child_usage.codex_unresolved_fork_parent); + assert!(cache.codex_pending_paths.contains(&child_key)); + assert!(!summary.history_coverage_established); + assert!(!summary.known_zero); +} + +#[test] +fn fork_child_resolves_after_parent_is_cached() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let base = Utc::now() - Duration::hours(1); + let fork = base + Duration::seconds(2); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("late-parent"), + fork, + base + Duration::seconds(3), + &[1_000_000, 1_000_140], + ); + let scanner = CostScanner::new(7) + .with_options({ + let mut options = CostScanOptions::app_driven(); + options.prefer_newest_codex_sessions_first = false; + options + }) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + + let (first, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(first.input_tokens, 0); + assert!( + first_cache + .codex_pending_paths + .contains(&child.to_string_lossy().to_string()) + ); + + let _parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "late-parent", + None, + base, + base, + &[1_000_000], + ); + + let mut resolved = None; + for _ in 0..3 { + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + if !cache.codex_scan_incomplete { + resolved = Some((summary, cache)); + break; + } + } + let (summary, cache) = resolved.expect("later bounded pass resolves the child"); + let child_usage = cached_file(&cache, &child); + assert_eq!(summary.input_tokens, 1_000_140); + assert_eq!(cached_input_total(child_usage), 140); + assert!(!child_usage.codex_unresolved_fork_parent); + assert!(cache.codex_pending_paths.is_empty()); +} + +#[test] +fn parent_last_token_after_fork_keeps_child_unresolved() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let base = Utc::now() - Duration::hours(1); + let parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base + Duration::seconds(10), + base + Duration::seconds(10), + &[1_000_000], + ); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("parent-id"), + base + Duration::seconds(1), + base + Duration::seconds(2), + &[1_000_000, 1_000_140], + ); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + let child_usage = cached_file(&cache, &child); + assert_eq!(summary.input_tokens, 1_000_000); + assert_eq!(cached_input_total(child_usage), 0); + assert!(child_usage.codex_unresolved_fork_parent); + assert!( + cache + .codex_pending_paths + .contains(&child.to_string_lossy().to_string()) + ); + assert!( + cache + .files + .contains_key(&parent.to_string_lossy().to_string()) + ); +} + +#[test] +fn deleted_unresolved_child_is_pruned_without_resurrection() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let base = Utc::now() - Duration::hours(1); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("missing-parent"), + base + Duration::seconds(1), + base + Duration::seconds(2), + &[1_000_000, 1_000_140], + ); + let scanner = app_scanner(7, &cache_root, &sessions); + let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert!( + first_cache + .codex_pending_paths + .contains(&child.to_string_lossy().to_string()) + ); + + std::fs::remove_file(&child).unwrap(); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + assert!(summary.history_coverage_established); + assert!(summary.known_zero); + assert!(cache.codex_pending_paths.is_empty()); + assert!( + !cache + .files + .contains_key(&child.to_string_lossy().to_string()) + ); +} + +#[test] +fn fork_baseline_reset_fails_closed_instead_of_billing_fresh_usage() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let base = Utc::now() - Duration::hours(1); + let _parent = write_codex_fork_session_fixture( + &sessions, + "parent.jsonl", + "parent-id", + None, + base, + base, + &[1_000_000], + ); + let child = write_codex_fork_session_fixture( + &sessions, + "child.jsonl", + "child-id", + Some("parent-id"), + base + Duration::seconds(1), + base + Duration::seconds(2), + &[1_000_000, 999_900, 1_000_140], + ); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + let child_usage = cached_file(&cache, &child); + + assert_eq!(summary.input_tokens, 1_000_000); + assert_eq!(cached_input_total(child_usage), 0); + assert!(child_usage.codex_unresolved_fork_parent); + assert!(!summary.history_coverage_established); +} diff --git a/rust/src/cost_scanner/tests/codex_pending.rs b/rust/src/cost_scanner/tests/codex_pending.rs new file mode 100644 index 0000000000..358bb6469c --- /dev/null +++ b/rust/src/cost_scanner/tests/codex_pending.rs @@ -0,0 +1,445 @@ +//! Pending Codex scan starts, trace validation and empty fragments. + +use super::*; + +fn expected_codex_scan_start(days: u32) -> String { + let today = Local::now().date_naive(); + let report_start = today - Duration::days(i64::from(days.saturating_sub(1))); + CostUsageDayRange::new(report_start, today).scan_since_key +} + +fn pending_codex_options(force: bool) -> CostScanOptions { + let mut options = if force { + CostScanOptions::app_driven() + } else { + CostScanOptions::default() + }; + options.codex_candidate_limit = 1; + options +} + +#[test] +fn pending_codex_scan_30_to_7_keeps_wide_start_and_narrow_report() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + for index in 0..3 { + write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); + } + + let first = CostScanner::new(30) + .with_options(pending_codex_options(true)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); + assert!(first_cache.codex_scan_incomplete); + + let second = CostScanner::new(7) + .with_options(pending_codex_options(false)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + let (summary, _, second_cache) = second.scan_codex_detailed_with_cache(None); + let today = Local::now().date_naive(); + assert!(second_cache.codex_scan_incomplete); + assert_eq!( + second_cache.codex_pending_scan_since_key.as_deref(), + Some(expected_codex_scan_start(30).as_str()) + ); + assert_eq!(summary.period_start, Some(today - Duration::days(6))); + assert_eq!(summary.period_end, Some(today)); +} + +#[test] +fn pending_codex_scan_7_to_30_expands_to_earliest_start() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + for index in 0..3 { + write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); + } + + let first = CostScanner::new(7) + .with_options(pending_codex_options(true)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); + assert!(first_cache.codex_scan_incomplete); + + let second = CostScanner::new(30) + .with_options(pending_codex_options(false)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + let (_, _, second_cache) = second.scan_codex_detailed_with_cache(None); + assert!(second_cache.codex_scan_incomplete); + assert_eq!( + second_cache.codex_pending_scan_since_key.as_deref(), + Some(expected_codex_scan_start(30).as_str()) + ); +} + +#[test] +fn pending_codex_scan_repeated_narrow_wide_alternation_is_monotonic() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + for index in 0..8 { + write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); + } + + let first = CostScanner::new(30) + .with_options(pending_codex_options(true)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); + assert!(first_cache.codex_scan_incomplete); + + for days in [7, 30, 7, 30, 7] { + let scanner = CostScanner::new(days) + .with_options(pending_codex_options(false)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); + assert!(cache.codex_scan_incomplete); + assert_eq!( + cache.codex_pending_scan_since_key.as_deref(), + Some(expected_codex_scan_start(30).as_str()) + ); + } +} + +#[test] +fn pending_codex_scan_incompatible_root_timezone_or_end_resets_start() { + for incompatibility in ["root", "timezone", "end"] { + let root = tempfile::tempdir().unwrap(); + let sessions = root.path().join("sessions"); + let alternate_sessions = root.path().join("alternate-sessions"); + let cache_root = root.path().join("cache"); + for index in 0..3 { + write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); + write_codex_session_fixture( + &alternate_sessions, + &format!("alternate-{index}.jsonl"), + 200 + index, + ); + } + + let first = CostScanner::new(30) + .with_options(pending_codex_options(true)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); + assert!(first_cache.codex_scan_incomplete); + + let mut persisted = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + match incompatibility { + "root" => {} + "timezone" => { + persisted.codex_pending_scan_timezone = Some("not-the-local-zone".to_string()) + } + "end" => persisted.codex_pending_scan_until_key = Some("2099-01-01".to_string()), + _ => unreachable!(), + } + JsonlScanner::save_cache(ProviderId::Codex, &mut persisted, Some(&cache_root)); + + let roots = if incompatibility == "root" { + vec![alternate_sessions] + } else { + vec![sessions] + }; + let second = CostScanner::new(7) + .with_options(pending_codex_options(false)) + .with_cache_root(&cache_root) + .with_sessions_dirs(roots); + let (_, _, second_cache) = second.scan_codex_detailed_with_cache(None); + assert!(second_cache.codex_scan_incomplete); + assert_eq!( + second_cache.codex_pending_scan_since_key.as_deref(), + Some(expected_codex_scan_start(7).as_str()), + "{incompatibility} context must reset the pending start" + ); + } +} + +#[test] +fn pending_codex_scan_force_rescan_resets_start() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + for index in 0..3 { + write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); + } + + let first = CostScanner::new(30) + .with_options(pending_codex_options(true)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); + assert!(first_cache.codex_scan_incomplete); + + let forced = CostScanner::new(7) + .with_options(pending_codex_options(true)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + let (_, _, forced_cache) = forced.scan_codex_detailed_with_cache(None); + assert!(forced_cache.codex_scan_incomplete); + assert_eq!( + forced_cache.codex_pending_scan_since_key.as_deref(), + Some(expected_codex_scan_start(7).as_str()) + ); +} + +#[test] +fn pending_codex_scan_start_survives_restart() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + for index in 0..3 { + write_codex_session_fixture(&sessions, &format!("pending-{index}.jsonl"), 100 + index); + } + + let first = CostScanner::new(30) + .with_options(pending_codex_options(true)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (_, _, first_cache) = first.scan_codex_detailed_with_cache(None); + assert_eq!( + first_cache.codex_pending_scan_since_key.as_deref(), + Some(expected_codex_scan_start(30).as_str()) + ); + + let persisted = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert_eq!( + persisted.codex_pending_scan_since_key.as_deref(), + Some(expected_codex_scan_start(30).as_str()) + ); + + let restarted = CostScanner::new(7) + .with_options(pending_codex_options(false)) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + let (_, _, restarted_cache) = restarted.scan_codex_detailed_with_cache(None); + assert_eq!( + restarted_cache.codex_pending_scan_since_key.as_deref(), + Some(expected_codex_scan_start(30).as_str()) + ); +} + +#[test] +fn disappeared_trace_path_waits_for_explicit_validation() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "disappeared.jsonl", 100); + + let initial = app_scanner(7, &cache_root, &sessions); + let (initial_summary, _, _) = initial.scan_codex_detailed_with_cache(None); + assert_eq!(initial_summary.input_tokens, 100); + + let mut cache = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + cache.last_scan_unix_ms = 1; + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + std::fs::remove_file(&path).unwrap(); + + let background = CostScanner::new(7) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions.clone()]); + let (paused_summary, _, paused_cache) = background.scan_codex_detailed_with_cache(None); + assert_eq!(paused_summary.input_tokens, 100); + assert_eq!( + paused_cache.codex_scan_pause_reason, + Some(CodexScanPauseReason::NoProgress) + ); + assert_eq!( + paused_cache.codex_pending_paths, + vec![path.to_string_lossy().to_string()] + ); + + let explicit = app_scanner(7, &cache_root, &sessions); + let (resumed_summary, _, resumed_cache) = explicit.scan_codex_detailed_with_cache(None); + assert_eq!(resumed_summary.sessions_count, 0); + assert!(resumed_summary.history_coverage_established); + assert!(resumed_cache.codex_scan_pause_reason.is_none()); + assert!(resumed_cache.codex_pending_paths.is_empty()); + assert!(resumed_cache.previous_report.is_none()); +} + +#[test] +fn paused_catch_up_round_trips_without_retrying_in_background() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let pending = write_codex_session_fixture(&sessions, "pending.jsonl", 100); + let report = CachedCostReport { + total_cost_usd: 9.25, + input_tokens: 21, + cached_tokens: 1, + output_tokens: 5, + reasoning_tokens: None, + sessions_count: 2, + updated_at: Some("2026-09-06T00:00:00Z".to_string()), + partial: true, + }; + let mut cache = CostUsageCache { + previous_report: Some(report.clone()), + codex_pending_paths: vec![pending.to_string_lossy().to_string()], + codex_scan_incomplete: true, + codex_scan_pause_reason: Some(CodexScanPauseReason::NoProgress), + ..Default::default() + }; + + let encoded = serde_json::to_string(&cache).unwrap(); + let decoded: CostUsageCache = serde_json::from_str(&encoded).unwrap(); + assert_eq!( + decoded.codex_scan_pause_reason, + cache.codex_scan_pause_reason + ); + + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + let scanner = CostScanner::new(7) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + let (summary, stats, saved) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(stats.files_parsed, 0); + assert_eq!(summary.total_cost_usd, report.total_cost_usd); + assert_eq!(saved.codex_pending_paths, cache.codex_pending_paths); + assert_eq!( + saved + .previous_report + .as_ref() + .map(|saved| saved.total_cost_usd), + Some(report.total_cost_usd) + ); + assert_eq!(saved.codex_scan_pause_reason, cache.codex_scan_pause_reason); +} + +#[test] +fn pending_and_incomplete_round_trip_through_cache_json() { + let cache = CostUsageCache { + codex_pending_paths: vec!["C:\\sessions\\pending.jsonl".to_string()], + codex_scan_incomplete: true, + ..Default::default() + }; + + let encoded = serde_json::to_string(&cache).unwrap(); + let decoded: CostUsageCache = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded.codex_pending_paths, cache.codex_pending_paths); + assert!(decoded.codex_scan_incomplete); +} + +#[test] +fn deleted_pending_path_is_pruned_after_complete_discovery() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture_with_inputs(&sessions, "partial.jsonl", &[100, 200]); + let first_line_bytes = i64::try_from( + std::fs::read(&path) + .unwrap() + .split(|byte| *byte == b'\n') + .next() + .unwrap() + .len(), + ) + .expect("fixture line length fits i64") + + 1; + let mut options = CostScanOptions::app_driven(); + options.codex_max_session_file_bytes = first_line_bytes; + options.codex_max_scan_bytes_per_refresh = first_line_bytes; + let scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&cache_root) + .with_sessions_dirs(vec![sessions]); + let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + assert!(first_cache.codex_scan_incomplete); + assert_eq!(first_cache.codex_pending_paths.len(), 1); + + std::fs::remove_file(&path).unwrap(); + let (summary, _, second_cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(summary.sessions_count, 0); + assert!(summary.history_coverage_established); + assert!(second_cache.codex_pending_paths.is_empty()); + assert!(!second_cache.codex_scan_incomplete); + assert!( + !second_cache + .files + .contains_key(&path.to_string_lossy().to_string()) + ); + assert!(second_cache.days.is_empty()); +} + +#[test] +fn legacy_cache_json_defaults_bounded_scan_state() { + let legacy = r#"{"last_scan_unix_ms":0,"files":{},"days":{}}"#; + let cache: CostUsageCache = serde_json::from_str(legacy).unwrap(); + assert!(cache.codex_pending_paths.is_empty()); + assert!(!cache.codex_scan_incomplete); +} + +#[test] +fn complete_empty_codex_fragment_persists_in_cache() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "empty.jsonl", 100); + std::fs::write(&path, b"\n").unwrap(); + let key = path.to_string_lossy().to_string(); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(summary.input_tokens, 0); + let entry = cache.files.get(&key).expect("empty fragment is cached"); + assert!(entry.days.is_empty()); + assert_eq!(entry.parsed_bytes, Some(1)); + assert_eq!(entry.codex_scan_target_size, Some(1)); + + let persisted = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!(persisted.files.contains_key(&key)); +} + +#[test] +fn complete_empty_codex_fragment_reparses_from_start_after_growth() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "empty.jsonl", 100); + std::fs::write(&path, b"\n").unwrap(); + let key = path.to_string_lossy().to_string(); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (_, _, first_cache) = scanner.scan_codex_detailed_with_cache(None); + let first = first_cache.files.get(&key).expect("initial empty fragment"); + assert_eq!(first.parsed_bytes, Some(1)); + assert_eq!(first.codex_scan_target_size, Some(1)); + + write_codex_session_fixture(&sessions, "empty.jsonl", 100); + let (grown_summary, stats, grown_cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(grown_summary.input_tokens, 100); + assert_eq!(stats.files_resumed, 0); + assert!(!grown_cache.files[&key].days.is_empty()); +} + +#[test] +fn incomplete_or_buffered_empty_codex_fragment_is_not_marked_complete() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let path = write_codex_session_fixture(&sessions, "incomplete.jsonl", 100); + std::fs::write(&path, br#"{"timestamp":"2026-09-07T00:00:00Z""#).unwrap(); + let key = path.to_string_lossy().to_string(); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (_, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + let entry = cache + .files + .get(&key) + .expect("incomplete fragment is tracked"); + assert!(entry.days.is_empty()); + assert_ne!(entry.parsed_bytes, Some(entry.size)); + assert!(cache.codex_scan_incomplete); + assert!(cache.codex_pending_paths.contains(&key)); + + let buffered_root = tempfile::tempdir().unwrap(); + let buffered_sessions = buffered_root.path().join("sessions"); + let buffered_cache_root = buffered_root.path().join("cache"); + let buffered_path = write_codex_session_fixture(&buffered_sessions, "buffered.jsonl", 100); + std::fs::write(&buffered_path, b"\nnot-yet-read").unwrap(); + let buffered_key = buffered_path.to_string_lossy().to_string(); + let mut options = CostScanOptions::app_driven(); + options.codex_max_session_file_bytes = 1; + options.codex_max_scan_bytes_per_refresh = 1; + let buffered_scanner = CostScanner::new(7) + .with_options(options) + .with_cache_root(&buffered_cache_root) + .with_sessions_dirs(vec![buffered_sessions]); + let (_, _, buffered_cache) = buffered_scanner.scan_codex_detailed_with_cache(None); + + let buffered_entry = buffered_cache + .files + .get(&buffered_key) + .expect("buffered fragment is tracked"); + assert!(buffered_entry.days.is_empty()); + assert_ne!(buffered_entry.parsed_bytes, Some(buffered_entry.size)); + assert!(buffered_cache.codex_scan_incomplete); + assert!(buffered_cache.codex_pending_paths.contains(&buffered_key)); +} diff --git a/rust/src/cost_scanner/tests/codex_usage.rs b/rust/src/cost_scanner/tests/codex_usage.rs new file mode 100644 index 0000000000..a7f2104589 --- /dev/null +++ b/rust/src/cost_scanner/tests/codex_usage.rs @@ -0,0 +1,329 @@ +//! Codex token payloads, reasoning tokens and cached day rebuilds. + +use super::*; + +#[test] +fn parses_current_codex_payload_token_count_events() { + let path = std::env::temp_dir().join(format!( + "codexbar-current-codex-token-count-{}.jsonl", + std::process::id() + )); + // Use a recent timestamp so the event stays inside the scanner's + // 30-day window no matter when the test runs. A hardcoded date + // silently ages out of the window and makes this test fail with 0 + // sessions once it is more than 30 days in the past. + let recent = (Utc::now() - Duration::hours(1)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string(); + let mut file = File::create(&path).unwrap(); + writeln!( + file, + r#"{{"timestamp":"{ts}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":125,"cached_input_tokens":30,"output_tokens":15}}}}}}}}"#, + ts = recent + ) + .unwrap(); + let scanner = CostScanner::new(30); + let mut summary = CostSummary::default(); + let today = Local::now().date_naive(); + let range = CostUsageDayRange::new(codex_period_start(today, 30), today); + let mut cache = CostUsageCache::default(); + let mut stats = CostScanStats::default(); + scanner.parse_codex_file(&path, &range, &mut summary, &mut cache, None, &mut stats); + + assert_eq!(summary.sessions_count, 1); + assert_eq!(summary.input_tokens, 125); + assert_eq!(summary.cached_tokens, 30); + assert_eq!(summary.output_tokens, 15); + assert_eq!( + summary + .by_model_tokens + .get("gpt-5") + .map(ModelTokenCounts::total), + Some(140) + ); + assert!(scan_codex_file_cost(&path) > 0.0); + // Best-effort test cleanup; the file may already be gone. + let _removed = std::fs::remove_file(&path); +} + +#[test] +fn scans_gpt6_astra_usage_with_cached_and_reasoning_tokens() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + let today = Local::now().date_naive(); + let day = today.format("%Y-%m-%d").to_string(); + let day_dir = partition_dir(&sessions, today); + std::fs::create_dir_all(&day_dir).unwrap(); + let line = serde_json::json!({ + "timestamp": Local::now().to_rfc3339(), + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "model": "gpt-6-astra", + "total_token_usage": { + "input_tokens": 1000, + "cached_input_tokens": 300, + "output_tokens": 100, + "reasoning_output_tokens": 7 + } + } + } + }); + std::fs::write(day_dir.join("astra.jsonl"), format!("{line}\n")).unwrap(); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + + assert_eq!(summary.input_tokens, 1000); + assert_eq!(summary.cached_tokens, 300); + assert_eq!(summary.output_tokens, 100); + assert_eq!(summary.reasoning_tokens, Some(7)); + // Codex token-count rows expose cache reads, not cache writes. The 700 + // non-cached input tokens therefore use Astra's standard input rate. + assert!((summary.total_cost_usd - 0.0123).abs() < 1e-12); + assert_eq!(cache.days[&day]["gpt-6-astra"], vec![1000, 300, 100, 7]); +} + +#[test] +fn recent_codex_fixture_time_stays_on_the_local_day() { + let utc_plus_7 = FixedOffset::east_opt(7 * 3600).unwrap(); + let at = |hour, minute| { + utc_plus_7 + .with_ymd_and_hms(2026, 10, 1, hour, minute, 0) + .unwrap() + }; + assert_eq!(recent_fixture_time_at(at(8, 30)), at(7, 30)); + assert_eq!(recent_fixture_time_at(at(1, 0)), at(0, 0)); + assert_eq!(recent_fixture_time_at(at(0, 40)), at(0, 0)); + assert_eq!(recent_fixture_time_at(at(0, 0)), at(0, 0)); + + let ci_run = Utc.with_ymd_and_hms(2026, 10, 1, 0, 45, 0).unwrap(); + assert_eq!( + recent_fixture_time_at(ci_run), + Utc.with_ymd_and_hms(2026, 10, 1, 0, 0, 0).unwrap() + ); +} + +#[test] +fn rebuild_cache_days_preserves_known_reasoning() { + let day = Local::now().format("%Y-%m-%d").to_string(); + let mut cache = CostUsageCache { + files: HashMap::from([ + ( + "a".to_string(), + cached_usage_with_packed(&day, "gpt-5", vec![10, 0, 4, 3]), + ), + ( + "b".to_string(), + cached_usage_with_packed(&day, "gpt-5", vec![5, 0, 2, 1]), + ), + ]), + ..CostUsageCache::default() + }; + + rebuild_cache_days(&mut cache); + + assert_eq!(cache.days[&day]["gpt-5"], vec![15, 0, 6, 4]); +} + +#[test] +fn rebuild_cache_days_reasoning_unknown_is_order_independent() { + let run = |first: Vec, second: Vec| { + let day = Local::now().format("%Y-%m-%d").to_string(); + let mut cache = CostUsageCache { + files: HashMap::from([ + ( + "a".to_string(), + cached_usage_with_packed(&day, "gpt-5", first), + ), + ( + "b".to_string(), + cached_usage_with_packed(&day, "gpt-5", second), + ), + ]), + ..CostUsageCache::default() + }; + + rebuild_cache_days(&mut cache); + cache.days[&day]["gpt-5"].clone() + }; + + assert_eq!(run(vec![10, 0, 4, 3], vec![5, 0, 2]), vec![15, 0, 6]); + assert_eq!(run(vec![5, 0, 2], vec![10, 0, 4, 3]), vec![15, 0, 6]); +} + +#[test] +fn rebuild_cache_days_zero_row_does_not_poison_reasoning() { + let day = Local::now().format("%Y-%m-%d").to_string(); + let mut cache = CostUsageCache { + files: HashMap::from([ + ( + "a".to_string(), + cached_usage_with_packed(&day, "gpt-5", vec![10, 0, 4, 3]), + ), + ( + "b".to_string(), + cached_usage_with_packed(&day, "gpt-5", vec![0, 0, 0]), + ), + ]), + ..CostUsageCache::default() + }; + + rebuild_cache_days(&mut cache); + + assert_eq!(cache.days[&day]["gpt-5"], vec![10, 0, 4, 3]); +} + +#[test] +fn rebuild_cache_days_aggregates_multiple_files_above_i32_max() { + let day = Local::now().format("%Y-%m-%d").to_string(); + let mut cache = CostUsageCache { + files: HashMap::from([ + ( + "a".to_string(), + cached_usage_with_packed(&day, "gpt-5", vec![1_500_000_000, 1_400_000_000, 100]), + ), + ( + "b".to_string(), + cached_usage_with_packed(&day, "gpt-5", vec![1_500_000_000, 1_400_000_000, 100]), + ), + ]), + ..CostUsageCache::default() + }; + + rebuild_cache_days(&mut cache); + + assert_eq!( + cache.days[&day]["gpt-5"], + vec![3_000_000_000, 2_800_000_000, 200] + ); +} + +#[test] +fn retained_report_sums_multiple_days_above_i32_max() { + let day_a = "2026-09-08"; + let day_b = "2026-09-09"; + let mut cache = CostUsageCache { + files: HashMap::from([ + ( + "a".to_string(), + cached_usage_with_packed( + day_a, + "gpt-5.6-sol", + vec![1_500_000_000, 1_400_000_000, 1_000_000], + ), + ), + ( + "b".to_string(), + cached_usage_with_packed( + day_b, + "gpt-5.6-sol", + vec![1_500_000_000, 1_400_000_000, 1_000_000], + ), + ), + ]), + ..CostUsageCache::default() + }; + rebuild_cache_days(&mut cache); + + let report = JsonlScanner::cached_cost_report_from_days(&cache); + assert_eq!(report.input_tokens, 3_000_000_000); + assert_eq!(report.cached_tokens, 2_800_000_000); + assert_eq!(report.output_tokens, 2_000_000); + + let start = chrono::NaiveDate::from_ymd_opt(2026, 9, 8).unwrap(); + let end = chrono::NaiveDate::from_ymd_opt(2026, 9, 9).unwrap(); + let summary = summary_from_cached_report(&report, start, end); + assert_eq!(summary.input_tokens, 3_000_000_000); + assert_eq!(summary.cached_tokens, 2_800_000_000); + assert_eq!(summary.output_tokens, 2_000_000); + assert_eq!(summary.sessions_count, 2); +} + +#[test] +fn reasoning_survives_scan_rebuild_and_cache_reload() { + let (_root, sessions, cache_root) = codex_scan_dirs(); + // The event timestamp (now − 1h) decides the parsed day key, so derive the + // fixture day from that same instant: at local 00:00–01:00 now − 1h falls + // on the previous local day and the row would land there, not on today. + let event_time = Utc::now() - Duration::hours(1); + let today = event_time.with_timezone(&Local).date_naive(); + let day = today.format("%Y-%m-%d").to_string(); + let day_dir = partition_dir(&sessions, today); + std::fs::create_dir_all(&day_dir).unwrap(); + let timestamp = event_time.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let reasoning_line = serde_json::json!({ + "timestamp": timestamp, + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "model": "gpt-5", + "total_token_usage": { + "input_tokens": 100, + "cached_input_tokens": 0, + "output_tokens": 20, + "reasoning_output_tokens": 7 + } + } + } + }); + std::fs::write( + day_dir.join("reasoning.jsonl"), + format!("{reasoning_line}\n"), + ) + .unwrap(); + + let scanner = app_scanner(7, &cache_root, &sessions); + let (summary, _, cache) = scanner.scan_codex_detailed_with_cache(None); + assert_eq!(summary.output_tokens, 20); + assert_eq!(summary.reasoning_tokens, Some(7)); + let row = &cache.days[&day]["gpt-5"]; + assert!(row.len() >= 4); + assert_eq!(row[3], 7); + + let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + let loaded_row = &loaded.days[&day]["gpt-5"]; + assert!(loaded_row.len() >= 4); + assert_eq!(loaded_row[3], 7); + assert_eq!( + JsonlScanner::cached_cost_report_from_days(&loaded).reasoning_tokens, + Some(7) + ); + + let legacy_root = tempfile::tempdir().unwrap(); + let legacy_sessions = legacy_root.path().join("sessions"); + let legacy_cache_root = legacy_root.path().join("cache"); + let legacy_day_dir = partition_dir(&legacy_sessions, today); + std::fs::create_dir_all(&legacy_day_dir).unwrap(); + let legacy_line = serde_json::json!({ + "timestamp": timestamp, + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "model": "gpt-5", + "total_token_usage": { + "input_tokens": 100, + "cached_input_tokens": 0, + "output_tokens": 20 + } + } + } + }); + std::fs::write( + legacy_day_dir.join("legacy.jsonl"), + format!("{legacy_line}\n"), + ) + .unwrap(); + + let legacy_scanner = app_scanner(7, &legacy_cache_root, &legacy_sessions); + let (legacy_summary, _, legacy_cache) = legacy_scanner.scan_codex_detailed_with_cache(None); + assert_eq!(legacy_summary.output_tokens, 20); + assert_eq!(legacy_summary.reasoning_tokens, None); + assert_eq!(legacy_cache.days[&day]["gpt-5"], vec![100, 0, 20]); + assert!( + (summary.total_cost_usd - legacy_summary.total_cost_usd).abs() < 1e-12, + "reasoning metadata must not change cost" + ); +} diff --git a/rust/src/cost_scanner/tests/support.rs b/rust/src/cost_scanner/tests/support.rs index 1dc4aa823c..08d25462b5 100644 --- a/rust/src/cost_scanner/tests/support.rs +++ b/rust/src/cost_scanner/tests/support.rs @@ -36,3 +36,178 @@ pub(super) fn cached_file<'a>(cache: &'a CostUsageCache, path: &Path) -> &'a Cos .get(&path.to_string_lossy().to_string()) .unwrap() } + +/// An event time for a fresh Codex session fixture: an hour ago, kept on today's local date. +/// +/// The scanner files each event under its local date, and the session fixtures live in today's +/// date folder. A plain `now - 1h` lands on yesterday in the first hour after local midnight +/// (00:00-01:00Z on the UTC CI runner), so tests that read today's bucket or rely on the day +/// folder scan order failed in that hour. +pub(super) fn recent_codex_fixture_time() -> DateTime { + recent_fixture_time_at(Local::now()) +} + +/// `now - 1h`, or the start of `now`'s local day when that hour reaches back into yesterday. +pub(super) fn recent_fixture_time_at(now: DateTime) -> DateTime { + let hour_ago = now.clone() - Duration::hours(1); + if hour_ago.date_naive() == now.date_naive() { + return hour_ago.with_timezone(&Utc); + } + now.timezone() + .from_local_datetime(&now.date_naive().and_time(NaiveTime::MIN)) + .earliest() + .unwrap_or(now) + .with_timezone(&Utc) +} + +pub(super) fn write_codex_session_fixture( + sessions_root: &Path, + name: &str, + input_tokens: u64, +) -> PathBuf { + let event_time = recent_codex_fixture_time(); + let today = event_time.with_timezone(&Local).date_naive(); + let day_dir = partition_dir(sessions_root, today); + std::fs::create_dir_all(&day_dir).unwrap(); + let path = day_dir.join(name); + let ts = event_time.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let body = format!( + r#"{{"timestamp":"{ts}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":{input_tokens},"cached_input_tokens":0,"output_tokens":5}}}}}}}} +"# + ); + std::fs::write(&path, body).unwrap(); + path +} + +pub(super) fn write_codex_session_fixture_with_inputs( + sessions_root: &Path, + name: &str, + input_tokens: &[u64], +) -> PathBuf { + let base = recent_codex_fixture_time(); + let today = base.with_timezone(&Local).date_naive(); + let day_dir = partition_dir(sessions_root, today); + std::fs::create_dir_all(&day_dir).unwrap(); + let mut body = String::new(); + for (index, input) in input_tokens.iter().enumerate() { + let timestamp = (base + + Duration::seconds(i64::try_from(index).expect("fixture index fits i64"))) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string(); + body.push_str(&format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5","total_token_usage":{{"input_tokens":{input},"cached_input_tokens":0,"output_tokens":5}}}}}}}} +"# + )); + } + let path = day_dir.join(name); + std::fs::write(&path, body).unwrap(); + path +} + +pub(super) fn cached_usage_with_packed( + day: &str, + model: &str, + packed: Vec, +) -> CostUsageFileUsage { + CostUsageFileUsage { + parsed_bytes: Some(1), + ..test_file_usage( + 1, + HashMap::from([( + day.to_string(), + HashMap::from([(model.to_string(), packed)]), + )]), + ) + } +} + +pub(super) fn write_codex_fork_session_fixture( + sessions_root: &Path, + name: &str, + session_id: &str, + parent_id: Option<&str>, + fork_timestamp: DateTime, + token_start: DateTime, + totals: &[i64], +) -> PathBuf { + let today = Local::now().date_naive(); + let day_dir = partition_dir(sessions_root, today); + std::fs::create_dir_all(&day_dir).unwrap(); + + let mut body = format!( + "{{\"type\":\"session_meta\",\"timestamp\":\"{}\",\"payload\":{{\"session_id\":\"{}\"", + fork_timestamp.to_rfc3339(), + session_id + ); + if let Some(parent_id) = parent_id { + body.push_str(&format!(",\"forked_from_id\":\"{parent_id}\"")); + } + body.push_str("}}\n"); + + for (index, total) in totals.iter().enumerate() { + let timestamp = (token_start + + Duration::seconds(i64::try_from(index).expect("fixture index fits i64"))) + .to_rfc3339(); + let line = serde_json::json!({ + "timestamp": timestamp, + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "model": "gpt-5", + "total_token_usage": { + "input_tokens": total, + "cached_input_tokens": 0, + "output_tokens": 5 + } + } + } + }); + body.push_str(&line.to_string()); + body.push('\n'); + } + + let path = day_dir.join(name); + std::fs::write(&path, body).unwrap(); + path +} + +pub(super) fn cached_input_total(usage: &CostUsageFileUsage) -> i64 { + usage + .days + .values() + .flat_map(|models| models.values()) + .map(|tokens| tokens.first().copied().unwrap_or_default()) + .sum() +} + +/// Stream the de-duplicated, in-window usage records from one transcript +/// file into `on_record`, returning how many it consumed. +pub(super) fn for_each_claude_usage_record( + path: &Path, + cutoff: &DateTime, + seen: &mut HashSet, + cancel: Option<&AtomicBool>, + on_record: F, +) -> usize +where + F: FnMut(&ClaudeUsageRecord), +{ + let mut pricing = ClaudeScanPricingResolver::default(); + let mut incomplete = ClaudeIncompleteTracker::default(); + scan_claude_file_with_pricing( + path, + cutoff, + seen, + cancel, + &mut pricing, + &mut incomplete, + on_record, + ) + .counted +} + +pub(super) fn claude_usage_record_from_event(event: &ClaudeEvent) -> Option { + let mut pricing = ClaudeScanPricingResolver::default(); + claude_usage_record_from_event_with_pricing(event, &mut pricing) +} diff --git a/rust/src/cost_scanner/tests/totals.rs b/rust/src/cost_scanner/tests/totals.rs new file mode 100644 index 0000000000..2cdf8e2a58 --- /dev/null +++ b/rust/src/cost_scanner/tests/totals.rs @@ -0,0 +1,88 @@ +//! Provider-specific cache rules across model, day and window totals. + +use super::*; + +// Regression (PR #611 review): Codex `input_tokens` already contains cached +// input, Claude cache buckets are separate. Every total must follow the +// provider's rule, and the model, daily and window totals must agree. +#[test] +fn provider_totals_add_cache_only_when_it_is_separate_from_input() { + let counts = ModelTokenCounts { + input_tokens: 100, + output_tokens: 5, + cached_tokens: 90, + reasoning_tokens: None, + }; + assert_eq!(counts.total_for_provider("codex"), 105); + assert_eq!(counts.total_for_provider("claude"), 195); + + let summary = CostSummary { + input_tokens: 100, + output_tokens: 5, + cached_tokens: 90, + ..CostSummary::default() + }; + assert_eq!(summary.total_tokens_for_provider("codex"), 105); + assert_eq!(summary.total_tokens_for_provider("claude"), 195); +} + +#[test] +fn codex_day_total_excludes_cached_input_and_matches_model_totals() { + let day_key = "2026-08-18".to_string(); + let day = CostUsageDayRange::parse_day_key(&day_key).expect("day"); + let range = CostUsageDayRange::new(day, day); + let mut one_day = HashMap::new(); + one_day.insert( + day_key, + HashMap::from([("gpt-5".to_string(), vec![1_000, 900, 50])]), + ); + let mut scratch = CostSummary::default(); + add_codex_days_map_to_summary(&mut scratch, &one_day, &range); + + assert_eq!(scratch.input_tokens, 1_000); + assert_eq!(scratch.cached_tokens, 900); + // The value `get_daily_token_history("codex")` stores for the day. + let day_total = scratch.total_tokens_for_provider("codex"); + assert_eq!(day_total, 1_050, "cached input is already inside input"); + let model_total: u64 = scratch + .by_model_tokens + .values() + .map(|counts| counts.total_for_provider("codex")) + .sum(); + assert_eq!(model_total, day_total); +} + +#[test] +fn claude_day_total_includes_cache_and_matches_summary_and_model_totals() { + use chrono::TimeZone as _; + let timestamp = Local + .with_ymd_and_hms(2026, 8, 18, 12, 0, 0) + .single() + .expect("local time") + .with_timezone(&Utc); + let record = ClaudeUsageRecord { + model: "claude-sonnet-4-5".to_string(), + pricing_known: true, + timestamp: Some(timestamp), + dedup_key: None, + input: 10, + output: 20, + cache_create: 300, + cache_read: 4_000, + cost: Some(0.0), + }; + let mut summary = CostSummary::default(); + assert!(add_claude_record_to_summary(&mut summary, &record)); + let mut daily = HashMap::from([("2026-08-18".to_string(), 0u64)]); + assert!(add_claude_record_to_daily_tokens(&mut daily, &record)); + + let window_total = summary.total_tokens_for_provider("claude"); + assert_eq!(window_total, 4_330); + assert_eq!(daily["2026-08-18"], window_total); + let model_total: u64 = summary + .by_model_tokens + .values() + .map(|counts| counts.total_for_provider("claude")) + .sum(); + assert_eq!(model_total, window_total); +} From 9b152574c7884e105f6209251ed53a6d10516f0c Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:00:25 +0700 Subject: [PATCH 36/41] Split the Codex JSONL scanner tests by topic --- rust/src/core/jsonl_scanner/tests.rs | 1774 +---------------- .../core/jsonl_scanner/tests/cache_files.rs | 588 ++++++ .../core/jsonl_scanner/tests/line_parsing.rs | 414 ++++ .../core/jsonl_scanner/tests/timestamps.rs | 198 ++ .../jsonl_scanner/tests/token_accounting.rs | 360 ++++ .../core/jsonl_scanner/tests/usage_rows.rs | 214 ++ 6 files changed, 1789 insertions(+), 1759 deletions(-) create mode 100644 rust/src/core/jsonl_scanner/tests/cache_files.rs create mode 100644 rust/src/core/jsonl_scanner/tests/line_parsing.rs create mode 100644 rust/src/core/jsonl_scanner/tests/timestamps.rs create mode 100644 rust/src/core/jsonl_scanner/tests/token_accounting.rs create mode 100644 rust/src/core/jsonl_scanner/tests/usage_rows.rs diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 11f8d38bb0..43c261bdfa 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -19,1765 +19,6 @@ impl CodexParserState { } } -fn codex_timestamp_day_key(timestamp: &str) -> Option { - parse_codex_timestamp(timestamp).map(|parsed| parsed.day_key()) -} - -fn last_usage_delta(last: &Value) -> (i64, i64, i64, Option) { - let totals = read_token_totals(last); - ( - totals.input.max(0), - totals.cached.max(0), - totals.output.max(0), - totals.reasoning, - ) -} - -#[test] -fn test_day_range() { - let since = NaiveDate::from_ymd_opt(2026, 1, 15).unwrap(); - let until = NaiveDate::from_ymd_opt(2026, 1, 20).unwrap(); - let range = CostUsageDayRange::new(since, until); - - assert_eq!(range.since_key, "2026-01-15"); - assert_eq!(range.until_key, "2026-01-20"); - assert_eq!(range.scan_since_key, "2026-01-14"); - assert_eq!(range.scan_until_key, "2026-01-21"); -} - -#[test] -fn reasoning_output_is_clamped_and_preserved_for_value_and_fast_shapes() { - let value = serde_json::json!({ - "output_tokens": 20, - "reasoning_output_tokens": 7, - }); - let totals = read_token_totals(&value); - assert_eq!(totals.output, 20); - assert_eq!(totals.reasoning, Some(7)); - assert_eq!(last_usage_delta(&value), (0, 0, 20, Some(7))); - - let fast: CodexFastTotals = serde_json::from_value(serde_json::json!({ - "output_tokens": 20, - "reasoning_output_tokens": 99, - })) - .unwrap(); - let fast_totals = codex_totals_from_fast(fast); - assert_eq!(fast_totals.output, 20); - assert_eq!(fast_totals.reasoning, Some(20)); -} - -#[test] -fn missing_reasoning_stays_unknown_for_cumulative_and_event_usage() { - let value = serde_json::json!({ "output_tokens": 20 }); - assert_eq!(read_token_totals(&value).reasoning, None); - assert_eq!(last_usage_delta(&value), (0, 0, 20, None)); - - let mut state = CodexParserState::new(None, None); - assert_eq!( - state.total_usage_delta(&serde_json::json!({ - "output_tokens": 10, - })), - (0, 0, 10, None) - ); -} - -#[test] -fn cumulative_reasoning_uses_the_comparable_previous_total() { - let mut state = CodexParserState::new(None, None); - assert_eq!( - state.total_usage_delta(&serde_json::json!({ - "output_tokens": 10, - "reasoning_output_tokens": 4, - })), - (0, 0, 10, Some(4)) - ); - assert_eq!( - state.total_usage_delta(&serde_json::json!({ - "output_tokens": 20, - "reasoning_output_tokens": 9, - })), - (0, 0, 10, Some(5)) - ); -} - -#[test] -fn fork_baseline_subtracts_known_reasoning_without_affecting_core_tokens() { - let baseline = CodexTotals { - input: 10, - cached: 2, - output: 10, - reasoning: Some(4), - }; - let mut state = CodexParserState::from_mode(CodexParseMode::ParentBaseline { - baseline, - paginated_continuation: false, - remaining_inherited_totals: None, - }); - assert_eq!( - state.apply_totals_delta(CodexTotals { - input: 20, - cached: 5, - output: 20, - reasoning: Some(9), - }), - (10, 3, 10, Some(5)) - ); - - let baseline_without_reasoning = CodexTotals { - input: 10, - cached: 2, - output: 10, - reasoning: None, - }; - let mut state = CodexParserState::from_mode(CodexParseMode::ParentBaseline { - baseline: baseline_without_reasoning, - paginated_continuation: false, - remaining_inherited_totals: None, - }); - assert_eq!( - state.apply_totals_delta(CodexTotals { - input: 20, - cached: 5, - output: 20, - reasoning: Some(9), - }), - (10, 3, 10, None) - ); - assert!(!state.fork_baseline_ambiguous); -} - -#[test] -fn inferred_fork_waits_for_present_explicit_start_ordinal() { - let range = CostUsageDayRange::new( - NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), - NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), - ); - let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { - start_ordinal: Some(10), - }); - - state.process_line( - r#"{"timestamp":"2026-09-22T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.6-sol","total_token_usage":{"input_tokens":100,"cached_input_tokens":20,"output_tokens":10},"last_token_usage":{"input_tokens":0,"cached_input_tokens":0,"output_tokens":0}}}}"#, - &range, - ); - - assert!(state.records.is_empty()); - assert!(state.fork_baseline.is_none()); - assert!(!state.fork_baseline_locally_resolved()); - - state.process_line( - r#"{"ordinal":10,"timestamp":"2026-09-22T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.6-sol","total_token_usage":{"input_tokens":110,"cached_input_tokens":22,"output_tokens":11},"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":1}}}}"#, - &range, - ); - - assert_eq!(state.records.len(), 1); - assert_eq!(state.records[0].0.input, 10); - assert_eq!(state.records[0].0.cached, 2); - assert_eq!(state.records[0].0.output, 1); - assert!(!state.fork_baseline_locally_resolved()); -} - -#[test] -fn inferred_fork_keeps_missing_ordinal_unresolved_after_boundary_opens() { - let range = CostUsageDayRange::new( - NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), - NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), - ); - let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { - start_ordinal: Some(10), - }); - let token_line = |ordinal: Option, total: i64, last: i64| { - let mut value = serde_json::json!({ - "timestamp": "2026-09-22T10:00:00Z", - "type": "event_msg", - "payload": {"type": "token_count", "info": { - "model": "gpt-5.6-sol", - "total_token_usage": {"input_tokens": total, "cached_input_tokens": 0, "output_tokens": 0}, - "last_token_usage": {"input_tokens": last, "cached_input_tokens": 0, "output_tokens": 0} - }} - }); - if let Some(ordinal) = ordinal { - value["ordinal"] = serde_json::json!(ordinal); - } - value.to_string() - }; - - state.process_line(&token_line(Some(9), 100, 0), &range); - state.process_line(&token_line(Some(10), 100, 0), &range); - state.process_line(&token_line(Some(11), 110, 110), &range); - assert!(state.fork_baseline_locally_resolved()); - state.process_line(&token_line(None, 120, 10), &range); - assert!(!state.fork_baseline_locally_resolved()); - state.process_line(&token_line(Some(12), 130, 10), &range); - - assert_eq!(state.records.len(), 1); - assert_eq!(state.records[0].0.input, 10); - assert!(!state.fork_baseline_locally_resolved()); -} - -#[test] -fn inferred_fork_keeps_missing_ordinal_unresolved_after_local_resolution() { - let range = CostUsageDayRange::new( - NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), - NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), - ); - let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { - start_ordinal: Some(10), - }); - let token_line = |ordinal: Option, total: i64, last: i64| { - let mut value = serde_json::json!({ - "timestamp": "2026-09-22T10:00:00Z", - "type": "event_msg", - "payload": {"type": "token_count", "info": { - "model": "gpt-5.6-sol", - "total_token_usage": {"input_tokens": total, "cached_input_tokens": 0, "output_tokens": 0}, - "last_token_usage": {"input_tokens": last, "cached_input_tokens": 0, "output_tokens": 0} - }} - }); - if let Some(ordinal) = ordinal { - value["ordinal"] = serde_json::json!(ordinal); - } - value.to_string() - }; - - state.process_line(&token_line(Some(9), 100, 0), &range); - state.process_line(&token_line(Some(10), 100, 0), &range); - state.process_line(&token_line(Some(11), 110, 10), &range); - assert!(state.fork_baseline_locally_resolved()); - state.process_line(&token_line(None, 120, 10), &range); - - assert!(!state.fork_baseline_locally_resolved()); -} - -#[test] -fn codex_token_pipeline_preserves_counts_above_i32_max() { - let parsed = read_token_totals(&serde_json::json!({ - "input_tokens": 3_000_000_000_i64, - "cached_input_tokens": 2_800_000_000_i64, - "output_tokens": 200, - })); - assert_eq!(parsed.input, 3_000_000_000); - assert_eq!(parsed.cached, 2_800_000_000); - assert_eq!(parsed.output, 200); - - let mut packed = Vec::new(); - for _ in 0..2 { - JsonlScanner::merge_codex_record_into_packed( - &mut packed, - &CodexUsageRecord { - day_key: "2026-09-09".to_string(), - timestamp: None, - model: "gpt-5.6-luna".to_string(), - input: 1_500_000_000, - cached: 1_400_000_000, - output: 100, - reasoning: None, - turn_id: None, - }, - ); - } - assert_eq!(packed, vec![3_000_000_000, 2_800_000_000, 200]); - - let mut cache = CostUsageCache::default(); - cache.days.insert( - "2026-09-09".to_string(), - HashMap::from([("gpt-5.6-luna".to_string(), packed)]), - ); - let report = JsonlScanner::cached_cost_report_from_days(&cache); - assert_eq!(report.input_tokens, 3_000_000_000); - assert_eq!(report.cached_tokens, 2_800_000_000); - assert_eq!(report.output_tokens, 200); -} - -#[test] -fn negative_cumulative_components_are_clamped_at_the_source() { - let value = serde_json::json!({ - "input_tokens": -5, - "cached_input_tokens": -9, - "cache_read_input_tokens": -3, - "output_tokens": -2, - "reasoning_output_tokens": -1, - }); - let totals = read_token_totals(&value); - assert_eq!(totals.input, 0); - assert_eq!(totals.cached, 0); - assert_eq!(totals.output, 0); - assert_eq!(totals.reasoning, Some(0)); - - let fast: CodexFastTotals = serde_json::from_value(value.clone()).unwrap(); - let fast_totals = codex_totals_from_fast(fast); - assert_eq!(fast_totals.input, 0); - assert_eq!(fast_totals.cached, 0); - assert_eq!(fast_totals.output, 0); - assert_eq!(fast_totals.reasoning, Some(0)); - - // The payload borrows `&str` fields, so deserialize from a str rather than - // an owned `Value`. - let payload_json = value.to_string(); - let payload: CodexFastPayload<'_> = serde_json::from_str(&payload_json).unwrap(); - let payload_totals = fast_totals_from_payload(&payload); - assert_eq!(payload_totals.input, 0); - assert_eq!(payload_totals.cached, 0); - assert_eq!(payload_totals.output, 0); - assert_eq!(payload_totals.reasoning, Some(0)); -} - -#[test] -fn negative_cumulative_totals_do_not_inflate_later_deltas() { - let mut state = CodexParserState::new(None, None); - // A malformed cumulative record with negative counts must be clamped so it - // cannot lower the high watermark below zero. - assert_eq!( - state.total_usage_delta(&serde_json::json!({ - "input_tokens": -5, - "output_tokens": -2, - })), - (0, 0, 0, None) - ); - // A later normal climb only counts its true growth above the clamped zero. - assert_eq!( - state.total_usage_delta(&serde_json::json!({ - "input_tokens": 3, - "output_tokens": 1, - })), - (3, 0, 1, None) - ); -} - -#[test] -fn legacy_packed_rows_remain_three_slots_and_report_reasoning_is_unknown() { - let record = CodexUsageRecord { - day_key: "2026-05-31".to_string(), - timestamp: None, - model: "gpt-5.6-sol".to_string(), - input: 5, - cached: 1, - output: 3, - reasoning: Some(2), - turn_id: None, - }; - let mut packed = vec![10, 2, 4]; - JsonlScanner::merge_codex_record_into_packed(&mut packed, &record); - assert_eq!(packed, vec![15, 3, 7]); - - let mut cache = CostUsageCache::default(); - cache.days.insert( - "2026-05-31".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), packed)]), - ); - let report = JsonlScanner::cached_cost_report_from_days(&cache); - assert_eq!(report.reasoning_tokens, None); -} - -#[test] -fn known_packed_rows_report_reasoning_only_when_all_token_rows_are_known() { - let mut cache = CostUsageCache::default(); - cache.days.insert( - "2026-05-31".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![10, 2, 4, 3])]), - ); - assert_eq!( - JsonlScanner::cached_cost_report_from_days(&cache).reasoning_tokens, - Some(3) - ); - - cache - .days - .get_mut("2026-05-31") - .unwrap() - .insert("gpt-5.6-fast".to_string(), vec![1, 0, 1]); - assert_eq!( - JsonlScanner::cached_cost_report_from_days(&cache).reasoning_tokens, - None - ); -} - -#[test] -fn test_is_in_range() { - assert!(CostUsageDayRange::is_in_range( - "2026-01-15", - "2026-01-10", - "2026-01-20" - )); - assert!(!CostUsageDayRange::is_in_range( - "2026-01-05", - "2026-01-10", - "2026-01-20" - )); - assert!(!CostUsageDayRange::is_in_range( - "2026-01-25", - "2026-01-10", - "2026-01-20" - )); -} - -#[test] -fn test_parse_day_key() { - let date = CostUsageDayRange::parse_day_key("2026-01-15"); - assert!(date.is_some()); - let date = date.unwrap(); - assert_eq!(date.year(), 2026); - assert_eq!(date.month(), 1); - assert_eq!(date.day(), 15); -} - -#[test] -fn codex_timestamp_day_key_uses_local_calendar_day() { - let today = Local::now().date_naive(); - let local_midnight = today.and_hms_opt(0, 30, 0).unwrap(); - let Some(local_time) = Local.from_local_datetime(&local_midnight).earliest() else { - return; - }; - let utc_timestamp = local_time.with_timezone(&chrono::Utc).to_rfc3339(); - let expected = today.format("%Y-%m-%d").to_string(); - - assert_eq!( - codex_timestamp_day_key(&utc_timestamp).as_deref(), - Some(expected.as_str()) - ); -} - -#[test] -fn native_codex_timestamp_parser_matches_chrono_for_supported_spellings() { - for timestamp in [ - "2026-05-31T10:00:00Z", - "2026-05-31T10:00:00.123Z", - "2024-02-29T23:59:59.999+05:30", - "1900-02-28T00:00:00-08:00", - "1899-12-31T23:59:59.000Z", - ] { - assert_eq!( - parse_rfc3339_timestamp(timestamp), - DateTime::parse_from_rfc3339(timestamp).ok(), - "native parser changed {timestamp}" - ); - } - for timestamp in [ - "2026-02-29T10:00:00Z", - "2026-05-31T10:00:00.1234567890Z", - "2026-05-31T10:00:00+0530", - "2026-05-31T24:00:00Z", - ] { - assert_eq!( - parse_rfc3339_timestamp(timestamp), - DateTime::parse_from_rfc3339(timestamp).ok(), - "native parser changed invalid {timestamp}" - ); - } -} - -#[test] -fn codex_timestamp_fallback_rejects_invalid_calendar_prefixes() { - for timestamp in [ - "2026-02-29T10:00:00Z", - "2026-04-31T10:00:00Z", - "not-a-dateT10:00:00Z", - "2026-05-31", - ] { - assert!( - parse_codex_timestamp(timestamp).is_none(), - "invalid timestamp must not be accepted by the day-key fallback: {timestamp}" - ); - } - - for timestamp in ["2026-05-31T10:00:00+0530", "2026-05-31T10:00:00+05"] { - let parsed = parse_codex_timestamp(timestamp).expect("historical timestamp shape"); - assert_eq!(parsed.fallback_day_key, "2026-05-31"); - assert!(parsed.parsed.is_none()); - } -} - -#[test] -fn codex_timestamp_order_latches_false_and_stops_rechecking() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); - - for (timestamp, input) in [ - ("2026-05-31T10:00:02Z", 10), - ("2026-05-31T10:00:01Z", 20), - ("2026-05-31T10:00:03Z", 30), - ] { - parser.process_line( - &format!( - r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":{input},"cached_input_tokens":0,"output_tokens":1}}}}}}}}"# - ), - &range, - ); - } - - assert_eq!(parser.token_timestamps_monotonic, Some(false)); - assert_eq!(parser.token_timestamp_comparisons, 1); - assert_eq!(parser.records.len(), 3); -} - -#[test] -fn codex_timestamp_order_ignores_sub_millisecond_fraction() { - let day = NaiveDate::from_ymd_opt(2026, 8, 30).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); - - for timestamp in ["2026-08-30T12:00:00.1239Z", "2026-08-30T12:00:00.1231Z"] { - parser.process_line( - &format!( - r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}}}}}}}"# - ), - &range, - ); - } - - assert_eq!(parser.token_timestamps_monotonic, Some(true)); - assert_eq!(parser.token_timestamp_comparisons, 1); -} - -#[test] -fn codex_timestamp_order_detects_millisecond_decrease() { - let day = NaiveDate::from_ymd_opt(2026, 8, 30).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); - - for timestamp in ["2026-08-30T12:00:00.124Z", "2026-08-30T12:00:00.123Z"] { - parser.process_line( - &format!( - r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}}}}}}}"# - ), - &range, - ); - } - - assert_eq!(parser.token_timestamps_monotonic, Some(false)); - assert_eq!(parser.token_timestamp_comparisons, 1); -} - -#[test] -fn codex_timestamp_order_checks_token_history_outside_requested_window() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); - - for line in [ - r#"{"timestamp":"2026-06-01T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":10,"cached_input_tokens":0,"output_tokens":1}}}}"#, - r#"{"timestamp":"2026-05-31T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":20,"cached_input_tokens":0,"output_tokens":2}}}}"#, - ] { - parser.process_line(line, &range); - } - - assert_eq!(parser.token_timestamps_monotonic, Some(false)); - assert_eq!(parser.token_timestamp_comparisons, 1); - assert_eq!( - parser.records.len(), - 1, - "only the in-range event is recorded" - ); -} - -#[test] -fn test_fast_codex_parser_reads_last_usage_from_payload() { - let range = CostUsageDayRange::new( - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - ); - let mut parser = CodexParserState::new(None, None); - - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:00.000Z","type":"turn_context","payload":{"info":{"model":"gpt-5.5"}}}"#, - &range, - ); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:02.000Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":120,"cache_read_input_tokens":40,"output_tokens":9}}}}"#, - &range, - ); - - assert_eq!(parser.records.len(), 1); - let (record, _) = &parser.records[0]; - assert_eq!(record.day_key, "2026-05-31"); - assert_eq!(record.model, "gpt-5.5"); - assert_eq!((record.input, record.cached, record.output), (120, 40, 9)); - assert_eq!(parser.current_model.as_deref(), Some("gpt-5.5")); -} - -#[test] -fn test_fast_codex_parser_diffs_total_usage() { - let range = CostUsageDayRange::new( - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - ); - let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); - - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:01.000Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":1000,"cached_input_tokens":200,"output_tokens":50}}}}"#, - &range, - ); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:02.000Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":1250,"cached_input_tokens":260,"output_tokens":90}}}}"#, - &range, - ); - - assert_eq!(parser.records.len(), 2); - assert_eq!( - parser - .records - .iter() - .map(|(record, _)| (record.input, record.cached, record.output)) - .collect::>(), - vec![(1_000, 200, 50), (250, 60, 40)] - ); - let totals = parser.previous_totals.expect("last totals"); - assert_eq!(totals.input, 1250); - assert_eq!(totals.cached, 260); - assert_eq!(totals.output, 90); -} - -#[test] -fn test_fast_codex_parser_reads_legacy_event_msg_shape() { - let range = CostUsageDayRange::new( - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - ); - let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); - - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:02.000Z","type":"event_msg","event_msg":{"type":"token_count","input_tokens":20,"cached_input_tokens":5,"output_tokens":3}}"#, - &range, - ); - - assert_eq!(parser.records.len(), 1); - let (record, _) = &parser.records[0]; - assert_eq!(record.model, "gpt-5"); - assert_eq!((record.input, record.cached, record.output), (20, 5, 3)); -} - -#[test] -fn codex_fast_parser_accepts_compact_and_spaced_event_records_equally() { - let range = CostUsageDayRange::new( - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - ); - let parsed = |line: &str| { - let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); - parser.process_line(line, &range); - ( - parser.current_model, - parser - .records - .into_iter() - .map(|(record, _)| { - ( - record.day_key, - record.model, - record.input, - record.cached, - record.output, - record.reasoning, - ) - }) - .collect::>(), - ) - }; - - let compact_event = r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":20,"cached_input_tokens":5,"output_tokens":3}}}}"#; - let spaced_event = r#"{ "timestamp": "2026-05-31T10:00:01Z", "type": "event_msg", "payload": { "type": "token_count", "info": { "last_token_usage": { "input_tokens": 20, "cached_input_tokens": 5, "output_tokens": 3 } } } }"#; - assert!(is_candidate_codex_line(spaced_event)); - assert_eq!(parsed(compact_event), parsed(spaced_event)); - - let compact_context = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{"model":"gpt-5.5"}}"#; - let spaced_context = "{\t\"timestamp\": \"2026-05-31T10:00:00Z\",\t\"type\":\t\"turn_context\",\t\"payload\": {\t\"model\": \"gpt-5.5\"\t}\t}"; - assert!(is_candidate_codex_line(spaced_context)); - assert_eq!(parsed(compact_context), parsed(spaced_context)); - - let unrelated = r#"{ "timestamp": "2026-05-31T10:00:00Z", "type": "response", "payload": { "model": "gpt-5.5" } }"#; - assert!(!is_candidate_codex_line(unrelated)); - assert_eq!(parsed(unrelated), (Some("gpt-5".to_string()), Vec::new())); -} - -#[test] -fn test_parse_codex_file_uses_fast_parser_for_current_logs() { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - writeln!( - file, - r#"{{"timestamp":"2026-05-31T10:00:00.000Z","type":"turn_context","payload":{{"model":"gpt-5.5"}}}}"# - ) - .unwrap(); - writeln!( - file, - r#"{{"timestamp":"2026-05-31T10:00:01.000Z","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":45,"cached_input_tokens":12,"output_tokens":8}}}}}}}}"# - ) - .unwrap(); - - let range = CostUsageDayRange::new( - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), - ); - let parsed = JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None).expect("parse"); - - assert_eq!(parsed.last_model.as_deref(), Some("gpt-5.5")); - assert_eq!(parsed.records.len(), 1); - let (record, _) = &parsed.records[0]; - assert_eq!(record.day_key, "2026-05-31"); - assert_eq!(record.model, "gpt-5.5"); - assert_eq!((record.input, record.cached, record.output), (45, 12, 8)); -} - -#[test] -fn codex_append_timestamp_state_is_output_equivalent_and_boundary_only() { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - for (timestamp, input, output) in [ - ("2026-05-31T10:00:01.000Z", 10, 1), - ("2026-05-31T10:00:02.000Z", 20, 2), - ] { - writeln!( - file, - r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5.5","total_token_usage":{{"input_tokens":{input},"cached_input_tokens":0,"output_tokens":{output}}}}}}}}}"# - ) - .unwrap(); - } - - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let prefix = - JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None).expect("parse prefix"); - assert_eq!(prefix.token_timestamps_monotonic, Some(true)); - assert_eq!(prefix.token_timestamp_comparisons, 1); - let prefix_input: i64 = prefix.records.iter().map(|(record, _)| record.input).sum(); - - writeln!( - file, - r#"{{"timestamp":"2026-05-31T10:00:03.000Z","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5.5","total_token_usage":{{"input_tokens":30,"cached_input_tokens":0,"output_tokens":3}}}}}}}}"# - ) - .unwrap(); - - let appended = JsonlScanner::parse_codex( - file.path(), - &range, - CodexParseMode::Standard { - start_offset: prefix.parsed_bytes, - initial_model: prefix.last_model.clone(), - initial_totals: prefix.last_totals.clone(), - previous_token_timestamp: prefix.last_token_timestamp.clone(), - token_timestamps_monotonic: prefix.token_timestamps_monotonic, - }, - None, - None, - None, - ) - .expect("parse appended suffix"); - assert_eq!(appended.token_timestamps_monotonic, Some(true)); - assert_eq!( - appended.token_timestamp_comparisons, 1, - "only the cached-prefix boundary is compared" - ); - - let full = JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None) - .expect("parse complete file"); - let full_input: i64 = full.records.iter().map(|(record, _)| record.input).sum(); - let appended_input: i64 = appended - .records - .iter() - .map(|(record, _)| record.input) - .sum(); - assert_eq!(prefix_input + appended_input, full_input); - assert_eq!(full_input, 30); -} - -#[test] -fn codex_parse_publishes_only_the_committed_prefix_before_an_incomplete_tail() { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let committed_line = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.5","total_token_usage":{"input_tokens":10,"cached_input_tokens":0,"output_tokens":1}}}}"#; - writeln!(file, "{committed_line}").unwrap(); - let committed_bytes = - i64::try_from(committed_line.len() + 1).expect("fixture line length fits i64"); - - let complete_tail = r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.5","total_token_usage":{"input_tokens":20,"cached_input_tokens":0,"output_tokens":2}}}}"#; - let split = complete_tail.len() / 2; - write!(file, "{}", &complete_tail[..split]).unwrap(); - file.flush().unwrap(); - - let partial = JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None) - .expect("parse committed prefix"); - assert_eq!(partial.records.len(), 1); - assert_eq!(partial.records[0].0.input, 10); - assert_eq!(partial.parsed_bytes, committed_bytes); - assert_eq!(partial.scan_target_size, committed_bytes); - assert!(partial.is_complete, "the logical prefix is complete"); - - writeln!(file, "{}", &complete_tail[split..]).unwrap(); - let resumed = JsonlScanner::parse_codex( - file.path(), - &range, - CodexParseMode::Standard { - start_offset: partial.parsed_bytes, - initial_model: partial.last_model, - initial_totals: partial.last_totals, - previous_token_timestamp: partial.last_token_timestamp, - token_timestamps_monotonic: partial.token_timestamps_monotonic, - }, - None, - None, - None, - ) - .expect("resume completed tail"); - assert_eq!(resumed.records.len(), 1); - assert_eq!(resumed.records[0].0.input, 10); - assert_eq!( - resumed.parsed_bytes, - i64::try_from(std::fs::metadata(file.path()).unwrap().len()) - .expect("fixture file length fits i64") - ); - assert_eq!(resumed.scan_target_size, resumed.parsed_bytes); - assert!(resumed.is_complete); -} - -#[test] -fn codex_parser_discards_oversized_line_and_recovers_next_record() { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - let padding = "x".repeat(CODEX_JSONL_MAX_LINE_BYTES); - writeln!( - file, - r#"{{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{{"model":"{padding}"}}}}"# - ) - .unwrap(); - writeln!( - file, - r#"{{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":9,"cached_input_tokens":2,"output_tokens":1}}}}}}}}"# - ) - .unwrap(); - - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let parsed = JsonlScanner::parse_codex_file( - file.path(), - &CostUsageDayRange::new(day, day), - 0, - None, - None, - ) - .expect("parse"); - - assert_eq!(parsed.records.len(), 1); - assert_eq!( - parsed.records[0].0.model, - CostUsagePricing::CODEX_UNATTRIBUTED_MODEL - ); - assert_eq!( - ( - parsed.records[0].0.input, - parsed.records[0].0.cached, - parsed.records[0].0.output - ), - (9, 2, 1) - ); -} - -#[test] -fn codex_parser_validates_a_record_at_the_line_limit() { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - let prefix = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":9,"cached_input_tokens":2,"output_tokens":1}}},"padding":""}"#; - let padding_len = CODEX_JSONL_MAX_LINE_BYTES - prefix.len(); - let line = format!( - "{}{}\"}}", - &prefix[..prefix.len() - 2], - "x".repeat(padding_len) - ); - assert_eq!(line.len(), CODEX_JSONL_MAX_LINE_BYTES); - writeln!(file, "{line}").unwrap(); - - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let parsed = JsonlScanner::parse_codex_file( - file.path(), - &CostUsageDayRange::new(day, day), - 0, - None, - None, - ) - .expect("parse"); - - assert_eq!(parsed.records.len(), 1); - assert_eq!( - ( - parsed.records[0].0.input, - parsed.records[0].0.cached, - parsed.records[0].0.output - ), - (9, 2, 1) - ); -} - -#[test] -fn codex_parser_discards_a_line_at_limit_plus_one_and_keeps_following_record() { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - let prefix = r#"{"padding":""}"#; - let padding_len = CODEX_JSONL_MAX_LINE_BYTES + 1 - prefix.len(); - let oversized = format!( - "{}{}\"}}", - &prefix[..prefix.len() - 2], - "x".repeat(padding_len) - ); - assert_eq!(oversized.len(), CODEX_JSONL_MAX_LINE_BYTES + 1); - writeln!(file, "{oversized}").unwrap(); - let valid = r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":9,"cached_input_tokens":2,"output_tokens":1}}}}"#; - writeln!(file, "{valid}").unwrap(); - - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let parsed = JsonlScanner::parse_codex_file( - file.path(), - &CostUsageDayRange::new(day, day), - 0, - None, - None, - ) - .expect("parse"); - - assert_eq!(parsed.records.len(), 1); - assert_eq!(parsed.records[0].0.input, 9); -} - -#[test] -fn codex_parser_discards_huge_malformed_lines_before_and_after_valid_records() { - let mut file = tempfile::NamedTempFile::new().expect("temp file"); - let valid = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":9,"cached_input_tokens":2,"output_tokens":1}}}}"#; - let malformed = format!("{{{}", "x".repeat(CODEX_JSONL_MAX_LINE_BYTES * 4)); - writeln!(file, "{malformed}").unwrap(); - writeln!(file, "{valid}").unwrap(); - writeln!(file, "{malformed}").unwrap(); - - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let parsed = JsonlScanner::parse_codex_file( - file.path(), - &CostUsageDayRange::new(day, day), - 0, - None, - None, - ) - .expect("parse"); - - assert_eq!(parsed.records.len(), 1); - assert_eq!(parsed.records[0].0.input, 9); -} - -#[test] -fn bounded_jsonl_reader_accepts_exact_limit_without_retaining_larger_input() { - let mut input = vec![b'x'; CODEX_JSONL_MAX_LINE_BYTES]; - input.push(b'\n'); - input.extend_from_slice(br#"{"type":"event_msg"}"#); - input.push(b'\n'); - let mut reader = BufReader::with_capacity(64 * 1024, std::io::Cursor::new(input)); - - let exact = match read_bounded_jsonl_line(&mut reader, CODEX_JSONL_MAX_LINE_BYTES) - .expect("read") - .expect("line") - { - BoundedJsonlLine::Retained { bytes, .. } => bytes, - BoundedJsonlLine::Discarded { .. } => panic!("exact-limit line was discarded"), - }; - let later = match read_bounded_jsonl_line(&mut reader, CODEX_JSONL_MAX_LINE_BYTES) - .expect("read") - .expect("line") - { - BoundedJsonlLine::Retained { bytes, .. } => bytes, - BoundedJsonlLine::Discarded { .. } => panic!("following line was discarded"), - }; - - assert_eq!(exact.len(), CODEX_JSONL_MAX_LINE_BYTES); - assert_eq!(later, br#"{"type":"event_msg"}"#); -} - -#[test] -fn codex_turn_context_wins_over_conflicting_event_model() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(None, None); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{"model":"gpt-5.5"}}"#, - &range, - ); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","model":"gpt-5.6-sol","info":{"last_token_usage":{"input_tokens":5,"cached_input_tokens":1,"output_tokens":2}}}}"#, - &range, - ); - - assert_eq!(parser.records[0].0.model, "gpt-5.5"); -} - -#[test] -fn codex_blank_context_clears_stale_model_and_emits_unattributed_usage() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(Some("gpt-5.5".to_string()), None); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{"model":" "}}"#, - &range, - ); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":5,"cached_input_tokens":1,"output_tokens":2}}}}"#, - &range, - ); - - assert_eq!( - parser.records[0].0.model, - CostUsagePricing::CODEX_UNATTRIBUTED_MODEL - ); -} - -#[test] -fn codex_model_less_token_event_uses_unpriced_sentinel() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(None, None); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":10,"cached_input_tokens":0,"output_tokens":2}}}}"#, - &range, - ); - - assert_eq!(parser.records.len(), 1); - assert_eq!( - parser.records[0].0.model, - CostUsagePricing::CODEX_UNATTRIBUTED_MODEL - ); -} - -#[test] -fn cached_tokens_use_larger_cached_or_cache_read_field() { - let value = serde_json::json!({ - "input_tokens": 100, - "cached_input_tokens": 20, - "cache_read_input_tokens": 35, - "output_tokens": 10 - }); - let totals = read_token_totals(&value); - assert_eq!(totals.cached, 35); -} - -#[test] -fn parses_bare_usage_rows_outside_token_count_envelope() { - let value = serde_json::json!({ - "model": "gpt-5.6-sol", - "usage": { - "prompt_tokens": 120, - "completion_tokens": 30, - "cached_input_tokens": 40, - "cache_read_input_tokens": 55 - } - }); - let (totals, model) = bare_usage_totals(&value).expect("bare usage"); - assert_eq!(totals.input, 120); - assert_eq!(totals.output, 30); - assert_eq!(totals.cached, 55); - assert_eq!(model.as_deref(), Some("gpt-5.6-sol")); -} - -#[test] -fn process_line_accepts_type_less_bare_usage_row() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(None, None); - - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:01Z","model":"gpt-5.6-sol","usage":{"prompt_tokens":120,"completion_tokens":30,"cache_read_input_tokens":55}}"#, - &range, - ); - - assert_eq!(parser.records.len(), 1); - assert_eq!(parser.records[0].0.model, "gpt-5.6-sol"); - assert_eq!( - ( - parser.records[0].0.input, - parser.records[0].0.cached, - parser.records[0].0.output - ), - (120, 55, 30) - ); -} - -#[test] -fn process_line_keeps_bare_usage_when_model_contains_turn_context() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(None, None); - - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:01Z","model":"turn_context","usage":{"prompt_tokens":120,"completion_tokens":30}}"#, - &range, - ); - - assert_eq!(parser.records.len(), 1); - assert_eq!(parser.records[0].0.input, 120); - assert_eq!(parser.records[0].0.output, 30); -} - -#[test] -fn timestamp_less_bare_usage_uses_last_accepted_usage_day() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(Some("gpt-5.6-sol".to_string()), None); - - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":1}}}}"#, - &range, - ); - parser.process_line( - r#"{"usage":{"prompt_tokens":20,"completion_tokens":4,"cache_read_input_tokens":3}}"#, - &range, - ); - - assert_eq!(parser.records.len(), 2); - assert_eq!(parser.records[1].0.day_key, "2026-05-31"); - assert_eq!( - ( - parser.records[1].0.input, - parser.records[1].0.cached, - parser.records[1].0.output - ), - (20, 3, 4) - ); -} - -#[test] -fn interleaved_lineage_totals_never_exceed_high_watermark_growth() { - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(Some("gpt-5.6-sol".to_string()), None); - - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":100,"cached_input_tokens":0,"output_tokens":20}}}}"#, - &range, - ); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:02Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":5,"cached_input_tokens":0,"output_tokens":1}}}}"#, - &range, - ); - parser.process_line( - r#"{"timestamp":"2026-05-31T10:00:03Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":101,"cached_input_tokens":0,"output_tokens":21}}}}"#, - &range, - ); - - let total_input: i64 = parser.records.iter().map(|(r, _)| r.input).sum(); - let total_output: i64 = parser.records.iter().map(|(r, _)| r.output).sum(); - assert!( - total_input <= 101, - "input inflated to {total_input}, expected <= 101" - ); - assert!( - total_output <= 21, - "output inflated to {total_output}, expected <= 21" - ); -} - -#[test] -fn interleaved_lineage_mid_range_climb_below_watermark_does_not_readd() { - // 100 → 5 (rewind) → 80 (mid-range below water) → 101 (above water). - // Phase-1 containment: do not re-add the 5→80 climb; only growth above - // the historical high watermark counts. - let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); - let range = CostUsageDayRange::new(day, day); - let mut parser = CodexParserState::new(Some("gpt-5.6-sol".to_string()), None); - - for (input, output) in [(100, 20), (5, 1), (80, 10), (101, 21)] { - parser.process_line( - &format!( - r#"{{"timestamp":"2026-05-31T10:00:0{input}Z","type":"event_msg","payload":{{"type":"token_count","info":{{"total_token_usage":{{"input_tokens":{input},"cached_input_tokens":0,"output_tokens":{output}}}}}}}"# - ), - &range, - ); - } - - let total_input: i64 = parser.records.iter().map(|(r, _)| r.input).sum(); - let total_output: i64 = parser.records.iter().map(|(r, _)| r.output).sum(); - assert!( - total_input <= 101, - "mid-range climb re-added input to {total_input}, expected <= 101" - ); - assert!( - total_output <= 21, - "mid-range climb re-added output to {total_output}, expected <= 21" - ); -} - -#[test] -fn cost_scan_options_app_driven_bypasses_debounce() { - let debounced = CostScanOptions::default(); - let forced = CostScanOptions::app_driven(); - assert!(!debounced.is_app_driven()); - assert!(forced.is_app_driven()); - let last = 1_000_000_i64; - let now = last + 1_000; // 1s later, within 60s window - - assert!(debounced.should_skip_scan(last, now)); - assert!(!forced.should_skip_scan(last, now)); - assert!(!debounced.should_skip_scan(last, last + 61_000)); - - let cache = CostUsageCache { - last_scan_unix_ms: last, - ..Default::default() - }; - assert!(JsonlScanner::should_skip_cached_scan( - &cache, - CostScanOptions::default(), - now - )); - assert!(!JsonlScanner::should_skip_cached_scan( - &cache, - CostScanOptions::app_driven(), - now - )); -} - -#[test] -fn session_meta_pre_read_accepts_snake_and_camel_fork_identity() { - let root = tempfile::tempdir().unwrap(); - let snake = root.path().join("snake.jsonl"); - std::fs::write( - &snake, - concat!( - r#"{"type":"session_meta","timestamp":"2026-05-31T10:00:00Z","payload":{"session_id":"child-snake","forked_from_id":"parent-snake","history_base":{"thread_id":"history-snake"}}}"#, - "\n" - ), - ) - .unwrap(); - assert_eq!( - JsonlScanner::read_codex_session_metadata(&snake).unwrap(), - CodexSessionMetadata { - session_id: Some("child-snake".to_string()), - forked_from_id: Some("parent-snake".to_string()), - lineage: CodexSessionLineage::Child, - fork_timestamp: Some("2026-05-31T10:00:00Z".to_string()), - history_base_thread_id: Some("history-snake".to_string()), - is_subagent: false, - subagent_history_start_ordinal: None, - } - ); - - let camel = root.path().join("camel.jsonl"); - std::fs::write( - &camel, - concat!( - r#"{"type":"session_meta","payload":{"sessionId":"child-camel","forkedFromId":"parent-camel","timestamp":"2026-05-31T10:00:01Z"}}"#, - "\n" - ), - ) - .unwrap(); - let metadata = JsonlScanner::read_codex_session_metadata(&camel).unwrap(); - assert_eq!(metadata.session_id.as_deref(), Some("child-camel")); - assert_eq!(metadata.forked_from_id.as_deref(), Some("parent-camel")); - assert_eq!( - metadata.fork_timestamp.as_deref(), - Some("2026-05-31T10:00:01Z") - ); -} - -#[test] -fn legacy_file_usage_json_defaults_fork_metadata() { - let usage: CostUsageFileUsage = serde_json::from_str( - r#"{"mtime_unix_ms":0,"size":0,"days":{},"parsed_bytes":null,"last_model":null,"last_totals":null}"#, - ) - .unwrap(); - assert_eq!(usage.codex_session_id, None); - assert_eq!(usage.codex_forked_from_id, None); - assert_eq!(usage.codex_fork_timestamp, None); - assert!(!usage.codex_unresolved_fork_parent); - - let report: CachedCostReport = serde_json::from_str( - r#"{"total_cost_usd":1.5,"input_tokens":10,"cached_tokens":2,"output_tokens":3,"sessions_count":1,"updated_at":null,"partial":false}"#, - ) - .unwrap(); - assert_eq!(report.reasoning_tokens, None); -} - -#[test] -fn is_line_boundary_offset_zero_returns_true() { - // F2: offset 0 is always a valid boundary (start of file). - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("f.jsonl"); - std::fs::write( - &path, - b"hello -world -", - ) - .unwrap(); - assert!(JsonlScanner::is_line_boundary_offset(&path, 0)); -} - -#[test] -fn is_line_boundary_offset_at_or_past_size_returns_true() { - // F2: offset >= file_size returns true (EOF or beyond is a valid boundary). - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("f.jsonl"); - let content = b"line1 -line2 -"; - std::fs::write(&path, content).unwrap(); - let size = i64::try_from(content.len()).unwrap(); - assert!(JsonlScanner::is_line_boundary_offset(&path, size)); - assert!(JsonlScanner::is_line_boundary_offset(&path, size + 100)); -} - -#[test] -fn is_line_boundary_offset_exact_newline_returns_true() { - // F2: offset pointing right after a newline is a valid boundary. - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("f.jsonl"); - // "line1\nline2\n" — offset 6 is right after first \n - std::fs::write(&path, b"line1\nline2\n").unwrap(); - assert!(JsonlScanner::is_line_boundary_offset(&path, 6)); -} - -#[test] -fn is_line_boundary_offset_midline_returns_false() { - // F2: offset pointing mid-line (byte before is not \n) returns false. - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("f.jsonl"); - // "line1\nline2\n" — offset 3 is mid-line (byte before is 'n') - std::fs::write(&path, b"line1\nline2\n").unwrap(); - assert!(!JsonlScanner::is_line_boundary_offset(&path, 3)); -} - -#[test] -fn is_line_boundary_offset_missing_file_returns_false() { - // F2: missing file returns false (probe fails). - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("nonexistent.jsonl"); - // offset > 0 so it doesn't short-circuit to true - assert!(!JsonlScanner::is_line_boundary_offset(&path, 10)); -} - -#[test] -fn catch_up_snapshot_preserves_established_codex_cost_and_tokens() { - let mut cache = CostUsageCache::default(); - cache.files.insert( - "session.jsonl".to_string(), - CostUsageFileUsage { - parsed_bytes: Some(100), - last_model: Some("gpt-5.6-sol".to_string()), - codex_token_timestamps_monotonic: Some(true), - ..test_file_usage( - 100, - HashMap::from([( - "2026-08-20".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![1_000, 250, 100])]), - )]), - ) - }, - ); - cache.files.insert( - "empty.jsonl".to_string(), - CostUsageFileUsage { - parsed_bytes: Some(10), - ..test_file_usage(10, HashMap::new()) - }, - ); - cache.days.insert( - "2026-08-20".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![1_000, 250, 100])]), - ); - - let report = JsonlScanner::cached_cost_report_from_days(&cache); - let expected = CostUsagePricing::codex_cost_usd_at_date( - "gpt-5.6-sol", - 1_000, - 250, - 100, - NaiveDate::from_ymd_opt(2026, 8, 20).unwrap(), - ) - .expect("known model price"); - - assert!((report.total_cost_usd - expected).abs() < 1e-12); - assert!(report.total_cost_usd > 0.0); - assert_eq!(report.input_tokens, 1_000); - assert_eq!(report.cached_tokens, 250); - assert_eq!(report.output_tokens, 100); - assert_eq!(report.sessions_count, 1); - assert!(!report.partial); - assert!(report.updated_at.is_some()); -} - -#[test] -fn ranged_catch_up_snapshot_excludes_historical_days_and_keeps_measurement_time() { - let usage = |day: &str| CostUsageFileUsage { - parsed_bytes: Some(100), - last_model: Some("gpt-5.6-sol".to_string()), - codex_token_timestamps_monotonic: Some(true), - ..test_file_usage( - 100, - HashMap::from([( - day.to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![100, 25, 10, 4])]), - )]), - ) - }; - let mut cache = CostUsageCache { - last_scan_unix_ms: 1, - ..CostUsageCache::default() - }; - cache - .files - .insert("current.jsonl".to_string(), usage("2026-09-19")); - cache - .files - .insert("historical.jsonl".to_string(), usage("2026-09-01")); - cache.days.insert( - "2026-09-19".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![100, 25, 10, 4])]), - ); - cache.days.insert( - "2026-09-01".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![900, 225, 90, 36])]), - ); - - let range = CostUsageDayRange { - since_key: "2026-09-19".to_string(), - until_key: "2026-09-19".to_string(), - scan_since_key: "2026-09-18".to_string(), - scan_until_key: "2026-09-20".to_string(), - }; - let report = JsonlScanner::cached_cost_report_for_range(&cache, &range); - - assert_eq!(report.input_tokens, 100); - assert_eq!(report.cached_tokens, 25); - assert_eq!(report.output_tokens, 10); - assert_eq!(report.reasoning_tokens, Some(4)); - assert_eq!(report.sessions_count, 1); - assert_eq!( - report.updated_at, - DateTime::::from_timestamp_millis(1).map(|timestamp| timestamp.to_rfc3339()) - ); -} - -#[test] -fn codex_cache_round_trip_preserves_64_bit_counts_and_rebuilds_legacy_schema() { - let root = tempfile::tempdir().unwrap(); - let cache_root = root.path(); - let mut cache = CostUsageCache::default(); - cache.days.insert( - "2026-09-09".to_string(), - HashMap::from([( - "gpt-5.6-luna".to_string(), - vec![3_000_000_000, 2_800_000_000, 200], - )]), - ); - - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(cache_root)); - let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); - assert_eq!( - loaded.days["2026-09-09"]["gpt-5.6-luna"], - vec![3_000_000_000, 2_800_000_000, 200] - ); - - let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(cache_root)); - let mut legacy: serde_json::Value = - serde_json::from_slice(&std::fs::read(&cache_path).unwrap()).unwrap(); - legacy - .as_object_mut() - .unwrap() - .remove("codex_cache_schema_version"); - std::fs::write(&cache_path, serde_json::to_vec(&legacy).unwrap()).unwrap(); - - let invalidated = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); - assert!(invalidated.days.is_empty()); - assert!(invalidated.files.is_empty()); - let status = JsonlScanner::load_cache_status(ProviderId::Codex, Some(cache_root)); - assert!(!status.has_days); - assert!(status.previous_report.is_none()); -} - -#[test] -fn codex_v1_cache_rebuild_clears_stalled_subagent_refresh_state() { - let root = tempfile::tempdir().unwrap(); - let mut cache = CostUsageCache { - codex_scan_incomplete: true, - codex_pending_paths: vec!["stalled-subagent.jsonl".to_string()], - codex_scan_pause_reason: Some(CodexScanPauseReason::NoProgress), - previous_report: Some(CachedCostReport { - total_cost_usd: 1.0, - input_tokens: 11, - cached_tokens: 2, - output_tokens: 3, - reasoning_tokens: None, - sessions_count: 1, - updated_at: Some("2026-09-16T10:00:00Z".to_string()), - partial: false, - }), - last_scan_unix_ms: i64::MAX, - ..CostUsageCache::default() - }; - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(root.path())); - let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(root.path())); - let mut old: serde_json::Value = - serde_json::from_slice(&std::fs::read(&cache_path).unwrap()).unwrap(); - old["codex_cache_schema_version"] = serde_json::json!(1); - std::fs::write(&cache_path, serde_json::to_vec(&old).unwrap()).unwrap(); - - let rebuilt = JsonlScanner::load_cache(ProviderId::Codex, Some(root.path())); - assert!(!rebuilt.codex_scan_incomplete); - assert!(rebuilt.codex_pending_paths.is_empty()); - assert!(rebuilt.codex_scan_pause_reason.is_none()); - assert!(rebuilt.previous_report.is_none()); - assert_eq!(rebuilt.last_scan_unix_ms, 0); - let status = JsonlScanner::load_cache_status(ProviderId::Codex, Some(root.path())); - assert!(status.previous_report.is_none()); -} - -#[test] -fn codex_cache_schema_policy_helpers_rebuild_mismatched_load() { - let stamp = CacheStamp::from_bytes(b"baseline"); - - let legacy = CostUsageCache { - codex_cache_schema_version: 0, - days: HashMap::from([( - "2026-09-09".to_string(), - HashMap::from([("gpt-5.6-luna".to_string(), vec![1, 2, 3])]), - )]), - ..CostUsageCache::default() - }; - let rebuilt = codex_cache_apply_load_policy(legacy, stamp.clone()); - assert_eq!( - rebuilt.codex_cache_schema_version, - CODEX_CACHE_SCHEMA_VERSION - ); - assert!(rebuilt.days.is_empty()); - assert!(rebuilt.files.is_empty()); - assert!(rebuilt.loaded_stamp.is_some()); - - let current = CostUsageCache { - codex_cache_schema_version: CODEX_CACHE_SCHEMA_VERSION, - days: HashMap::from([( - "2026-09-09".to_string(), - HashMap::from([("gpt-5.6-luna".to_string(), vec![1, 2, 3])]), - )]), - ..CostUsageCache::default() - }; - let kept = codex_cache_apply_load_policy(current, stamp); - assert_eq!(kept.codex_cache_schema_version, CODEX_CACHE_SCHEMA_VERSION); - assert_eq!(kept.days["2026-09-09"]["gpt-5.6-luna"], vec![1, 2, 3]); - assert!(kept.loaded_stamp.is_some()); - - assert!(codex_cache_schema_is_current(CODEX_CACHE_SCHEMA_VERSION)); - assert!(!codex_cache_schema_is_current(0)); - - let mut stamped = CostUsageCache::default(); - codex_cache_stamp_schema_version(&mut stamped); - assert_eq!( - stamped.codex_cache_schema_version, - CODEX_CACHE_SCHEMA_VERSION - ); -} - -#[test] -fn save_cache_persists_small_codex_artifact() { - // F19 integration: a normal-sized Codex cache is persisted and - // reloadable — the MAX_LOAD_BYTES refusal does not false-positive. - let root = tempfile::tempdir().unwrap(); - let cache_root = root.path().to_path_buf(); - let mut cache = CostUsageCache { - scan_since_key: Some("2026-01-01".to_string()), - scan_until_key: Some("2026-01-31".to_string()), - files: HashMap::from([( - "a.jsonl".to_string(), - test_file_usage( - 100, - HashMap::from([( - "2026-01-10".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![10, 0, 1])]), - )]), - ), - )]), - ..Default::default() - }; - - JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); - - // File should exist and be reloadable. - let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!( - loaded.files.contains_key("a.jsonl"), - "small artifact persisted" - ); - assert_eq!(loaded.scan_since_key, Some("2026-01-01".to_string())); -} - -#[test] -fn stale_loaded_cache_does_not_replace_newer_baseline() { - let root = tempfile::tempdir().unwrap(); - let cache_root = root.path(); - - let mut initial = CostUsageCache { - last_scan_unix_ms: 1, - ..Default::default() - }; - JsonlScanner::save_cache(ProviderId::Codex, &mut initial, Some(cache_root)); - - let mut stale = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); - let mut newer = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); - newer.last_scan_unix_ms = 2; - newer.scan_since_key = Some("2026-01-01".to_string()); - JsonlScanner::save_cache(ProviderId::Codex, &mut newer, Some(cache_root)); - - stale.last_scan_unix_ms = 3; - JsonlScanner::save_cache(ProviderId::Codex, &mut stale, Some(cache_root)); - - let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); - assert_eq!( - loaded.last_scan_unix_ms, 2, - "a stale decoded baseline must not overwrite the newer cache" - ); -} - -#[test] -fn save_cache_refuses_non_bounded_provider_oversize() { - // F19: non-bounded providers (e.g. Claude) skip the refusal check - // entirely — the MAX_LOAD_BYTES guard only applies to bounded providers. - // This test confirms the is_bounded_provider gate works: Claude cache - // is saved regardless of the MAX_LOAD_BYTES check (which is Codex-only). - let root = tempfile::tempdir().unwrap(); - let cache_root = root.path().to_path_buf(); - let mut cache = CostUsageCache::default(); - cache.files.insert( - "claude.jsonl".to_string(), - test_file_usage(100, HashMap::new()), - ); - - JsonlScanner::save_cache(ProviderId::Claude, &mut cache, Some(&cache_root)); - let loaded = JsonlScanner::load_cache(ProviderId::Claude, Some(&cache_root)); - assert!(loaded.files.contains_key("claude.jsonl")); -} - -#[test] -fn save_cache_refusal_removes_preexisting_destination_artifact() { - // F19 integration: when the post-encode check refuses the artifact, any - // pre-existing destination file is removed so a stale/oversized artifact - // cannot persist and trigger load/refuse/rebuild behavior on next scan. - let root = tempfile::tempdir().unwrap(); - let cache_root = root.path().to_path_buf(); - - let mut cache = CostUsageCache::default(); - cache.files.insert( - "big.jsonl".to_string(), - test_file_usage( - 100, - HashMap::from([( - "2026-01-10".to_string(), - HashMap::from([("gpt-5.6-sol".to_string(), vec![10, 0, 1])]), - )]), - ), - ); - - // Precreate a "stale" destination artifact so the refusal must remove - // it. We seed it via a large (over_max) save_limit so the save_cache_with_limit - // first ENCODES the small cache fine under a generous limit, writes the file, - // then a follow-up call with a tiny limit must refuse AND remove. - let cache_path = { - // Exercise the private helper indirectly via the public path: first - // persist a valid artifact under a generous limit via save_cache. - // Then call with an impossible limit (encoded JSON ~hundreds of - // bytes, limit = 1 byte) to force refusal. - JsonlScanner::save_cache_with_limit( - ProviderId::Codex, - &mut cache, - Some(&cache_root), - usize::MAX, - ); - let p = JsonlScanner::cache_path(ProviderId::Codex, Some(&cache_root)); - assert!(p.exists(), "precreate destination artifact"); - p - }; - - // Sanity: a normal load succeeds against the precreated artifact. - let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!(loaded.files.contains_key("big.jsonl")); - - // Force refusal with a 1-byte limit: encoded cache will exceed it. - JsonlScanner::save_cache_with_limit(ProviderId::Codex, &mut cache, Some(&cache_root), 1); - - // Destination must be gone — no stale artifact may persist. - assert!( - !cache_path.exists(), - "refusal must remove preexisting destination artifact" - ); - - // No temp file should remain in the cache root (only unique tmp name was used). - let mut tmp_entries = Vec::new(); - for entry in std::fs::read_dir(&cache_root).unwrap() { - let name = entry.unwrap().file_name(); - let name = name.to_string_lossy(); - if name.starts_with('.') && name.ends_with(".tmp") { - tmp_entries.push(name.into_owned()); - } - } - // Best-effort temp cleanup writes an empty file at the unique name; the - // invariant is that NO tmp file contains a complete artifact. The set - // should at most contain a single zero-byte remnant from the cleanup - // (or be empty); we persist via copy() rather than rename so no live - // tmp holds data after the save path completes. - for t in &tmp_entries { - let meta = std::fs::metadata(cache_root.join(t)).unwrap(); - assert_eq!(meta.len(), 0, "tmp remnant must be empty: {t}"); - } - - // Loading after removal yields a fresh default cache (no rebuild loop). - let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); - assert!( - loaded.files.is_empty(), - "no rebuild loop from removed artifact" - ); -} - -#[test] -fn save_cache_at_exact_limit_is_accepted() { - // F19 boundary: an encoded artifact at exactly the injected limit is - // accepted (only strictly-larger artifacts are refused). - let root = tempfile::tempdir().unwrap(); - let cache_root = root.path().to_path_buf(); - - let mut cache = CostUsageCache::default(); - // Saving stamps the schema version and bucket zone first; serialize the - // stamped struct to learn the exact encoded size. - codex_cache_stamp_schema_version(&mut cache); - let json = serde_json::to_string(&cache).unwrap(); - let exact_limit = json.len(); - - let mut cache_for_save = cache; - JsonlScanner::save_cache_with_limit( - ProviderId::Codex, - &mut cache_for_save, - Some(&cache_root), - exact_limit, - ); - - let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(&cache_root)); - assert!( - cache_path.exists(), - "artifact at exact limit must be persisted" - ); -} - -#[test] -fn save_cache_one_over_limit_is_refused_and_removes_destination() { - // F19 boundary: an encoded artifact one byte over the injected limit is - // refused, and any pre-existing destination is removed. - let root = tempfile::tempdir().unwrap(); - let cache_root = root.path().to_path_buf(); - - let mut cache = CostUsageCache::default(); - codex_cache_stamp_schema_version(&mut cache); - let json = serde_json::to_string(&cache).unwrap(); - // One byte short of the encoded size forces refusal on the next attempt. - let under_by_one = json.len().saturating_sub(1); - - let mut cache_for_save = cache; - JsonlScanner::save_cache_with_limit( - ProviderId::Codex, - &mut cache_for_save, - Some(&cache_root), - under_by_one, - ); - - let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(&cache_root)); - assert!( - !cache_path.exists(), - "one-over-limit encoded artifact must be refused" - ); -} - #[cfg(test)] #[path = "tests/codex_metadata.rs"] mod codex_metadata; @@ -1788,3 +29,18 @@ mod save_skip; #[path = "tests/cache_zone.rs"] mod cache_zone; + +#[path = "tests/timestamps.rs"] +mod timestamps; + +#[path = "tests/token_accounting.rs"] +mod token_accounting; + +#[path = "tests/line_parsing.rs"] +mod line_parsing; + +#[path = "tests/usage_rows.rs"] +mod usage_rows; + +#[path = "tests/cache_files.rs"] +mod cache_files; diff --git a/rust/src/core/jsonl_scanner/tests/cache_files.rs b/rust/src/core/jsonl_scanner/tests/cache_files.rs new file mode 100644 index 0000000000..430c14636d --- /dev/null +++ b/rust/src/core/jsonl_scanner/tests/cache_files.rs @@ -0,0 +1,588 @@ +//! Scan options, session metadata, line boundaries and the cache file. + +use super::*; + +#[test] +fn cost_scan_options_app_driven_bypasses_debounce() { + let debounced = CostScanOptions::default(); + let forced = CostScanOptions::app_driven(); + assert!(!debounced.is_app_driven()); + assert!(forced.is_app_driven()); + let last = 1_000_000_i64; + let now = last + 1_000; // 1s later, within 60s window + + assert!(debounced.should_skip_scan(last, now)); + assert!(!forced.should_skip_scan(last, now)); + assert!(!debounced.should_skip_scan(last, last + 61_000)); + + let cache = CostUsageCache { + last_scan_unix_ms: last, + ..Default::default() + }; + assert!(JsonlScanner::should_skip_cached_scan( + &cache, + CostScanOptions::default(), + now + )); + assert!(!JsonlScanner::should_skip_cached_scan( + &cache, + CostScanOptions::app_driven(), + now + )); +} + +#[test] +fn session_meta_pre_read_accepts_snake_and_camel_fork_identity() { + let root = tempfile::tempdir().unwrap(); + let snake = root.path().join("snake.jsonl"); + std::fs::write( + &snake, + concat!( + r#"{"type":"session_meta","timestamp":"2026-05-31T10:00:00Z","payload":{"session_id":"child-snake","forked_from_id":"parent-snake","history_base":{"thread_id":"history-snake"}}}"#, + "\n" + ), + ) + .unwrap(); + assert_eq!( + JsonlScanner::read_codex_session_metadata(&snake).unwrap(), + CodexSessionMetadata { + session_id: Some("child-snake".to_string()), + forked_from_id: Some("parent-snake".to_string()), + lineage: CodexSessionLineage::Child, + fork_timestamp: Some("2026-05-31T10:00:00Z".to_string()), + history_base_thread_id: Some("history-snake".to_string()), + is_subagent: false, + subagent_history_start_ordinal: None, + } + ); + + let camel = root.path().join("camel.jsonl"); + std::fs::write( + &camel, + concat!( + r#"{"type":"session_meta","payload":{"sessionId":"child-camel","forkedFromId":"parent-camel","timestamp":"2026-05-31T10:00:01Z"}}"#, + "\n" + ), + ) + .unwrap(); + let metadata = JsonlScanner::read_codex_session_metadata(&camel).unwrap(); + assert_eq!(metadata.session_id.as_deref(), Some("child-camel")); + assert_eq!(metadata.forked_from_id.as_deref(), Some("parent-camel")); + assert_eq!( + metadata.fork_timestamp.as_deref(), + Some("2026-05-31T10:00:01Z") + ); +} + +#[test] +fn legacy_file_usage_json_defaults_fork_metadata() { + let usage: CostUsageFileUsage = serde_json::from_str( + r#"{"mtime_unix_ms":0,"size":0,"days":{},"parsed_bytes":null,"last_model":null,"last_totals":null}"#, + ) + .unwrap(); + assert_eq!(usage.codex_session_id, None); + assert_eq!(usage.codex_forked_from_id, None); + assert_eq!(usage.codex_fork_timestamp, None); + assert!(!usage.codex_unresolved_fork_parent); + + let report: CachedCostReport = serde_json::from_str( + r#"{"total_cost_usd":1.5,"input_tokens":10,"cached_tokens":2,"output_tokens":3,"sessions_count":1,"updated_at":null,"partial":false}"#, + ) + .unwrap(); + assert_eq!(report.reasoning_tokens, None); +} + +#[test] +fn is_line_boundary_offset_zero_returns_true() { + // F2: offset 0 is always a valid boundary (start of file). + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("f.jsonl"); + std::fs::write( + &path, + b"hello +world +", + ) + .unwrap(); + assert!(JsonlScanner::is_line_boundary_offset(&path, 0)); +} + +#[test] +fn is_line_boundary_offset_at_or_past_size_returns_true() { + // F2: offset >= file_size returns true (EOF or beyond is a valid boundary). + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("f.jsonl"); + let content = b"line1 +line2 +"; + std::fs::write(&path, content).unwrap(); + let size = i64::try_from(content.len()).unwrap(); + assert!(JsonlScanner::is_line_boundary_offset(&path, size)); + assert!(JsonlScanner::is_line_boundary_offset(&path, size + 100)); +} + +#[test] +fn is_line_boundary_offset_exact_newline_returns_true() { + // F2: offset pointing right after a newline is a valid boundary. + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("f.jsonl"); + // "line1\nline2\n" — offset 6 is right after first \n + std::fs::write(&path, b"line1\nline2\n").unwrap(); + assert!(JsonlScanner::is_line_boundary_offset(&path, 6)); +} + +#[test] +fn is_line_boundary_offset_midline_returns_false() { + // F2: offset pointing mid-line (byte before is not \n) returns false. + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("f.jsonl"); + // "line1\nline2\n" — offset 3 is mid-line (byte before is 'n') + std::fs::write(&path, b"line1\nline2\n").unwrap(); + assert!(!JsonlScanner::is_line_boundary_offset(&path, 3)); +} + +#[test] +fn is_line_boundary_offset_missing_file_returns_false() { + // F2: missing file returns false (probe fails). + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("nonexistent.jsonl"); + // offset > 0 so it doesn't short-circuit to true + assert!(!JsonlScanner::is_line_boundary_offset(&path, 10)); +} + +#[test] +fn catch_up_snapshot_preserves_established_codex_cost_and_tokens() { + let mut cache = CostUsageCache::default(); + cache.files.insert( + "session.jsonl".to_string(), + CostUsageFileUsage { + parsed_bytes: Some(100), + last_model: Some("gpt-5.6-sol".to_string()), + codex_token_timestamps_monotonic: Some(true), + ..test_file_usage( + 100, + HashMap::from([( + "2026-08-20".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![1_000, 250, 100])]), + )]), + ) + }, + ); + cache.files.insert( + "empty.jsonl".to_string(), + CostUsageFileUsage { + parsed_bytes: Some(10), + ..test_file_usage(10, HashMap::new()) + }, + ); + cache.days.insert( + "2026-08-20".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![1_000, 250, 100])]), + ); + + let report = JsonlScanner::cached_cost_report_from_days(&cache); + let expected = CostUsagePricing::codex_cost_usd_at_date( + "gpt-5.6-sol", + 1_000, + 250, + 100, + NaiveDate::from_ymd_opt(2026, 8, 20).unwrap(), + ) + .expect("known model price"); + + assert!((report.total_cost_usd - expected).abs() < 1e-12); + assert!(report.total_cost_usd > 0.0); + assert_eq!(report.input_tokens, 1_000); + assert_eq!(report.cached_tokens, 250); + assert_eq!(report.output_tokens, 100); + assert_eq!(report.sessions_count, 1); + assert!(!report.partial); + assert!(report.updated_at.is_some()); +} + +#[test] +fn ranged_catch_up_snapshot_excludes_historical_days_and_keeps_measurement_time() { + let usage = |day: &str| CostUsageFileUsage { + parsed_bytes: Some(100), + last_model: Some("gpt-5.6-sol".to_string()), + codex_token_timestamps_monotonic: Some(true), + ..test_file_usage( + 100, + HashMap::from([( + day.to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![100, 25, 10, 4])]), + )]), + ) + }; + let mut cache = CostUsageCache { + last_scan_unix_ms: 1, + ..CostUsageCache::default() + }; + cache + .files + .insert("current.jsonl".to_string(), usage("2026-09-19")); + cache + .files + .insert("historical.jsonl".to_string(), usage("2026-09-01")); + cache.days.insert( + "2026-09-19".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![100, 25, 10, 4])]), + ); + cache.days.insert( + "2026-09-01".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![900, 225, 90, 36])]), + ); + + let range = CostUsageDayRange { + since_key: "2026-09-19".to_string(), + until_key: "2026-09-19".to_string(), + scan_since_key: "2026-09-18".to_string(), + scan_until_key: "2026-09-20".to_string(), + }; + let report = JsonlScanner::cached_cost_report_for_range(&cache, &range); + + assert_eq!(report.input_tokens, 100); + assert_eq!(report.cached_tokens, 25); + assert_eq!(report.output_tokens, 10); + assert_eq!(report.reasoning_tokens, Some(4)); + assert_eq!(report.sessions_count, 1); + assert_eq!( + report.updated_at, + DateTime::::from_timestamp_millis(1).map(|timestamp| timestamp.to_rfc3339()) + ); +} + +#[test] +fn codex_cache_round_trip_preserves_64_bit_counts_and_rebuilds_legacy_schema() { + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path(); + let mut cache = CostUsageCache::default(); + cache.days.insert( + "2026-09-09".to_string(), + HashMap::from([( + "gpt-5.6-luna".to_string(), + vec![3_000_000_000, 2_800_000_000, 200], + )]), + ); + + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(cache_root)); + let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); + assert_eq!( + loaded.days["2026-09-09"]["gpt-5.6-luna"], + vec![3_000_000_000, 2_800_000_000, 200] + ); + + let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(cache_root)); + let mut legacy: serde_json::Value = + serde_json::from_slice(&std::fs::read(&cache_path).unwrap()).unwrap(); + legacy + .as_object_mut() + .unwrap() + .remove("codex_cache_schema_version"); + std::fs::write(&cache_path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + + let invalidated = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); + assert!(invalidated.days.is_empty()); + assert!(invalidated.files.is_empty()); + let status = JsonlScanner::load_cache_status(ProviderId::Codex, Some(cache_root)); + assert!(!status.has_days); + assert!(status.previous_report.is_none()); +} + +#[test] +fn codex_v1_cache_rebuild_clears_stalled_subagent_refresh_state() { + let root = tempfile::tempdir().unwrap(); + let mut cache = CostUsageCache { + codex_scan_incomplete: true, + codex_pending_paths: vec!["stalled-subagent.jsonl".to_string()], + codex_scan_pause_reason: Some(CodexScanPauseReason::NoProgress), + previous_report: Some(CachedCostReport { + total_cost_usd: 1.0, + input_tokens: 11, + cached_tokens: 2, + output_tokens: 3, + reasoning_tokens: None, + sessions_count: 1, + updated_at: Some("2026-09-16T10:00:00Z".to_string()), + partial: false, + }), + last_scan_unix_ms: i64::MAX, + ..CostUsageCache::default() + }; + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(root.path())); + let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(root.path())); + let mut old: serde_json::Value = + serde_json::from_slice(&std::fs::read(&cache_path).unwrap()).unwrap(); + old["codex_cache_schema_version"] = serde_json::json!(1); + std::fs::write(&cache_path, serde_json::to_vec(&old).unwrap()).unwrap(); + + let rebuilt = JsonlScanner::load_cache(ProviderId::Codex, Some(root.path())); + assert!(!rebuilt.codex_scan_incomplete); + assert!(rebuilt.codex_pending_paths.is_empty()); + assert!(rebuilt.codex_scan_pause_reason.is_none()); + assert!(rebuilt.previous_report.is_none()); + assert_eq!(rebuilt.last_scan_unix_ms, 0); + let status = JsonlScanner::load_cache_status(ProviderId::Codex, Some(root.path())); + assert!(status.previous_report.is_none()); +} + +#[test] +fn codex_cache_schema_policy_helpers_rebuild_mismatched_load() { + let stamp = CacheStamp::from_bytes(b"baseline"); + + let legacy = CostUsageCache { + codex_cache_schema_version: 0, + days: HashMap::from([( + "2026-09-09".to_string(), + HashMap::from([("gpt-5.6-luna".to_string(), vec![1, 2, 3])]), + )]), + ..CostUsageCache::default() + }; + let rebuilt = codex_cache_apply_load_policy(legacy, stamp.clone()); + assert_eq!( + rebuilt.codex_cache_schema_version, + CODEX_CACHE_SCHEMA_VERSION + ); + assert!(rebuilt.days.is_empty()); + assert!(rebuilt.files.is_empty()); + assert!(rebuilt.loaded_stamp.is_some()); + + let current = CostUsageCache { + codex_cache_schema_version: CODEX_CACHE_SCHEMA_VERSION, + days: HashMap::from([( + "2026-09-09".to_string(), + HashMap::from([("gpt-5.6-luna".to_string(), vec![1, 2, 3])]), + )]), + ..CostUsageCache::default() + }; + let kept = codex_cache_apply_load_policy(current, stamp); + assert_eq!(kept.codex_cache_schema_version, CODEX_CACHE_SCHEMA_VERSION); + assert_eq!(kept.days["2026-09-09"]["gpt-5.6-luna"], vec![1, 2, 3]); + assert!(kept.loaded_stamp.is_some()); + + assert!(codex_cache_schema_is_current(CODEX_CACHE_SCHEMA_VERSION)); + assert!(!codex_cache_schema_is_current(0)); + + let mut stamped = CostUsageCache::default(); + codex_cache_stamp_schema_version(&mut stamped); + assert_eq!( + stamped.codex_cache_schema_version, + CODEX_CACHE_SCHEMA_VERSION + ); +} + +#[test] +fn save_cache_persists_small_codex_artifact() { + // F19 integration: a normal-sized Codex cache is persisted and + // reloadable — the MAX_LOAD_BYTES refusal does not false-positive. + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path().to_path_buf(); + let mut cache = CostUsageCache { + scan_since_key: Some("2026-01-01".to_string()), + scan_until_key: Some("2026-01-31".to_string()), + files: HashMap::from([( + "a.jsonl".to_string(), + test_file_usage( + 100, + HashMap::from([( + "2026-01-10".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![10, 0, 1])]), + )]), + ), + )]), + ..Default::default() + }; + + JsonlScanner::save_cache(ProviderId::Codex, &mut cache, Some(&cache_root)); + + // File should exist and be reloadable. + let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!( + loaded.files.contains_key("a.jsonl"), + "small artifact persisted" + ); + assert_eq!(loaded.scan_since_key, Some("2026-01-01".to_string())); +} + +#[test] +fn stale_loaded_cache_does_not_replace_newer_baseline() { + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path(); + + let mut initial = CostUsageCache { + last_scan_unix_ms: 1, + ..Default::default() + }; + JsonlScanner::save_cache(ProviderId::Codex, &mut initial, Some(cache_root)); + + let mut stale = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); + let mut newer = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); + newer.last_scan_unix_ms = 2; + newer.scan_since_key = Some("2026-01-01".to_string()); + JsonlScanner::save_cache(ProviderId::Codex, &mut newer, Some(cache_root)); + + stale.last_scan_unix_ms = 3; + JsonlScanner::save_cache(ProviderId::Codex, &mut stale, Some(cache_root)); + + let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(cache_root)); + assert_eq!( + loaded.last_scan_unix_ms, 2, + "a stale decoded baseline must not overwrite the newer cache" + ); +} + +#[test] +fn save_cache_refuses_non_bounded_provider_oversize() { + // F19: non-bounded providers (e.g. Claude) skip the refusal check + // entirely — the MAX_LOAD_BYTES guard only applies to bounded providers. + // This test confirms the is_bounded_provider gate works: Claude cache + // is saved regardless of the MAX_LOAD_BYTES check (which is Codex-only). + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path().to_path_buf(); + let mut cache = CostUsageCache::default(); + cache.files.insert( + "claude.jsonl".to_string(), + test_file_usage(100, HashMap::new()), + ); + + JsonlScanner::save_cache(ProviderId::Claude, &mut cache, Some(&cache_root)); + let loaded = JsonlScanner::load_cache(ProviderId::Claude, Some(&cache_root)); + assert!(loaded.files.contains_key("claude.jsonl")); +} + +#[test] +fn save_cache_refusal_removes_preexisting_destination_artifact() { + // F19 integration: when the post-encode check refuses the artifact, any + // pre-existing destination file is removed so a stale/oversized artifact + // cannot persist and trigger load/refuse/rebuild behavior on next scan. + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path().to_path_buf(); + + let mut cache = CostUsageCache::default(); + cache.files.insert( + "big.jsonl".to_string(), + test_file_usage( + 100, + HashMap::from([( + "2026-01-10".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![10, 0, 1])]), + )]), + ), + ); + + // Precreate a "stale" destination artifact so the refusal must remove + // it. We seed it via a large (over_max) save_limit so the save_cache_with_limit + // first ENCODES the small cache fine under a generous limit, writes the file, + // then a follow-up call with a tiny limit must refuse AND remove. + let cache_path = { + // Exercise the private helper indirectly via the public path: first + // persist a valid artifact under a generous limit via save_cache. + // Then call with an impossible limit (encoded JSON ~hundreds of + // bytes, limit = 1 byte) to force refusal. + JsonlScanner::save_cache_with_limit( + ProviderId::Codex, + &mut cache, + Some(&cache_root), + usize::MAX, + ); + let p = JsonlScanner::cache_path(ProviderId::Codex, Some(&cache_root)); + assert!(p.exists(), "precreate destination artifact"); + p + }; + + // Sanity: a normal load succeeds against the precreated artifact. + let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!(loaded.files.contains_key("big.jsonl")); + + // Force refusal with a 1-byte limit: encoded cache will exceed it. + JsonlScanner::save_cache_with_limit(ProviderId::Codex, &mut cache, Some(&cache_root), 1); + + // Destination must be gone — no stale artifact may persist. + assert!( + !cache_path.exists(), + "refusal must remove preexisting destination artifact" + ); + + // No temp file should remain in the cache root (only unique tmp name was used). + let mut tmp_entries = Vec::new(); + for entry in std::fs::read_dir(&cache_root).unwrap() { + let name = entry.unwrap().file_name(); + let name = name.to_string_lossy(); + if name.starts_with('.') && name.ends_with(".tmp") { + tmp_entries.push(name.into_owned()); + } + } + // Best-effort temp cleanup writes an empty file at the unique name; the + // invariant is that NO tmp file contains a complete artifact. The set + // should at most contain a single zero-byte remnant from the cleanup + // (or be empty); we persist via copy() rather than rename so no live + // tmp holds data after the save path completes. + for t in &tmp_entries { + let meta = std::fs::metadata(cache_root.join(t)).unwrap(); + assert_eq!(meta.len(), 0, "tmp remnant must be empty: {t}"); + } + + // Loading after removal yields a fresh default cache (no rebuild loop). + let loaded = JsonlScanner::load_cache(ProviderId::Codex, Some(&cache_root)); + assert!( + loaded.files.is_empty(), + "no rebuild loop from removed artifact" + ); +} + +#[test] +fn save_cache_at_exact_limit_is_accepted() { + // F19 boundary: an encoded artifact at exactly the injected limit is + // accepted (only strictly-larger artifacts are refused). + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path().to_path_buf(); + + let mut cache = CostUsageCache::default(); + // Saving stamps the schema version and bucket zone first; serialize the + // stamped struct to learn the exact encoded size. + codex_cache_stamp_schema_version(&mut cache); + let json = serde_json::to_string(&cache).unwrap(); + let exact_limit = json.len(); + + let mut cache_for_save = cache; + JsonlScanner::save_cache_with_limit( + ProviderId::Codex, + &mut cache_for_save, + Some(&cache_root), + exact_limit, + ); + + let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(&cache_root)); + assert!( + cache_path.exists(), + "artifact at exact limit must be persisted" + ); +} + +#[test] +fn save_cache_one_over_limit_is_refused_and_removes_destination() { + // F19 boundary: an encoded artifact one byte over the injected limit is + // refused, and any pre-existing destination is removed. + let root = tempfile::tempdir().unwrap(); + let cache_root = root.path().to_path_buf(); + + let mut cache = CostUsageCache::default(); + codex_cache_stamp_schema_version(&mut cache); + let json = serde_json::to_string(&cache).unwrap(); + // One byte short of the encoded size forces refusal on the next attempt. + let under_by_one = json.len().saturating_sub(1); + + let mut cache_for_save = cache; + JsonlScanner::save_cache_with_limit( + ProviderId::Codex, + &mut cache_for_save, + Some(&cache_root), + under_by_one, + ); + + let cache_path = JsonlScanner::cache_path(ProviderId::Codex, Some(&cache_root)); + assert!( + !cache_path.exists(), + "one-over-limit encoded artifact must be refused" + ); +} diff --git a/rust/src/core/jsonl_scanner/tests/line_parsing.rs b/rust/src/core/jsonl_scanner/tests/line_parsing.rs new file mode 100644 index 0000000000..7359b5b3bb --- /dev/null +++ b/rust/src/core/jsonl_scanner/tests/line_parsing.rs @@ -0,0 +1,414 @@ +//! The fast Codex line parser, committed prefixes and line-length limits. + +use super::*; + +#[test] +fn test_fast_codex_parser_reads_last_usage_from_payload() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + ); + let mut parser = CodexParserState::new(None, None); + + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:00.000Z","type":"turn_context","payload":{"info":{"model":"gpt-5.5"}}}"#, + &range, + ); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:02.000Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":120,"cache_read_input_tokens":40,"output_tokens":9}}}}"#, + &range, + ); + + assert_eq!(parser.records.len(), 1); + let (record, _) = &parser.records[0]; + assert_eq!(record.day_key, "2026-05-31"); + assert_eq!(record.model, "gpt-5.5"); + assert_eq!((record.input, record.cached, record.output), (120, 40, 9)); + assert_eq!(parser.current_model.as_deref(), Some("gpt-5.5")); +} + +#[test] +fn test_fast_codex_parser_diffs_total_usage() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + ); + let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); + + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01.000Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":1000,"cached_input_tokens":200,"output_tokens":50}}}}"#, + &range, + ); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:02.000Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":1250,"cached_input_tokens":260,"output_tokens":90}}}}"#, + &range, + ); + + assert_eq!(parser.records.len(), 2); + assert_eq!( + parser + .records + .iter() + .map(|(record, _)| (record.input, record.cached, record.output)) + .collect::>(), + vec![(1_000, 200, 50), (250, 60, 40)] + ); + let totals = parser.previous_totals.expect("last totals"); + assert_eq!(totals.input, 1250); + assert_eq!(totals.cached, 260); + assert_eq!(totals.output, 90); +} + +#[test] +fn test_fast_codex_parser_reads_legacy_event_msg_shape() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + ); + let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); + + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:02.000Z","type":"event_msg","event_msg":{"type":"token_count","input_tokens":20,"cached_input_tokens":5,"output_tokens":3}}"#, + &range, + ); + + assert_eq!(parser.records.len(), 1); + let (record, _) = &parser.records[0]; + assert_eq!(record.model, "gpt-5"); + assert_eq!((record.input, record.cached, record.output), (20, 5, 3)); +} + +#[test] +fn codex_fast_parser_accepts_compact_and_spaced_event_records_equally() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + ); + let parsed = |line: &str| { + let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); + parser.process_line(line, &range); + ( + parser.current_model, + parser + .records + .into_iter() + .map(|(record, _)| { + ( + record.day_key, + record.model, + record.input, + record.cached, + record.output, + record.reasoning, + ) + }) + .collect::>(), + ) + }; + + let compact_event = r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":20,"cached_input_tokens":5,"output_tokens":3}}}}"#; + let spaced_event = r#"{ "timestamp": "2026-05-31T10:00:01Z", "type": "event_msg", "payload": { "type": "token_count", "info": { "last_token_usage": { "input_tokens": 20, "cached_input_tokens": 5, "output_tokens": 3 } } } }"#; + assert!(is_candidate_codex_line(spaced_event)); + assert_eq!(parsed(compact_event), parsed(spaced_event)); + + let compact_context = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{"model":"gpt-5.5"}}"#; + let spaced_context = "{\t\"timestamp\": \"2026-05-31T10:00:00Z\",\t\"type\":\t\"turn_context\",\t\"payload\": {\t\"model\": \"gpt-5.5\"\t}\t}"; + assert!(is_candidate_codex_line(spaced_context)); + assert_eq!(parsed(compact_context), parsed(spaced_context)); + + let unrelated = r#"{ "timestamp": "2026-05-31T10:00:00Z", "type": "response", "payload": { "model": "gpt-5.5" } }"#; + assert!(!is_candidate_codex_line(unrelated)); + assert_eq!(parsed(unrelated), (Some("gpt-5".to_string()), Vec::new())); +} + +#[test] +fn test_parse_codex_file_uses_fast_parser_for_current_logs() { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + writeln!( + file, + r#"{{"timestamp":"2026-05-31T10:00:00.000Z","type":"turn_context","payload":{{"model":"gpt-5.5"}}}}"# + ) + .unwrap(); + writeln!( + file, + r#"{{"timestamp":"2026-05-31T10:00:01.000Z","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":45,"cached_input_tokens":12,"output_tokens":8}}}}}}}}"# + ) + .unwrap(); + + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + ); + let parsed = JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None).expect("parse"); + + assert_eq!(parsed.last_model.as_deref(), Some("gpt-5.5")); + assert_eq!(parsed.records.len(), 1); + let (record, _) = &parsed.records[0]; + assert_eq!(record.day_key, "2026-05-31"); + assert_eq!(record.model, "gpt-5.5"); + assert_eq!((record.input, record.cached, record.output), (45, 12, 8)); +} + +#[test] +fn codex_append_timestamp_state_is_output_equivalent_and_boundary_only() { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + for (timestamp, input, output) in [ + ("2026-05-31T10:00:01.000Z", 10, 1), + ("2026-05-31T10:00:02.000Z", 20, 2), + ] { + writeln!( + file, + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5.5","total_token_usage":{{"input_tokens":{input},"cached_input_tokens":0,"output_tokens":{output}}}}}}}}}"# + ) + .unwrap(); + } + + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let prefix = + JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None).expect("parse prefix"); + assert_eq!(prefix.token_timestamps_monotonic, Some(true)); + assert_eq!(prefix.token_timestamp_comparisons, 1); + let prefix_input: i64 = prefix.records.iter().map(|(record, _)| record.input).sum(); + + writeln!( + file, + r#"{{"timestamp":"2026-05-31T10:00:03.000Z","type":"event_msg","payload":{{"type":"token_count","info":{{"model":"gpt-5.5","total_token_usage":{{"input_tokens":30,"cached_input_tokens":0,"output_tokens":3}}}}}}}}"# + ) + .unwrap(); + + let appended = JsonlScanner::parse_codex( + file.path(), + &range, + CodexParseMode::Standard { + start_offset: prefix.parsed_bytes, + initial_model: prefix.last_model.clone(), + initial_totals: prefix.last_totals.clone(), + previous_token_timestamp: prefix.last_token_timestamp.clone(), + token_timestamps_monotonic: prefix.token_timestamps_monotonic, + }, + None, + None, + None, + ) + .expect("parse appended suffix"); + assert_eq!(appended.token_timestamps_monotonic, Some(true)); + assert_eq!( + appended.token_timestamp_comparisons, 1, + "only the cached-prefix boundary is compared" + ); + + let full = JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None) + .expect("parse complete file"); + let full_input: i64 = full.records.iter().map(|(record, _)| record.input).sum(); + let appended_input: i64 = appended + .records + .iter() + .map(|(record, _)| record.input) + .sum(); + assert_eq!(prefix_input + appended_input, full_input); + assert_eq!(full_input, 30); +} + +#[test] +fn codex_parse_publishes_only_the_committed_prefix_before_an_incomplete_tail() { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let committed_line = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.5","total_token_usage":{"input_tokens":10,"cached_input_tokens":0,"output_tokens":1}}}}"#; + writeln!(file, "{committed_line}").unwrap(); + let committed_bytes = + i64::try_from(committed_line.len() + 1).expect("fixture line length fits i64"); + + let complete_tail = r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.5","total_token_usage":{"input_tokens":20,"cached_input_tokens":0,"output_tokens":2}}}}"#; + let split = complete_tail.len() / 2; + write!(file, "{}", &complete_tail[..split]).unwrap(); + file.flush().unwrap(); + + let partial = JsonlScanner::parse_codex_file(file.path(), &range, 0, None, None) + .expect("parse committed prefix"); + assert_eq!(partial.records.len(), 1); + assert_eq!(partial.records[0].0.input, 10); + assert_eq!(partial.parsed_bytes, committed_bytes); + assert_eq!(partial.scan_target_size, committed_bytes); + assert!(partial.is_complete, "the logical prefix is complete"); + + writeln!(file, "{}", &complete_tail[split..]).unwrap(); + let resumed = JsonlScanner::parse_codex( + file.path(), + &range, + CodexParseMode::Standard { + start_offset: partial.parsed_bytes, + initial_model: partial.last_model, + initial_totals: partial.last_totals, + previous_token_timestamp: partial.last_token_timestamp, + token_timestamps_monotonic: partial.token_timestamps_monotonic, + }, + None, + None, + None, + ) + .expect("resume completed tail"); + assert_eq!(resumed.records.len(), 1); + assert_eq!(resumed.records[0].0.input, 10); + assert_eq!( + resumed.parsed_bytes, + i64::try_from(std::fs::metadata(file.path()).unwrap().len()) + .expect("fixture file length fits i64") + ); + assert_eq!(resumed.scan_target_size, resumed.parsed_bytes); + assert!(resumed.is_complete); +} + +#[test] +fn codex_parser_discards_oversized_line_and_recovers_next_record() { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + let padding = "x".repeat(CODEX_JSONL_MAX_LINE_BYTES); + writeln!( + file, + r#"{{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{{"model":"{padding}"}}}}"# + ) + .unwrap(); + writeln!( + file, + r#"{{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":9,"cached_input_tokens":2,"output_tokens":1}}}}}}}}"# + ) + .unwrap(); + + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let parsed = JsonlScanner::parse_codex_file( + file.path(), + &CostUsageDayRange::new(day, day), + 0, + None, + None, + ) + .expect("parse"); + + assert_eq!(parsed.records.len(), 1); + assert_eq!( + parsed.records[0].0.model, + CostUsagePricing::CODEX_UNATTRIBUTED_MODEL + ); + assert_eq!( + ( + parsed.records[0].0.input, + parsed.records[0].0.cached, + parsed.records[0].0.output + ), + (9, 2, 1) + ); +} + +#[test] +fn codex_parser_validates_a_record_at_the_line_limit() { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + let prefix = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":9,"cached_input_tokens":2,"output_tokens":1}}},"padding":""}"#; + let padding_len = CODEX_JSONL_MAX_LINE_BYTES - prefix.len(); + let line = format!( + "{}{}\"}}", + &prefix[..prefix.len() - 2], + "x".repeat(padding_len) + ); + assert_eq!(line.len(), CODEX_JSONL_MAX_LINE_BYTES); + writeln!(file, "{line}").unwrap(); + + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let parsed = JsonlScanner::parse_codex_file( + file.path(), + &CostUsageDayRange::new(day, day), + 0, + None, + None, + ) + .expect("parse"); + + assert_eq!(parsed.records.len(), 1); + assert_eq!( + ( + parsed.records[0].0.input, + parsed.records[0].0.cached, + parsed.records[0].0.output + ), + (9, 2, 1) + ); +} + +#[test] +fn codex_parser_discards_a_line_at_limit_plus_one_and_keeps_following_record() { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + let prefix = r#"{"padding":""}"#; + let padding_len = CODEX_JSONL_MAX_LINE_BYTES + 1 - prefix.len(); + let oversized = format!( + "{}{}\"}}", + &prefix[..prefix.len() - 2], + "x".repeat(padding_len) + ); + assert_eq!(oversized.len(), CODEX_JSONL_MAX_LINE_BYTES + 1); + writeln!(file, "{oversized}").unwrap(); + let valid = r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":9,"cached_input_tokens":2,"output_tokens":1}}}}"#; + writeln!(file, "{valid}").unwrap(); + + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let parsed = JsonlScanner::parse_codex_file( + file.path(), + &CostUsageDayRange::new(day, day), + 0, + None, + None, + ) + .expect("parse"); + + assert_eq!(parsed.records.len(), 1); + assert_eq!(parsed.records[0].0.input, 9); +} + +#[test] +fn codex_parser_discards_huge_malformed_lines_before_and_after_valid_records() { + let mut file = tempfile::NamedTempFile::new().expect("temp file"); + let valid = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":9,"cached_input_tokens":2,"output_tokens":1}}}}"#; + let malformed = format!("{{{}", "x".repeat(CODEX_JSONL_MAX_LINE_BYTES * 4)); + writeln!(file, "{malformed}").unwrap(); + writeln!(file, "{valid}").unwrap(); + writeln!(file, "{malformed}").unwrap(); + + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let parsed = JsonlScanner::parse_codex_file( + file.path(), + &CostUsageDayRange::new(day, day), + 0, + None, + None, + ) + .expect("parse"); + + assert_eq!(parsed.records.len(), 1); + assert_eq!(parsed.records[0].0.input, 9); +} + +#[test] +fn bounded_jsonl_reader_accepts_exact_limit_without_retaining_larger_input() { + let mut input = vec![b'x'; CODEX_JSONL_MAX_LINE_BYTES]; + input.push(b'\n'); + input.extend_from_slice(br#"{"type":"event_msg"}"#); + input.push(b'\n'); + let mut reader = BufReader::with_capacity(64 * 1024, std::io::Cursor::new(input)); + + let exact = match read_bounded_jsonl_line(&mut reader, CODEX_JSONL_MAX_LINE_BYTES) + .expect("read") + .expect("line") + { + BoundedJsonlLine::Retained { bytes, .. } => bytes, + BoundedJsonlLine::Discarded { .. } => panic!("exact-limit line was discarded"), + }; + let later = match read_bounded_jsonl_line(&mut reader, CODEX_JSONL_MAX_LINE_BYTES) + .expect("read") + .expect("line") + { + BoundedJsonlLine::Retained { bytes, .. } => bytes, + BoundedJsonlLine::Discarded { .. } => panic!("following line was discarded"), + }; + + assert_eq!(exact.len(), CODEX_JSONL_MAX_LINE_BYTES); + assert_eq!(later, br#"{"type":"event_msg"}"#); +} diff --git a/rust/src/core/jsonl_scanner/tests/timestamps.rs b/rust/src/core/jsonl_scanner/tests/timestamps.rs new file mode 100644 index 0000000000..916a7f3263 --- /dev/null +++ b/rust/src/core/jsonl_scanner/tests/timestamps.rs @@ -0,0 +1,198 @@ +//! Day ranges, day keys and Codex timestamp parsing and ordering. + +use super::*; + +fn codex_timestamp_day_key(timestamp: &str) -> Option { + parse_codex_timestamp(timestamp).map(|parsed| parsed.day_key()) +} + +#[test] +fn test_day_range() { + let since = NaiveDate::from_ymd_opt(2026, 1, 15).unwrap(); + let until = NaiveDate::from_ymd_opt(2026, 1, 20).unwrap(); + let range = CostUsageDayRange::new(since, until); + + assert_eq!(range.since_key, "2026-01-15"); + assert_eq!(range.until_key, "2026-01-20"); + assert_eq!(range.scan_since_key, "2026-01-14"); + assert_eq!(range.scan_until_key, "2026-01-21"); +} + +#[test] +fn test_is_in_range() { + assert!(CostUsageDayRange::is_in_range( + "2026-01-15", + "2026-01-10", + "2026-01-20" + )); + assert!(!CostUsageDayRange::is_in_range( + "2026-01-05", + "2026-01-10", + "2026-01-20" + )); + assert!(!CostUsageDayRange::is_in_range( + "2026-01-25", + "2026-01-10", + "2026-01-20" + )); +} + +#[test] +fn test_parse_day_key() { + let date = CostUsageDayRange::parse_day_key("2026-01-15"); + assert!(date.is_some()); + let date = date.unwrap(); + assert_eq!(date.year(), 2026); + assert_eq!(date.month(), 1); + assert_eq!(date.day(), 15); +} + +#[test] +fn codex_timestamp_day_key_uses_local_calendar_day() { + let today = Local::now().date_naive(); + let local_midnight = today.and_hms_opt(0, 30, 0).unwrap(); + let Some(local_time) = Local.from_local_datetime(&local_midnight).earliest() else { + return; + }; + let utc_timestamp = local_time.with_timezone(&chrono::Utc).to_rfc3339(); + let expected = today.format("%Y-%m-%d").to_string(); + + assert_eq!( + codex_timestamp_day_key(&utc_timestamp).as_deref(), + Some(expected.as_str()) + ); +} + +#[test] +fn native_codex_timestamp_parser_matches_chrono_for_supported_spellings() { + for timestamp in [ + "2026-05-31T10:00:00Z", + "2026-05-31T10:00:00.123Z", + "2024-02-29T23:59:59.999+05:30", + "1900-02-28T00:00:00-08:00", + "1899-12-31T23:59:59.000Z", + ] { + assert_eq!( + parse_rfc3339_timestamp(timestamp), + DateTime::parse_from_rfc3339(timestamp).ok(), + "native parser changed {timestamp}" + ); + } + for timestamp in [ + "2026-02-29T10:00:00Z", + "2026-05-31T10:00:00.1234567890Z", + "2026-05-31T10:00:00+0530", + "2026-05-31T24:00:00Z", + ] { + assert_eq!( + parse_rfc3339_timestamp(timestamp), + DateTime::parse_from_rfc3339(timestamp).ok(), + "native parser changed invalid {timestamp}" + ); + } +} + +#[test] +fn codex_timestamp_fallback_rejects_invalid_calendar_prefixes() { + for timestamp in [ + "2026-02-29T10:00:00Z", + "2026-04-31T10:00:00Z", + "not-a-dateT10:00:00Z", + "2026-05-31", + ] { + assert!( + parse_codex_timestamp(timestamp).is_none(), + "invalid timestamp must not be accepted by the day-key fallback: {timestamp}" + ); + } + + for timestamp in ["2026-05-31T10:00:00+0530", "2026-05-31T10:00:00+05"] { + let parsed = parse_codex_timestamp(timestamp).expect("historical timestamp shape"); + assert_eq!(parsed.fallback_day_key, "2026-05-31"); + assert!(parsed.parsed.is_none()); + } +} + +#[test] +fn codex_timestamp_order_latches_false_and_stops_rechecking() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); + + for (timestamp, input) in [ + ("2026-05-31T10:00:02Z", 10), + ("2026-05-31T10:00:01Z", 20), + ("2026-05-31T10:00:03Z", 30), + ] { + parser.process_line( + &format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":{input},"cached_input_tokens":0,"output_tokens":1}}}}}}}}"# + ), + &range, + ); + } + + assert_eq!(parser.token_timestamps_monotonic, Some(false)); + assert_eq!(parser.token_timestamp_comparisons, 1); + assert_eq!(parser.records.len(), 3); +} + +#[test] +fn codex_timestamp_order_ignores_sub_millisecond_fraction() { + let day = NaiveDate::from_ymd_opt(2026, 8, 30).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); + + for timestamp in ["2026-08-30T12:00:00.1239Z", "2026-08-30T12:00:00.1231Z"] { + parser.process_line( + &format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}}}}}}}"# + ), + &range, + ); + } + + assert_eq!(parser.token_timestamps_monotonic, Some(true)); + assert_eq!(parser.token_timestamp_comparisons, 1); +} + +#[test] +fn codex_timestamp_order_detects_millisecond_decrease() { + let day = NaiveDate::from_ymd_opt(2026, 8, 30).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); + + for timestamp in ["2026-08-30T12:00:00.124Z", "2026-08-30T12:00:00.123Z"] { + parser.process_line( + &format!( + r#"{{"timestamp":"{timestamp}","type":"event_msg","payload":{{"type":"token_count","info":{{"last_token_usage":{{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}}}}}}}"# + ), + &range, + ); + } + + assert_eq!(parser.token_timestamps_monotonic, Some(false)); + assert_eq!(parser.token_timestamp_comparisons, 1); +} + +#[test] +fn codex_timestamp_order_checks_token_history_outside_requested_window() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); + + for line in [ + r#"{"timestamp":"2026-06-01T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":10,"cached_input_tokens":0,"output_tokens":1}}}}"#, + r#"{"timestamp":"2026-05-31T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":20,"cached_input_tokens":0,"output_tokens":2}}}}"#, + ] { + parser.process_line(line, &range); + } + + assert_eq!(parser.token_timestamps_monotonic, Some(false)); + assert_eq!(parser.token_timestamp_comparisons, 1); + assert_eq!( + parser.records.len(), + 1, + "only the in-range event is recorded" + ); +} diff --git a/rust/src/core/jsonl_scanner/tests/token_accounting.rs b/rust/src/core/jsonl_scanner/tests/token_accounting.rs new file mode 100644 index 0000000000..0dd8232c46 --- /dev/null +++ b/rust/src/core/jsonl_scanner/tests/token_accounting.rs @@ -0,0 +1,360 @@ +//! Codex token deltas: reasoning, fork baselines, clamping and packed rows. + +use super::*; + +fn last_usage_delta(last: &Value) -> (i64, i64, i64, Option) { + let totals = read_token_totals(last); + ( + totals.input.max(0), + totals.cached.max(0), + totals.output.max(0), + totals.reasoning, + ) +} + +#[test] +fn reasoning_output_is_clamped_and_preserved_for_value_and_fast_shapes() { + let value = serde_json::json!({ + "output_tokens": 20, + "reasoning_output_tokens": 7, + }); + let totals = read_token_totals(&value); + assert_eq!(totals.output, 20); + assert_eq!(totals.reasoning, Some(7)); + assert_eq!(last_usage_delta(&value), (0, 0, 20, Some(7))); + + let fast: CodexFastTotals = serde_json::from_value(serde_json::json!({ + "output_tokens": 20, + "reasoning_output_tokens": 99, + })) + .unwrap(); + let fast_totals = codex_totals_from_fast(fast); + assert_eq!(fast_totals.output, 20); + assert_eq!(fast_totals.reasoning, Some(20)); +} + +#[test] +fn missing_reasoning_stays_unknown_for_cumulative_and_event_usage() { + let value = serde_json::json!({ "output_tokens": 20 }); + assert_eq!(read_token_totals(&value).reasoning, None); + assert_eq!(last_usage_delta(&value), (0, 0, 20, None)); + + let mut state = CodexParserState::new(None, None); + assert_eq!( + state.total_usage_delta(&serde_json::json!({ + "output_tokens": 10, + })), + (0, 0, 10, None) + ); +} + +#[test] +fn cumulative_reasoning_uses_the_comparable_previous_total() { + let mut state = CodexParserState::new(None, None); + assert_eq!( + state.total_usage_delta(&serde_json::json!({ + "output_tokens": 10, + "reasoning_output_tokens": 4, + })), + (0, 0, 10, Some(4)) + ); + assert_eq!( + state.total_usage_delta(&serde_json::json!({ + "output_tokens": 20, + "reasoning_output_tokens": 9, + })), + (0, 0, 10, Some(5)) + ); +} + +#[test] +fn fork_baseline_subtracts_known_reasoning_without_affecting_core_tokens() { + let baseline = CodexTotals { + input: 10, + cached: 2, + output: 10, + reasoning: Some(4), + }; + let mut state = CodexParserState::from_mode(CodexParseMode::ParentBaseline { + baseline, + paginated_continuation: false, + remaining_inherited_totals: None, + }); + assert_eq!( + state.apply_totals_delta(CodexTotals { + input: 20, + cached: 5, + output: 20, + reasoning: Some(9), + }), + (10, 3, 10, Some(5)) + ); + + let baseline_without_reasoning = CodexTotals { + input: 10, + cached: 2, + output: 10, + reasoning: None, + }; + let mut state = CodexParserState::from_mode(CodexParseMode::ParentBaseline { + baseline: baseline_without_reasoning, + paginated_continuation: false, + remaining_inherited_totals: None, + }); + assert_eq!( + state.apply_totals_delta(CodexTotals { + input: 20, + cached: 5, + output: 20, + reasoning: Some(9), + }), + (10, 3, 10, None) + ); + assert!(!state.fork_baseline_ambiguous); +} + +#[test] +fn inferred_fork_waits_for_present_explicit_start_ordinal() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + ); + let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { + start_ordinal: Some(10), + }); + + state.process_line( + r#"{"timestamp":"2026-09-22T10:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.6-sol","total_token_usage":{"input_tokens":100,"cached_input_tokens":20,"output_tokens":10},"last_token_usage":{"input_tokens":0,"cached_input_tokens":0,"output_tokens":0}}}}"#, + &range, + ); + + assert!(state.records.is_empty()); + assert!(state.fork_baseline.is_none()); + assert!(!state.fork_baseline_locally_resolved()); + + state.process_line( + r#"{"ordinal":10,"timestamp":"2026-09-22T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"model":"gpt-5.6-sol","total_token_usage":{"input_tokens":110,"cached_input_tokens":22,"output_tokens":11},"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":1}}}}"#, + &range, + ); + + assert_eq!(state.records.len(), 1); + assert_eq!(state.records[0].0.input, 10); + assert_eq!(state.records[0].0.cached, 2); + assert_eq!(state.records[0].0.output, 1); + assert!(!state.fork_baseline_locally_resolved()); +} + +#[test] +fn inferred_fork_keeps_missing_ordinal_unresolved_after_boundary_opens() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + ); + let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { + start_ordinal: Some(10), + }); + let token_line = |ordinal: Option, total: i64, last: i64| { + let mut value = serde_json::json!({ + "timestamp": "2026-09-22T10:00:00Z", + "type": "event_msg", + "payload": {"type": "token_count", "info": { + "model": "gpt-5.6-sol", + "total_token_usage": {"input_tokens": total, "cached_input_tokens": 0, "output_tokens": 0}, + "last_token_usage": {"input_tokens": last, "cached_input_tokens": 0, "output_tokens": 0} + }} + }); + if let Some(ordinal) = ordinal { + value["ordinal"] = serde_json::json!(ordinal); + } + value.to_string() + }; + + state.process_line(&token_line(Some(9), 100, 0), &range); + state.process_line(&token_line(Some(10), 100, 0), &range); + state.process_line(&token_line(Some(11), 110, 110), &range); + assert!(state.fork_baseline_locally_resolved()); + state.process_line(&token_line(None, 120, 10), &range); + assert!(!state.fork_baseline_locally_resolved()); + state.process_line(&token_line(Some(12), 130, 10), &range); + + assert_eq!(state.records.len(), 1); + assert_eq!(state.records[0].0.input, 10); + assert!(!state.fork_baseline_locally_resolved()); +} + +#[test] +fn inferred_fork_keeps_missing_ordinal_unresolved_after_local_resolution() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + NaiveDate::from_ymd_opt(2026, 9, 22).unwrap(), + ); + let mut state = CodexParserState::from_mode(CodexParseMode::InferSubagent { + start_ordinal: Some(10), + }); + let token_line = |ordinal: Option, total: i64, last: i64| { + let mut value = serde_json::json!({ + "timestamp": "2026-09-22T10:00:00Z", + "type": "event_msg", + "payload": {"type": "token_count", "info": { + "model": "gpt-5.6-sol", + "total_token_usage": {"input_tokens": total, "cached_input_tokens": 0, "output_tokens": 0}, + "last_token_usage": {"input_tokens": last, "cached_input_tokens": 0, "output_tokens": 0} + }} + }); + if let Some(ordinal) = ordinal { + value["ordinal"] = serde_json::json!(ordinal); + } + value.to_string() + }; + + state.process_line(&token_line(Some(9), 100, 0), &range); + state.process_line(&token_line(Some(10), 100, 0), &range); + state.process_line(&token_line(Some(11), 110, 10), &range); + assert!(state.fork_baseline_locally_resolved()); + state.process_line(&token_line(None, 120, 10), &range); + + assert!(!state.fork_baseline_locally_resolved()); +} + +#[test] +fn codex_token_pipeline_preserves_counts_above_i32_max() { + let parsed = read_token_totals(&serde_json::json!({ + "input_tokens": 3_000_000_000_i64, + "cached_input_tokens": 2_800_000_000_i64, + "output_tokens": 200, + })); + assert_eq!(parsed.input, 3_000_000_000); + assert_eq!(parsed.cached, 2_800_000_000); + assert_eq!(parsed.output, 200); + + let mut packed = Vec::new(); + for _ in 0..2 { + JsonlScanner::merge_codex_record_into_packed( + &mut packed, + &CodexUsageRecord { + day_key: "2026-09-09".to_string(), + timestamp: None, + model: "gpt-5.6-luna".to_string(), + input: 1_500_000_000, + cached: 1_400_000_000, + output: 100, + reasoning: None, + turn_id: None, + }, + ); + } + assert_eq!(packed, vec![3_000_000_000, 2_800_000_000, 200]); + + let mut cache = CostUsageCache::default(); + cache.days.insert( + "2026-09-09".to_string(), + HashMap::from([("gpt-5.6-luna".to_string(), packed)]), + ); + let report = JsonlScanner::cached_cost_report_from_days(&cache); + assert_eq!(report.input_tokens, 3_000_000_000); + assert_eq!(report.cached_tokens, 2_800_000_000); + assert_eq!(report.output_tokens, 200); +} + +#[test] +fn negative_cumulative_components_are_clamped_at_the_source() { + let value = serde_json::json!({ + "input_tokens": -5, + "cached_input_tokens": -9, + "cache_read_input_tokens": -3, + "output_tokens": -2, + "reasoning_output_tokens": -1, + }); + let totals = read_token_totals(&value); + assert_eq!(totals.input, 0); + assert_eq!(totals.cached, 0); + assert_eq!(totals.output, 0); + assert_eq!(totals.reasoning, Some(0)); + + let fast: CodexFastTotals = serde_json::from_value(value.clone()).unwrap(); + let fast_totals = codex_totals_from_fast(fast); + assert_eq!(fast_totals.input, 0); + assert_eq!(fast_totals.cached, 0); + assert_eq!(fast_totals.output, 0); + assert_eq!(fast_totals.reasoning, Some(0)); + + // The payload borrows `&str` fields, so deserialize from a str rather than + // an owned `Value`. + let payload_json = value.to_string(); + let payload: CodexFastPayload<'_> = serde_json::from_str(&payload_json).unwrap(); + let payload_totals = fast_totals_from_payload(&payload); + assert_eq!(payload_totals.input, 0); + assert_eq!(payload_totals.cached, 0); + assert_eq!(payload_totals.output, 0); + assert_eq!(payload_totals.reasoning, Some(0)); +} + +#[test] +fn negative_cumulative_totals_do_not_inflate_later_deltas() { + let mut state = CodexParserState::new(None, None); + // A malformed cumulative record with negative counts must be clamped so it + // cannot lower the high watermark below zero. + assert_eq!( + state.total_usage_delta(&serde_json::json!({ + "input_tokens": -5, + "output_tokens": -2, + })), + (0, 0, 0, None) + ); + // A later normal climb only counts its true growth above the clamped zero. + assert_eq!( + state.total_usage_delta(&serde_json::json!({ + "input_tokens": 3, + "output_tokens": 1, + })), + (3, 0, 1, None) + ); +} + +#[test] +fn legacy_packed_rows_remain_three_slots_and_report_reasoning_is_unknown() { + let record = CodexUsageRecord { + day_key: "2026-05-31".to_string(), + timestamp: None, + model: "gpt-5.6-sol".to_string(), + input: 5, + cached: 1, + output: 3, + reasoning: Some(2), + turn_id: None, + }; + let mut packed = vec![10, 2, 4]; + JsonlScanner::merge_codex_record_into_packed(&mut packed, &record); + assert_eq!(packed, vec![15, 3, 7]); + + let mut cache = CostUsageCache::default(); + cache.days.insert( + "2026-05-31".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), packed)]), + ); + let report = JsonlScanner::cached_cost_report_from_days(&cache); + assert_eq!(report.reasoning_tokens, None); +} + +#[test] +fn known_packed_rows_report_reasoning_only_when_all_token_rows_are_known() { + let mut cache = CostUsageCache::default(); + cache.days.insert( + "2026-05-31".to_string(), + HashMap::from([("gpt-5.6-sol".to_string(), vec![10, 2, 4, 3])]), + ); + assert_eq!( + JsonlScanner::cached_cost_report_from_days(&cache).reasoning_tokens, + Some(3) + ); + + cache + .days + .get_mut("2026-05-31") + .unwrap() + .insert("gpt-5.6-fast".to_string(), vec![1, 0, 1]); + assert_eq!( + JsonlScanner::cached_cost_report_from_days(&cache).reasoning_tokens, + None + ); +} diff --git a/rust/src/core/jsonl_scanner/tests/usage_rows.rs b/rust/src/core/jsonl_scanner/tests/usage_rows.rs new file mode 100644 index 0000000000..b9d08ea52f --- /dev/null +++ b/rust/src/core/jsonl_scanner/tests/usage_rows.rs @@ -0,0 +1,214 @@ +//! Codex model attribution, bare usage rows and lineage watermarks. + +use super::*; + +#[test] +fn codex_turn_context_wins_over_conflicting_event_model() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(None, None); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{"model":"gpt-5.5"}}"#, + &range, + ); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","model":"gpt-5.6-sol","info":{"last_token_usage":{"input_tokens":5,"cached_input_tokens":1,"output_tokens":2}}}}"#, + &range, + ); + + assert_eq!(parser.records[0].0.model, "gpt-5.5"); +} + +#[test] +fn codex_blank_context_clears_stale_model_and_emits_unattributed_usage() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(Some("gpt-5.5".to_string()), None); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{"model":" "}}"#, + &range, + ); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":5,"cached_input_tokens":1,"output_tokens":2}}}}"#, + &range, + ); + + assert_eq!( + parser.records[0].0.model, + CostUsagePricing::CODEX_UNATTRIBUTED_MODEL + ); +} + +#[test] +fn codex_model_less_token_event_uses_unpriced_sentinel() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(None, None); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":10,"cached_input_tokens":0,"output_tokens":2}}}}"#, + &range, + ); + + assert_eq!(parser.records.len(), 1); + assert_eq!( + parser.records[0].0.model, + CostUsagePricing::CODEX_UNATTRIBUTED_MODEL + ); +} + +#[test] +fn cached_tokens_use_larger_cached_or_cache_read_field() { + let value = serde_json::json!({ + "input_tokens": 100, + "cached_input_tokens": 20, + "cache_read_input_tokens": 35, + "output_tokens": 10 + }); + let totals = read_token_totals(&value); + assert_eq!(totals.cached, 35); +} + +#[test] +fn parses_bare_usage_rows_outside_token_count_envelope() { + let value = serde_json::json!({ + "model": "gpt-5.6-sol", + "usage": { + "prompt_tokens": 120, + "completion_tokens": 30, + "cached_input_tokens": 40, + "cache_read_input_tokens": 55 + } + }); + let (totals, model) = bare_usage_totals(&value).expect("bare usage"); + assert_eq!(totals.input, 120); + assert_eq!(totals.output, 30); + assert_eq!(totals.cached, 55); + assert_eq!(model.as_deref(), Some("gpt-5.6-sol")); +} + +#[test] +fn process_line_accepts_type_less_bare_usage_row() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(None, None); + + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01Z","model":"gpt-5.6-sol","usage":{"prompt_tokens":120,"completion_tokens":30,"cache_read_input_tokens":55}}"#, + &range, + ); + + assert_eq!(parser.records.len(), 1); + assert_eq!(parser.records[0].0.model, "gpt-5.6-sol"); + assert_eq!( + ( + parser.records[0].0.input, + parser.records[0].0.cached, + parser.records[0].0.output + ), + (120, 55, 30) + ); +} + +#[test] +fn process_line_keeps_bare_usage_when_model_contains_turn_context() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(None, None); + + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01Z","model":"turn_context","usage":{"prompt_tokens":120,"completion_tokens":30}}"#, + &range, + ); + + assert_eq!(parser.records.len(), 1); + assert_eq!(parser.records[0].0.input, 120); + assert_eq!(parser.records[0].0.output, 30); +} + +#[test] +fn timestamp_less_bare_usage_uses_last_accepted_usage_day() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(Some("gpt-5.6-sol".to_string()), None); + + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":10,"cached_input_tokens":2,"output_tokens":1}}}}"#, + &range, + ); + parser.process_line( + r#"{"usage":{"prompt_tokens":20,"completion_tokens":4,"cache_read_input_tokens":3}}"#, + &range, + ); + + assert_eq!(parser.records.len(), 2); + assert_eq!(parser.records[1].0.day_key, "2026-05-31"); + assert_eq!( + ( + parser.records[1].0.input, + parser.records[1].0.cached, + parser.records[1].0.output + ), + (20, 3, 4) + ); +} + +#[test] +fn interleaved_lineage_totals_never_exceed_high_watermark_growth() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(Some("gpt-5.6-sol".to_string()), None); + + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":100,"cached_input_tokens":0,"output_tokens":20}}}}"#, + &range, + ); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:02Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":5,"cached_input_tokens":0,"output_tokens":1}}}}"#, + &range, + ); + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:03Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":101,"cached_input_tokens":0,"output_tokens":21}}}}"#, + &range, + ); + + let total_input: i64 = parser.records.iter().map(|(r, _)| r.input).sum(); + let total_output: i64 = parser.records.iter().map(|(r, _)| r.output).sum(); + assert!( + total_input <= 101, + "input inflated to {total_input}, expected <= 101" + ); + assert!( + total_output <= 21, + "output inflated to {total_output}, expected <= 21" + ); +} + +#[test] +fn interleaved_lineage_mid_range_climb_below_watermark_does_not_readd() { + // 100 → 5 (rewind) → 80 (mid-range below water) → 101 (above water). + // Phase-1 containment: do not re-add the 5→80 climb; only growth above + // the historical high watermark counts. + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(Some("gpt-5.6-sol".to_string()), None); + + for (input, output) in [(100, 20), (5, 1), (80, 10), (101, 21)] { + parser.process_line( + &format!( + r#"{{"timestamp":"2026-05-31T10:00:0{input}Z","type":"event_msg","payload":{{"type":"token_count","info":{{"total_token_usage":{{"input_tokens":{input},"cached_input_tokens":0,"output_tokens":{output}}}}}}}"# + ), + &range, + ); + } + + let total_input: i64 = parser.records.iter().map(|(r, _)| r.input).sum(); + let total_output: i64 = parser.records.iter().map(|(r, _)| r.output).sum(); + assert!( + total_input <= 101, + "mid-range climb re-added input to {total_input}, expected <= 101" + ); + assert!( + total_output <= 21, + "mid-range climb re-added output to {total_output}, expected <= 21" + ); +} From 6330fb70466e527b14e0285a2d828a829c029387 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:15:18 +0700 Subject: [PATCH 37/41] Split custom pricing and the merge helpers out of the spend contract --- rust/src/spend_contract.rs | 350 +--------------------- rust/src/spend_contract/custom_pricing.rs | 158 ++++++++++ rust/src/spend_contract/merge.rs | 203 +++++++++++++ rust/src/spend_contract/tests.rs | 1 + rust/src/spend_contract/tests/activity.rs | 3 +- 5 files changed, 375 insertions(+), 340 deletions(-) create mode 100644 rust/src/spend_contract/custom_pricing.rs create mode 100644 rust/src/spend_contract/merge.rs diff --git a/rust/src/spend_contract.rs b/rust/src/spend_contract.rs index f0fb0c7741..b99b1ff2e9 100644 --- a/rust/src/spend_contract.rs +++ b/rust/src/spend_contract.rs @@ -1,24 +1,30 @@ //! Unified Usage & Spend accounting contract for upstream 0.53 parity. //! Accounting semantics live here so UI/CLI never infer unknown vs zero. +mod custom_pricing; mod local_history; +mod merge; mod opencodex; pub use local_history::{ LocalCostEstimate, LocalHistoryCoverage, LocalTokenHistorySummary, local_token_history_json, }; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fs; -use std::path::PathBuf; +use std::collections::{HashMap, HashSet}; -use chrono::{DateTime, Datelike, Local, Timelike, Utc}; +use chrono::Utc; use serde::{Deserialize, Serialize}; use crate::codex_workspaces::{CodexWorkspacesIndex, ProjectUsage, SessionUsage, SourceStatus}; use crate::cost_reporting_period::{CostReportingPeriod, MAX_ROLLING_DAYS}; use crate::cost_scanner::{ - CostScanner, CostSummary, ModelTokenCounts, get_daily_cost_history, get_daily_token_history, + CostScanner, CostSummary, get_daily_cost_history, get_daily_token_history, +}; + +use custom_pricing::{CustomPricing, CustomRates}; +use merge::{ + ActivityHistogram, activity_from_sessions, add_optional, merge_activity, merge_coverage, + merge_daily, merge_models, merge_token_mix, sum_optional_cost, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -226,151 +232,6 @@ pub struct SpendContract { pub imports: Vec, } -#[derive(Debug, Clone, Default)] -struct CustomPricing { - entries: HashMap, -} - -/// One `custom-pricing.json` entry, in USD per million tokens. `Some(0.0)` is -/// free; `None` is unknown and is never filled from another source. -#[derive(Debug, Clone, Default)] -struct CustomRates { - input: Option, - output: Option, - cache_read: Option, - cache_write: Option, -} - -impl CustomPricing { - fn default_path() -> Option { - dirs::config_dir().map(|path| path.join("CodexBar").join("custom-pricing.json")) - } - - fn load() -> Self { - Self::default_path() - .and_then(|path| fs::read(path).ok()) - .map(|bytes| Self::parse(&bytes)) - .unwrap_or_default() - } - - /// Upstream `CostUsageCustomPricing.parse`: keys are trimmed and - /// lowercased, and every entry is read on its own, so one malformed entry - /// never discards the others. An entry without a single usable rate is - /// dropped. A document that is not a JSON object is empty. - fn parse(bytes: &[u8]) -> Self { - let Ok(serde_json::Value::Object(object)) = serde_json::from_slice(bytes) else { - return Self::default(); - }; - Self { - entries: object - .into_iter() - .filter_map(|(key, value)| { - let key = key.trim().to_ascii_lowercase(); - if key.is_empty() { - return None; - } - CustomRates::from_value(&value).map(|rates| (key, rates)) - }) - .collect(), - } - } - - fn rates(&self, provider_id: &str, model: &str) -> Option<&CustomRates> { - let model_key = model.trim().to_ascii_lowercase(); - let provider_key = format!("{}/{}", provider_id.trim().to_ascii_lowercase(), model_key); - self.entries - .get(&provider_key) - .or_else(|| self.entries.get(&model_key)) - } - - /// Upstream `CostUsageCustomPricing.rates(providerID:model:)` for imported - /// ledgers: the bare model key first, then `provider/model`. An empty model - /// has no override. - fn overlay_rates(&self, provider_id: &str, model: &str) -> Option<&CustomRates> { - let model_key = model.trim().to_ascii_lowercase(); - if model_key.is_empty() { - return None; - } - self.entries.get(&model_key).or_else(|| { - let provider_key = format!("{}/{}", provider_id.trim(), model.trim()); - self.entries.get(&provider_key.to_ascii_lowercase()) - }) - } -} - -impl CustomRates { - /// Upstream `rates(from:)`: a rate that is missing, not a number, - /// negative or non-finite is unknown, and the camelCase spelling wins over - /// the snake_case one. `None` when the entry has no usable rate at all. - fn from_value(value: &serde_json::Value) -> Option { - let object = value.as_object()?; - let rate = |key: &str| { - object - .get(key) - .and_then(serde_json::Value::as_f64) - .filter(|rate| rate.is_finite() && *rate >= 0.0) - }; - let rates = Self { - input: rate("input"), - output: rate("output"), - cache_read: rate("cacheRead").or_else(|| rate("cache_read")), - cache_write: rate("cacheWrite") - .or_else(|| rate("cache_write")) - .or_else(|| rate("cacheCreation")) - .or_else(|| rate("cache_creation")), - }; - [ - rates.input, - rates.output, - rates.cache_read, - rates.cache_write, - ] - .iter() - .any(Option::is_some) - .then_some(rates) - } - - fn cost(&self, counts: &ModelTokenCounts) -> Option { - self.cost_parts( - counts.input_tokens, - counts.output_tokens, - counts.cached_tokens, - 0, - ) - } - - fn cost_parts( - &self, - input: u64, - output: u64, - cache_read: u64, - cache_write: u64, - ) -> Option { - let cached = cache_read.min(input); - self.lane_cost(input.saturating_sub(cached), output, cached, cache_write) - } - - /// Upstream `CostUsageCustomPricing.costUSD(rates:...)`: every token lane is - /// billed on its own, and a lane with tokens but no rate leaves the cost - /// unknown. A missing rate is never filled from another source. - fn lane_cost(&self, input: u64, output: u64, cache_read: u64, cache_write: u64) -> Option { - let mut total = 0.0; - if input > 0 { - total += input as f64 * self.input? / 1_000_000.0; - } - if output > 0 { - total += output as f64 * self.output? / 1_000_000.0; - } - if cache_read > 0 { - total += cache_read as f64 * self.cache_read? / 1_000_000.0; - } - if cache_write > 0 { - total += cache_write as f64 * self.cache_write? / 1_000_000.0; - } - total.is_finite().then_some(total) - } -} - /// Refreshes models.dev prices for the OpenCodex ledger before a fresh /// Usage & Spend build (upstream 0.60.4 `refreshPricingIfNeeded`). /// @@ -816,194 +677,5 @@ fn daily_points(provider_id: &str, days: u32) -> Vec { .collect() } -/// Conversation counts per local (weekday, hour) cell. -#[derive(Default)] -struct ActivityHistogram(BTreeMap<(u8, u8), u32>); - -impl ActivityHistogram { - fn add_local(&mut self, timestamp: DateTime) { - let local = timestamp.with_timezone(&Local); - #[allow( - clippy::cast_possible_truncation, - reason = "weekday (0-6) and hour (0-23) fit u8" - )] - let key = ( - local.weekday().num_days_from_monday() as u8, - local.hour() as u8, - ); - self.add(key, 1); - } - - fn add(&mut self, key: (u8, u8), conversations: u32) { - let count = self.0.entry(key).or_insert(0); - *count = count.saturating_add(conversations); - } - - fn into_cells(self) -> Vec { - self.0 - .into_iter() - .map(|((weekday, hour), conversations)| SpendActivityCell { - weekday, - hour, - conversations, - }) - .collect() - } -} - -fn activity_from_sessions(sessions: &[SessionUsage]) -> Vec { - let mut activity = ActivityHistogram::default(); - for session in sessions { - if let Some(timestamp) = session.latest_activity.or(session.started_at) { - activity.add_local(timestamp); - } - } - activity.into_cells() -} -fn sum_optional_cost(left: Option, right: Option) -> Option { - let valid = |value: f64| value.is_finite() && value >= 0.0; - match (left, right) { - (Some(left), Some(right)) if valid(left) && valid(right) => { - let total = left + right; - valid(total).then_some(total) - } - (Some(value), None) | (None, Some(value)) if valid(value) => Some(value), - (None, None) => None, - _ => None, - } -} - -fn merge_coverage( - left: CostCoverageCounts, - right: &CostCoverageCounts, -) -> (CostCoverageCounts, bool) { - let priced = left.priced.checked_add(right.priced); - let unpriced = left.unpriced.checked_add(right.unpriced); - let unmetered = left.unmetered.checked_add(right.unmetered); - let estimated = left.estimated.checked_add(right.estimated); - let exact = - priced.is_some() && unpriced.is_some() && unmetered.is_some() && estimated.is_some(); - let merged = CostCoverageCounts { - priced: priced.unwrap_or(u32::MAX), - unpriced: unpriced.unwrap_or(u32::MAX), - unmetered: unmetered.unwrap_or(u32::MAX), - estimated: estimated.unwrap_or(u32::MAX), - }; - let exact = exact && merged.checked_total().is_some(); - (merged, exact) -} - -fn merge_token_mix(mut left: SpendTokenMix, right: &SpendTokenMix) -> SpendTokenMix { - let mut overflowed_classes = left.overflowed_classes | right.overflowed_classes; - for (bit, merged, incoming) in [ - (0, &mut left.input_tokens, right.input_tokens), - (1, &mut left.output_tokens, right.output_tokens), - (2, &mut left.cache_read_tokens, right.cache_read_tokens), - ( - 3, - &mut left.cache_creation_tokens, - right.cache_creation_tokens, - ), - (4, &mut left.reasoning_tokens, right.reasoning_tokens), - ] { - *merged = merge_token_class(*merged, incoming, &mut overflowed_classes, 1 << bit); - } - left.overflowed_classes = overflowed_classes; - left -} - -fn merge_token_class( - left: Option, - right: Option, - overflowed_classes: &mut u8, - bit: u8, -) -> Option { - if *overflowed_classes & bit != 0 { - return None; - } - match (left, right) { - (Some(left), Some(right)) => match left.checked_add(right) { - Some(total) => Some(total), - None => { - *overflowed_classes |= bit; - None - } - }, - (Some(left), None) => Some(left), - (None, Some(right)) => Some(right), - (None, None) => None, - } -} - -fn add_optional(left: Option, right: Option) -> Option { - match (left, right) { - (Some(left), Some(right)) => left.checked_add(right), - (Some(left), None) => Some(left), - (None, Some(right)) => Some(right), - (None, None) => None, - } -} - -fn merge_models(mut left: Vec, right: &[SpendModelRow]) -> Vec { - for incoming in right { - if let Some(existing) = left.iter_mut().find(|row| row.model == incoming.model) { - existing.cost_usd = sum_optional_cost(existing.cost_usd, incoming.cost_usd); - existing.input_tokens = existing.input_tokens.saturating_add(incoming.input_tokens); - existing.output_tokens = existing - .output_tokens - .saturating_add(incoming.output_tokens); - existing.cache_read_tokens = existing - .cache_read_tokens - .saturating_add(incoming.cache_read_tokens); - existing.total_tokens = existing.total_tokens.saturating_add(incoming.total_tokens); - existing.custom_pricing |= incoming.custom_pricing; - } else { - left.push(incoming.clone()); - } - } - left.sort_by(|a, b| { - b.cost_usd - .partial_cmp(&a.cost_usd) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| a.model.cmp(&b.model)) - }); - left -} - -fn merge_daily(left: Vec, right: &[SpendDailyPoint]) -> Vec { - let mut days: BTreeMap = left - .into_iter() - .map(|point| (point.day.clone(), point)) - .collect(); - for incoming in right { - let entry = days - .entry(incoming.day.clone()) - .or_insert_with(|| SpendDailyPoint { - day: incoming.day.clone(), - cost_usd: None, - total_tokens: None, - }); - entry.cost_usd = sum_optional_cost(entry.cost_usd, incoming.cost_usd); - entry.total_tokens = add_optional(entry.total_tokens, incoming.total_tokens); - } - days.into_values().collect() -} - -fn merge_activity( - left: Vec, - right: &[SpendActivityCell], -) -> Vec { - // Collected, not added: a duplicate cell on the left keeps the last count. - let mut activity = ActivityHistogram( - left.into_iter() - .map(|cell| ((cell.weekday, cell.hour), cell.conversations)) - .collect(), - ); - for cell in right { - activity.add((cell.weekday, cell.hour), cell.conversations); - } - activity.into_cells() -} - #[cfg(test)] mod tests; diff --git a/rust/src/spend_contract/custom_pricing.rs b/rust/src/spend_contract/custom_pricing.rs new file mode 100644 index 0000000000..f13d3bd4dc --- /dev/null +++ b/rust/src/spend_contract/custom_pricing.rs @@ -0,0 +1,158 @@ +//! `custom-pricing.json` overrides for local spend (upstream `CostUsageCustomPricing`). + +use std::collections::HashMap; +use std::fs; +use std::path::PathBuf; + +use crate::cost_scanner::ModelTokenCounts; + +#[derive(Debug, Clone, Default)] +pub(super) struct CustomPricing { + pub(super) entries: HashMap, +} + +/// One `custom-pricing.json` entry, in USD per million tokens. `Some(0.0)` is +/// free; `None` is unknown and is never filled from another source. +#[derive(Debug, Clone, Default)] +pub(super) struct CustomRates { + pub(super) input: Option, + pub(super) output: Option, + pub(super) cache_read: Option, + pub(super) cache_write: Option, +} + +impl CustomPricing { + pub(super) fn default_path() -> Option { + dirs::config_dir().map(|path| path.join("CodexBar").join("custom-pricing.json")) + } + + pub(super) fn load() -> Self { + Self::default_path() + .and_then(|path| fs::read(path).ok()) + .map(|bytes| Self::parse(&bytes)) + .unwrap_or_default() + } + + /// Upstream `CostUsageCustomPricing.parse`: keys are trimmed and + /// lowercased, and every entry is read on its own, so one malformed entry + /// never discards the others. An entry without a single usable rate is + /// dropped. A document that is not a JSON object is empty. + pub(super) fn parse(bytes: &[u8]) -> Self { + let Ok(serde_json::Value::Object(object)) = serde_json::from_slice(bytes) else { + return Self::default(); + }; + Self { + entries: object + .into_iter() + .filter_map(|(key, value)| { + let key = key.trim().to_ascii_lowercase(); + if key.is_empty() { + return None; + } + CustomRates::from_value(&value).map(|rates| (key, rates)) + }) + .collect(), + } + } + + pub(super) fn rates(&self, provider_id: &str, model: &str) -> Option<&CustomRates> { + let model_key = model.trim().to_ascii_lowercase(); + let provider_key = format!("{}/{}", provider_id.trim().to_ascii_lowercase(), model_key); + self.entries + .get(&provider_key) + .or_else(|| self.entries.get(&model_key)) + } + + /// Upstream `CostUsageCustomPricing.rates(providerID:model:)` for imported + /// ledgers: the bare model key first, then `provider/model`. An empty model + /// has no override. + pub(super) fn overlay_rates(&self, provider_id: &str, model: &str) -> Option<&CustomRates> { + let model_key = model.trim().to_ascii_lowercase(); + if model_key.is_empty() { + return None; + } + self.entries.get(&model_key).or_else(|| { + let provider_key = format!("{}/{}", provider_id.trim(), model.trim()); + self.entries.get(&provider_key.to_ascii_lowercase()) + }) + } +} + +impl CustomRates { + /// Upstream `rates(from:)`: a rate that is missing, not a number, + /// negative or non-finite is unknown, and the camelCase spelling wins over + /// the snake_case one. `None` when the entry has no usable rate at all. + pub(super) fn from_value(value: &serde_json::Value) -> Option { + let object = value.as_object()?; + let rate = |key: &str| { + object + .get(key) + .and_then(serde_json::Value::as_f64) + .filter(|rate| rate.is_finite() && *rate >= 0.0) + }; + let rates = Self { + input: rate("input"), + output: rate("output"), + cache_read: rate("cacheRead").or_else(|| rate("cache_read")), + cache_write: rate("cacheWrite") + .or_else(|| rate("cache_write")) + .or_else(|| rate("cacheCreation")) + .or_else(|| rate("cache_creation")), + }; + [ + rates.input, + rates.output, + rates.cache_read, + rates.cache_write, + ] + .iter() + .any(Option::is_some) + .then_some(rates) + } + + pub(super) fn cost(&self, counts: &ModelTokenCounts) -> Option { + self.cost_parts( + counts.input_tokens, + counts.output_tokens, + counts.cached_tokens, + 0, + ) + } + + pub(super) fn cost_parts( + &self, + input: u64, + output: u64, + cache_read: u64, + cache_write: u64, + ) -> Option { + let cached = cache_read.min(input); + self.lane_cost(input.saturating_sub(cached), output, cached, cache_write) + } + + /// Upstream `CostUsageCustomPricing.costUSD(rates:...)`: every token lane is + /// billed on its own, and a lane with tokens but no rate leaves the cost + /// unknown. A missing rate is never filled from another source. + pub(super) fn lane_cost( + &self, + input: u64, + output: u64, + cache_read: u64, + cache_write: u64, + ) -> Option { + let mut total = 0.0; + if input > 0 { + total += input as f64 * self.input? / 1_000_000.0; + } + if output > 0 { + total += output as f64 * self.output? / 1_000_000.0; + } + if cache_read > 0 { + total += cache_read as f64 * self.cache_read? / 1_000_000.0; + } + if cache_write > 0 { + total += cache_write as f64 * self.cache_write? / 1_000_000.0; + } + total.is_finite().then_some(total) + } +} diff --git a/rust/src/spend_contract/merge.rs b/rust/src/spend_contract/merge.rs new file mode 100644 index 0000000000..ae25325767 --- /dev/null +++ b/rust/src/spend_contract/merge.rs @@ -0,0 +1,203 @@ +//! Activity histograms and the helpers that merge native and imported spend. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Datelike, Local, Timelike, Utc}; + +use super::{CostCoverageCounts, SpendActivityCell, SpendDailyPoint, SpendModelRow, SpendTokenMix}; +use crate::codex_workspaces::SessionUsage; + +/// Conversation counts per local (weekday, hour) cell. +#[derive(Default)] +pub(super) struct ActivityHistogram(pub(super) BTreeMap<(u8, u8), u32>); + +impl ActivityHistogram { + pub(super) fn add_local(&mut self, timestamp: DateTime) { + let local = timestamp.with_timezone(&Local); + #[allow( + clippy::cast_possible_truncation, + reason = "weekday (0-6) and hour (0-23) fit u8" + )] + let key = ( + local.weekday().num_days_from_monday() as u8, + local.hour() as u8, + ); + self.add(key, 1); + } + + pub(super) fn add(&mut self, key: (u8, u8), conversations: u32) { + let count = self.0.entry(key).or_insert(0); + *count = count.saturating_add(conversations); + } + + pub(super) fn into_cells(self) -> Vec { + self.0 + .into_iter() + .map(|((weekday, hour), conversations)| SpendActivityCell { + weekday, + hour, + conversations, + }) + .collect() + } +} + +pub(super) fn activity_from_sessions(sessions: &[SessionUsage]) -> Vec { + let mut activity = ActivityHistogram::default(); + for session in sessions { + if let Some(timestamp) = session.latest_activity.or(session.started_at) { + activity.add_local(timestamp); + } + } + activity.into_cells() +} +pub(super) fn sum_optional_cost(left: Option, right: Option) -> Option { + let valid = |value: f64| value.is_finite() && value >= 0.0; + match (left, right) { + (Some(left), Some(right)) if valid(left) && valid(right) => { + let total = left + right; + valid(total).then_some(total) + } + (Some(value), None) | (None, Some(value)) if valid(value) => Some(value), + (None, None) => None, + _ => None, + } +} + +pub(super) fn merge_coverage( + left: CostCoverageCounts, + right: &CostCoverageCounts, +) -> (CostCoverageCounts, bool) { + let priced = left.priced.checked_add(right.priced); + let unpriced = left.unpriced.checked_add(right.unpriced); + let unmetered = left.unmetered.checked_add(right.unmetered); + let estimated = left.estimated.checked_add(right.estimated); + let exact = + priced.is_some() && unpriced.is_some() && unmetered.is_some() && estimated.is_some(); + let merged = CostCoverageCounts { + priced: priced.unwrap_or(u32::MAX), + unpriced: unpriced.unwrap_or(u32::MAX), + unmetered: unmetered.unwrap_or(u32::MAX), + estimated: estimated.unwrap_or(u32::MAX), + }; + let exact = exact && merged.checked_total().is_some(); + (merged, exact) +} + +pub(super) fn merge_token_mix(mut left: SpendTokenMix, right: &SpendTokenMix) -> SpendTokenMix { + let mut overflowed_classes = left.overflowed_classes | right.overflowed_classes; + for (bit, merged, incoming) in [ + (0, &mut left.input_tokens, right.input_tokens), + (1, &mut left.output_tokens, right.output_tokens), + (2, &mut left.cache_read_tokens, right.cache_read_tokens), + ( + 3, + &mut left.cache_creation_tokens, + right.cache_creation_tokens, + ), + (4, &mut left.reasoning_tokens, right.reasoning_tokens), + ] { + *merged = merge_token_class(*merged, incoming, &mut overflowed_classes, 1 << bit); + } + left.overflowed_classes = overflowed_classes; + left +} + +pub(super) fn merge_token_class( + left: Option, + right: Option, + overflowed_classes: &mut u8, + bit: u8, +) -> Option { + if *overflowed_classes & bit != 0 { + return None; + } + match (left, right) { + (Some(left), Some(right)) => match left.checked_add(right) { + Some(total) => Some(total), + None => { + *overflowed_classes |= bit; + None + } + }, + (Some(left), None) => Some(left), + (None, Some(right)) => Some(right), + (None, None) => None, + } +} + +pub(super) fn add_optional(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => left.checked_add(right), + (Some(left), None) => Some(left), + (None, Some(right)) => Some(right), + (None, None) => None, + } +} + +pub(super) fn merge_models( + mut left: Vec, + right: &[SpendModelRow], +) -> Vec { + for incoming in right { + if let Some(existing) = left.iter_mut().find(|row| row.model == incoming.model) { + existing.cost_usd = sum_optional_cost(existing.cost_usd, incoming.cost_usd); + existing.input_tokens = existing.input_tokens.saturating_add(incoming.input_tokens); + existing.output_tokens = existing + .output_tokens + .saturating_add(incoming.output_tokens); + existing.cache_read_tokens = existing + .cache_read_tokens + .saturating_add(incoming.cache_read_tokens); + existing.total_tokens = existing.total_tokens.saturating_add(incoming.total_tokens); + existing.custom_pricing |= incoming.custom_pricing; + } else { + left.push(incoming.clone()); + } + } + left.sort_by(|a, b| { + b.cost_usd + .partial_cmp(&a.cost_usd) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.model.cmp(&b.model)) + }); + left +} + +pub(super) fn merge_daily( + left: Vec, + right: &[SpendDailyPoint], +) -> Vec { + let mut days: BTreeMap = left + .into_iter() + .map(|point| (point.day.clone(), point)) + .collect(); + for incoming in right { + let entry = days + .entry(incoming.day.clone()) + .or_insert_with(|| SpendDailyPoint { + day: incoming.day.clone(), + cost_usd: None, + total_tokens: None, + }); + entry.cost_usd = sum_optional_cost(entry.cost_usd, incoming.cost_usd); + entry.total_tokens = add_optional(entry.total_tokens, incoming.total_tokens); + } + days.into_values().collect() +} + +pub(super) fn merge_activity( + left: Vec, + right: &[SpendActivityCell], +) -> Vec { + // Collected, not added: a duplicate cell on the left keeps the last count. + let mut activity = ActivityHistogram( + left.into_iter() + .map(|cell| ((cell.weekday, cell.hour), cell.conversations)) + .collect(), + ); + for cell in right { + activity.add((cell.weekday, cell.hour), cell.conversations); + } + activity.into_cells() +} diff --git a/rust/src/spend_contract/tests.rs b/rust/src/spend_contract/tests.rs index 29c348fe75..8e0a1c6acd 100644 --- a/rust/src/spend_contract/tests.rs +++ b/rust/src/spend_contract/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::cost_scanner::ModelTokenCounts; pub(super) mod activity; diff --git a/rust/src/spend_contract/tests/activity.rs b/rust/src/spend_contract/tests/activity.rs index 158b68dd53..d6a7ab9e08 100644 --- a/rust/src/spend_contract/tests/activity.rs +++ b/rust/src/spend_contract/tests/activity.rs @@ -3,7 +3,8 @@ use super::*; use crate::codex_workspaces::{CostEstimate, UsageTotals}; -use chrono::DateTime; +use chrono::{DateTime, Datelike, Local, Timelike}; +use std::collections::BTreeMap; fn at(timestamp: &str) -> DateTime { DateTime::parse_from_rfc3339(timestamp) From fd9f7a4616bb835e3eac854718b113eb28af3202 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:21:45 +0700 Subject: [PATCH 38/41] Move the JSONL scanner cache load and save code into its own module --- rust/src/core/jsonl_scanner.rs | 305 +------------------- rust/src/core/jsonl_scanner/persistence.rs | 311 +++++++++++++++++++++ 2 files changed, 312 insertions(+), 304 deletions(-) create mode 100644 rust/src/core/jsonl_scanner/persistence.rs diff --git a/rust/src/core/jsonl_scanner.rs b/rust/src/core/jsonl_scanner.rs index 8e55c9e619..523a85aadd 100755 --- a/rust/src/core/jsonl_scanner.rs +++ b/rust/src/core/jsonl_scanner.rs @@ -24,53 +24,6 @@ pub struct CachedCostReadStatus { pub codex_scan_pause_reason: Option, } -#[derive(Deserialize, Default)] -struct CachedCostReadStatusProjection { - #[serde(default)] - codex_cache_schema_version: u32, - #[serde( - default, - rename = "days", - deserialize_with = "deserialize_nonempty_object" - )] - has_days: bool, - #[serde(default)] - previous_report: Option, - #[serde(default)] - codex_scan_pause_reason: Option, - #[serde(default)] - bucket_time_zone: Option, -} - -fn deserialize_nonempty_object<'de, D>(deserializer: D) -> Result -where - D: serde::Deserializer<'de>, -{ - use serde::de::{IgnoredAny, MapAccess, Visitor}; - - struct NonemptyObjectVisitor; - - impl<'de> Visitor<'de> for NonemptyObjectVisitor { - type Value = bool; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("a JSON object") - } - - fn visit_map(self, mut map: A) -> Result - where - A: MapAccess<'de>, - { - let mut nonempty = false; - while map.next_entry::()?.is_some() { - nonempty = true; - } - Ok(nonempty) - } - } - - deserializer.deserialize_map(NonemptyObjectVisitor) -} /// Default scanner-side refresh debounce (upstream CostUsageScanner). pub const DEFAULT_COST_SCAN_REFRESH_MIN_INTERVAL_SECS: u64 = 60; /// Default number of dirty Codex rollouts inspected in one refresh. @@ -611,6 +564,7 @@ pub use codex::priority::{ CodexPriorityTurnMetadata, CodexPriorityTurnsCursor, }; pub(crate) use codex::{CodexForkParseResume, CodexParseMode}; +mod persistence; mod save_skip; #[cfg(test)] #[path = "jsonl_scanner/tests/fixtures.rs"] @@ -630,92 +584,6 @@ impl JsonlScanner { options.should_skip_scan(cache.last_scan_unix_ms, now_unix_ms) } - /// Load cache from disk. - /// - /// Refuses to decode artifacts larger than the load cap - /// (`crate::core::CostUsageCacheBudget::MAX_LOAD_BYTES`); an oversized artifact is - /// cheaper to rebuild bounded than to decode in one shot, so the caller - /// gets a fresh empty cache instead (upstream 0.48.0 overshoot contract). - /// Only Codex persistence is bounded; other providers load unbounded. - pub fn load_cache(provider: ProviderId, cache_root: Option<&Path>) -> CostUsageCache { - let cache_path = Self::cache_path(provider, cache_root); - - if crate::core::is_bounded_provider(provider) { - // Artifacts are bounded by MAX_LOAD_BYTES (320 MiB), fitting usize on - // any supported target even before the budget comparison below. - #[allow( - clippy::cast_possible_truncation, - reason = "bounded artifacts fit usize on any supported target" - )] - let file_bytes = crate::core::artifact_file_size(&cache_path) as usize; - if file_bytes > crate::core::CostUsageCacheBudget::MAX_LOAD_BYTES { - return CostUsageCache::default(); - } - } - - if let Ok(contents) = fs::read_to_string(&cache_path) - && let Ok(mut cache) = serde_json::from_str::(&contents) - { - let stamp = CacheStamp::from_bytes(contents.as_bytes()); - cache.loaded_payload_stamp = CacheStamp::from_cache_payload(contents.as_bytes()); - if let Some(scan_unix_ms) = save_skip::recorded_scan_time(&cache_path, &stamp) { - cache.last_scan_unix_ms = scan_unix_ms; - } - if provider == ProviderId::Codex { - return codex::codex_cache_apply_load_policy(cache, stamp); - } - cache.loaded_stamp = Some(Some(stamp)); - return cache; - } - - // Track a missing or unreadable baseline separately from a manually - // constructed cache so a concurrent first writer can invalidate it. - CostUsageCache { - loaded_stamp: Some(Self::cache_stamp(&cache_path)), - ..CostUsageCache::default() - } - } - - /// Read only the cache metadata needed by presentation surfaces. - /// - /// v0.56.0 performance parity: skip raw per-file scanner state and day - /// payloads when callers only need stale/catch-up status. - pub fn load_cache_status( - provider: ProviderId, - cache_root: Option<&Path>, - ) -> CachedCostReadStatus { - let cache_path = Self::cache_path(provider, cache_root); - if crate::core::is_bounded_provider(provider) { - #[allow( - clippy::cast_possible_truncation, - reason = "bounded artifacts fit usize on any supported target" - )] - let file_bytes = crate::core::artifact_file_size(&cache_path) as usize; - if file_bytes > crate::core::CostUsageCacheBudget::MAX_LOAD_BYTES { - return CachedCostReadStatus::default(); - } - } - - let Ok(file) = File::open(cache_path) else { - return CachedCostReadStatus::default(); - }; - let Ok(projection) = - serde_json::from_reader::<_, CachedCostReadStatusProjection>(BufReader::new(file)) - else { - return CachedCostReadStatus::default(); - }; - if provider == ProviderId::Codex - && (!codex::codex_cache_schema_is_current(projection.codex_cache_schema_version) - || !codex::codex_cache_zone_is_current(projection.bucket_time_zone.as_deref())) - { - return CachedCostReadStatus::default(); - } - CachedCostReadStatus { - has_days: projection.has_days, - previous_report: projection.previous_report, - codex_scan_pause_reason: projection.codex_scan_pause_reason, - } - } pub(crate) fn cached_cost_report_from_days(cache: &CostUsageCache) -> CachedCostReport { Self::cached_cost_report_from_days_filtered(cache, None) } @@ -845,177 +713,6 @@ impl JsonlScanner { } } - /// Save cache to disk (temp sibling + copy into place). - /// - /// Before encoding, prunes the cache to the persistence budget so the - /// artifact stays small enough to decode in one shot (upstream 0.48.0 - /// #2637). Only Codex persistence is bounded; the overshoot contract lets - /// the encoded size exceed `MAX_FILE_BYTES` up to `MAX_LOAD_BYTES` - /// when protected (partially parsed) entries cannot be trimmed further. - pub fn save_cache(provider: ProviderId, cache: &mut CostUsageCache, cache_root: Option<&Path>) { - Self::save_cache_with_limit( - provider, - cache, - cache_root, - crate::core::CostUsageCacheBudget::MAX_LOAD_BYTES, - ); - } - - /// Save with an explicit post-encode refusal limit, injected by tests. - /// - /// Identical to `save_cache` except the post-encode oversize check uses - /// `max_load_bytes` rather than the production `MAX_LOAD_BYTES` const. - /// Production callers MUST use `save_cache`; this helper exists so the - /// refusal / stale-destination removal can be exercised without encoding a - /// ~320 MiB test artifact. - fn save_cache_with_limit( - provider: ProviderId, - cache: &mut CostUsageCache, - cache_root: Option<&Path>, - max_load_bytes: usize, - ) { - let cache_path = Self::cache_path(provider, cache_root); - - // A decoded baseline is only valid for the file contents that produced - // it. Refuse a stale writer before pruning or creating directories so a - // concurrent scan remains authoritative. - if let Some(expected) = cache.loaded_stamp.as_ref() - && Self::cache_stamp(&cache_path).as_ref() != expected.as_ref() - { - return; - } - if provider == ProviderId::Codex { - codex::codex_cache_stamp_schema_version(cache); - } - - let Some(parent) = cache_path.parent() else { - return; - }; - // Best-effort cache dir creation; a missing dir surfaces as the write error below. - let _dir_created = fs::create_dir_all(parent); - - if crate::core::is_bounded_provider(provider) { - // v0.55.1 #3051: snapshot the fully validated report BEFORE persistence - // pruning. If budget trimming creates a catch-up cycle, this is the - // established spend/tokens users should keep seeing until replacement - // history finishes, not a zero-cost reconstruction of the trimmed cache. - let established_report = cache - .previous_report - .clone() - .unwrap_or_else(|| Self::cached_cost_report_from_days(cache)); - let pruned = crate::core::prune_out_of_window_for_budget( - &mut cache.files, - &mut cache.days, - cache.scan_since_key.as_deref(), - cache.scan_until_key.as_deref(), - false, - ); - let estimate = crate::core::estimated_cache_bytes(&cache.files, &cache.days); - let trimmed = if estimate > crate::core::CostUsageCacheBudget::MAX_FILE_BYTES { - crate::core::trim_in_window_for_budget( - &mut cache.files, - &mut cache.days, - cache.scan_since_key.as_deref(), - cache.scan_until_key.as_deref(), - crate::core::CostUsageCacheBudget::MAX_FILE_BYTES, - ) - } else { - Vec::new() - }; - // A16 (upstream 0.48.0): when entries were trimmed for budget, the persisted - // artifact no longer covers the full window — set previous_report so the - // next refresh can signal catch-up is pending (and spend surfaces can show - // the last-validated snapshot during the rescan). - if (!pruned.is_empty() || !trimmed.is_empty()) && cache.previous_report.is_none() { - cache.previous_report = Some(established_report); - } - } - - let Ok(json) = serde_json::to_string(cache) else { - return; - }; - - // F19 (upstream 0.48.0): after bounded encode, if the artifact still - // exceeds MAX_LOAD_BYTES, refuse persistence. Also remove any existing - // destination artifact so a stale/oversized file cannot persist and - // trip the load-refusal path on the next scan (which would force an - // unnecessary full rebuild from a poisoned artifact). This is a - // one-shot refusal (not a persist/refuse/rebuild loop): the budget - // enforcement above already pruned and trimmed; if the result is still - // too large (e.g. a single protected entry exceeds the limit), the - // artifact is dropped and the next scan rebuilds from scratch. - if crate::core::is_bounded_provider(provider) - && crate::core::CostUsageCacheBudget::should_refuse_persistence( - json.len(), - max_load_bytes, - ) - { - // Best-effort removal; ignore errors (file may not exist). - let _cleared = fs::remove_file(&cache_path); - return; - } - if save_skip::skip_unchanged_save(&cache_path, cache, &json) { - return; - } - - let tmp_name = format!( - ".{}.{}-{}.tmp", - provider.cli_name(), - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0) - ); - let tmp_path = parent.join(tmp_name); - if fs::write(&tmp_path, json.as_bytes()).is_err() { - return; - } - // Recheck after encoding/pruning: another scan may have replaced the - // destination while this writer was preparing its payload. - if let Some(expected) = cache.loaded_stamp.as_ref() - && Self::cache_stamp(&cache_path).as_ref() != expected.as_ref() - { - let _removed_tmp = fs::remove_file(&tmp_path); - return; - } - // `copy` replaces an existing target on Windows; prefer it over rename. - let wrote = if fs::copy(&tmp_path, &cache_path).is_ok() { - true - } else { - // Fallback direct write when copy fails; the copy error already surfaced. - fs::write(&cache_path, json.as_bytes()).is_ok() - }; - if wrote { - cache.loaded_stamp = Some(Some(CacheStamp::from_bytes(json.as_bytes()))); - cache.loaded_payload_stamp = CacheStamp::from_cache_payload(json.as_bytes()); - } - // Best-effort temp cleanup (ignore errors — unique name avoids clashes). - let _truncated_tmp = fs::File::create(&tmp_path).and_then(|f| f.set_len(0)); - } - - /// Default on-disk cache root: `%LOCALAPPDATA%\CodexBar` (via `dirs::cache_dir`). - pub fn default_cache_root() -> Option { - dirs::cache_dir().map(|d| d.join("CodexBar")) - } - - fn cache_path(provider: ProviderId, cache_root: Option<&Path>) -> PathBuf { - let root = cache_root - .map(|p| p.to_path_buf()) - .or_else(Self::default_cache_root) - .unwrap_or_else(|| PathBuf::from(".")); - - // Mirror upstream layout: {cacheRoot}/cost-usage/{provider}-v1.json - root.join("cost-usage") - .join(format!("{}-v1.json", provider.cli_name())) - } - - fn cache_stamp(cache_path: &Path) -> Option { - fs::read(cache_path) - .ok() - .map(|contents| CacheStamp::from_bytes(&contents)) - } - /// Whether `cache` covers the requested day window (for debounce short-circuit). pub fn cache_covers_range(cache: &CostUsageCache, range: &CostUsageDayRange) -> bool { match (&cache.scan_since_key, &cache.scan_until_key) { diff --git a/rust/src/core/jsonl_scanner/persistence.rs b/rust/src/core/jsonl_scanner/persistence.rs new file mode 100644 index 0000000000..4c827a3772 --- /dev/null +++ b/rust/src/core/jsonl_scanner/persistence.rs @@ -0,0 +1,311 @@ +//! Loading and saving the per-provider cost cache artifact. + +use super::*; + +#[derive(Deserialize, Default)] +struct CachedCostReadStatusProjection { + #[serde(default)] + codex_cache_schema_version: u32, + #[serde( + default, + rename = "days", + deserialize_with = "deserialize_nonempty_object" + )] + has_days: bool, + #[serde(default)] + previous_report: Option, + #[serde(default)] + codex_scan_pause_reason: Option, + #[serde(default)] + bucket_time_zone: Option, +} + +fn deserialize_nonempty_object<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + use serde::de::{IgnoredAny, MapAccess, Visitor}; + + struct NonemptyObjectVisitor; + + impl<'de> Visitor<'de> for NonemptyObjectVisitor { + type Value = bool; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a JSON object") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut nonempty = false; + while map.next_entry::()?.is_some() { + nonempty = true; + } + Ok(nonempty) + } + } + + deserializer.deserialize_map(NonemptyObjectVisitor) +} + +impl JsonlScanner { + /// Load cache from disk. + /// + /// Refuses to decode artifacts larger than the load cap + /// (`crate::core::CostUsageCacheBudget::MAX_LOAD_BYTES`); an oversized artifact is + /// cheaper to rebuild bounded than to decode in one shot, so the caller + /// gets a fresh empty cache instead (upstream 0.48.0 overshoot contract). + /// Only Codex persistence is bounded; other providers load unbounded. + pub fn load_cache(provider: ProviderId, cache_root: Option<&Path>) -> CostUsageCache { + let cache_path = Self::cache_path(provider, cache_root); + + if crate::core::is_bounded_provider(provider) { + // Artifacts are bounded by MAX_LOAD_BYTES (320 MiB), fitting usize on + // any supported target even before the budget comparison below. + #[allow( + clippy::cast_possible_truncation, + reason = "bounded artifacts fit usize on any supported target" + )] + let file_bytes = crate::core::artifact_file_size(&cache_path) as usize; + if file_bytes > crate::core::CostUsageCacheBudget::MAX_LOAD_BYTES { + return CostUsageCache::default(); + } + } + + if let Ok(contents) = fs::read_to_string(&cache_path) + && let Ok(mut cache) = serde_json::from_str::(&contents) + { + let stamp = CacheStamp::from_bytes(contents.as_bytes()); + cache.loaded_payload_stamp = CacheStamp::from_cache_payload(contents.as_bytes()); + if let Some(scan_unix_ms) = save_skip::recorded_scan_time(&cache_path, &stamp) { + cache.last_scan_unix_ms = scan_unix_ms; + } + if provider == ProviderId::Codex { + return codex::codex_cache_apply_load_policy(cache, stamp); + } + cache.loaded_stamp = Some(Some(stamp)); + return cache; + } + + // Track a missing or unreadable baseline separately from a manually + // constructed cache so a concurrent first writer can invalidate it. + CostUsageCache { + loaded_stamp: Some(Self::cache_stamp(&cache_path)), + ..CostUsageCache::default() + } + } + + /// Read only the cache metadata needed by presentation surfaces. + /// + /// v0.56.0 performance parity: skip raw per-file scanner state and day + /// payloads when callers only need stale/catch-up status. + pub fn load_cache_status( + provider: ProviderId, + cache_root: Option<&Path>, + ) -> CachedCostReadStatus { + let cache_path = Self::cache_path(provider, cache_root); + if crate::core::is_bounded_provider(provider) { + #[allow( + clippy::cast_possible_truncation, + reason = "bounded artifacts fit usize on any supported target" + )] + let file_bytes = crate::core::artifact_file_size(&cache_path) as usize; + if file_bytes > crate::core::CostUsageCacheBudget::MAX_LOAD_BYTES { + return CachedCostReadStatus::default(); + } + } + + let Ok(file) = File::open(cache_path) else { + return CachedCostReadStatus::default(); + }; + let Ok(projection) = + serde_json::from_reader::<_, CachedCostReadStatusProjection>(BufReader::new(file)) + else { + return CachedCostReadStatus::default(); + }; + if provider == ProviderId::Codex + && (!codex::codex_cache_schema_is_current(projection.codex_cache_schema_version) + || !codex::codex_cache_zone_is_current(projection.bucket_time_zone.as_deref())) + { + return CachedCostReadStatus::default(); + } + CachedCostReadStatus { + has_days: projection.has_days, + previous_report: projection.previous_report, + codex_scan_pause_reason: projection.codex_scan_pause_reason, + } + } + + /// Save cache to disk (temp sibling + copy into place). + /// + /// Before encoding, prunes the cache to the persistence budget so the + /// artifact stays small enough to decode in one shot (upstream 0.48.0 + /// #2637). Only Codex persistence is bounded; the overshoot contract lets + /// the encoded size exceed `MAX_FILE_BYTES` up to `MAX_LOAD_BYTES` + /// when protected (partially parsed) entries cannot be trimmed further. + pub fn save_cache(provider: ProviderId, cache: &mut CostUsageCache, cache_root: Option<&Path>) { + Self::save_cache_with_limit( + provider, + cache, + cache_root, + crate::core::CostUsageCacheBudget::MAX_LOAD_BYTES, + ); + } + + /// Save with an explicit post-encode refusal limit, injected by tests. + /// + /// Identical to `save_cache` except the post-encode oversize check uses + /// `max_load_bytes` rather than the production `MAX_LOAD_BYTES` const. + /// Production callers MUST use `save_cache`; this helper exists so the + /// refusal / stale-destination removal can be exercised without encoding a + /// ~320 MiB test artifact. + pub(super) fn save_cache_with_limit( + provider: ProviderId, + cache: &mut CostUsageCache, + cache_root: Option<&Path>, + max_load_bytes: usize, + ) { + let cache_path = Self::cache_path(provider, cache_root); + + // A decoded baseline is only valid for the file contents that produced + // it. Refuse a stale writer before pruning or creating directories so a + // concurrent scan remains authoritative. + if let Some(expected) = cache.loaded_stamp.as_ref() + && Self::cache_stamp(&cache_path).as_ref() != expected.as_ref() + { + return; + } + if provider == ProviderId::Codex { + codex::codex_cache_stamp_schema_version(cache); + } + + let Some(parent) = cache_path.parent() else { + return; + }; + // Best-effort cache dir creation; a missing dir surfaces as the write error below. + let _dir_created = fs::create_dir_all(parent); + + if crate::core::is_bounded_provider(provider) { + // v0.55.1 #3051: snapshot the fully validated report BEFORE persistence + // pruning. If budget trimming creates a catch-up cycle, this is the + // established spend/tokens users should keep seeing until replacement + // history finishes, not a zero-cost reconstruction of the trimmed cache. + let established_report = cache + .previous_report + .clone() + .unwrap_or_else(|| Self::cached_cost_report_from_days(cache)); + let pruned = crate::core::prune_out_of_window_for_budget( + &mut cache.files, + &mut cache.days, + cache.scan_since_key.as_deref(), + cache.scan_until_key.as_deref(), + false, + ); + let estimate = crate::core::estimated_cache_bytes(&cache.files, &cache.days); + let trimmed = if estimate > crate::core::CostUsageCacheBudget::MAX_FILE_BYTES { + crate::core::trim_in_window_for_budget( + &mut cache.files, + &mut cache.days, + cache.scan_since_key.as_deref(), + cache.scan_until_key.as_deref(), + crate::core::CostUsageCacheBudget::MAX_FILE_BYTES, + ) + } else { + Vec::new() + }; + // A16 (upstream 0.48.0): when entries were trimmed for budget, the persisted + // artifact no longer covers the full window — set previous_report so the + // next refresh can signal catch-up is pending (and spend surfaces can show + // the last-validated snapshot during the rescan). + if (!pruned.is_empty() || !trimmed.is_empty()) && cache.previous_report.is_none() { + cache.previous_report = Some(established_report); + } + } + + let Ok(json) = serde_json::to_string(cache) else { + return; + }; + + // F19 (upstream 0.48.0): after bounded encode, if the artifact still + // exceeds MAX_LOAD_BYTES, refuse persistence. Also remove any existing + // destination artifact so a stale/oversized file cannot persist and + // trip the load-refusal path on the next scan (which would force an + // unnecessary full rebuild from a poisoned artifact). This is a + // one-shot refusal (not a persist/refuse/rebuild loop): the budget + // enforcement above already pruned and trimmed; if the result is still + // too large (e.g. a single protected entry exceeds the limit), the + // artifact is dropped and the next scan rebuilds from scratch. + if crate::core::is_bounded_provider(provider) + && crate::core::CostUsageCacheBudget::should_refuse_persistence( + json.len(), + max_load_bytes, + ) + { + // Best-effort removal; ignore errors (file may not exist). + let _cleared = fs::remove_file(&cache_path); + return; + } + if save_skip::skip_unchanged_save(&cache_path, cache, &json) { + return; + } + + let tmp_name = format!( + ".{}.{}-{}.tmp", + provider.cli_name(), + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + ); + let tmp_path = parent.join(tmp_name); + if fs::write(&tmp_path, json.as_bytes()).is_err() { + return; + } + // Recheck after encoding/pruning: another scan may have replaced the + // destination while this writer was preparing its payload. + if let Some(expected) = cache.loaded_stamp.as_ref() + && Self::cache_stamp(&cache_path).as_ref() != expected.as_ref() + { + let _removed_tmp = fs::remove_file(&tmp_path); + return; + } + // `copy` replaces an existing target on Windows; prefer it over rename. + let wrote = if fs::copy(&tmp_path, &cache_path).is_ok() { + true + } else { + // Fallback direct write when copy fails; the copy error already surfaced. + fs::write(&cache_path, json.as_bytes()).is_ok() + }; + if wrote { + cache.loaded_stamp = Some(Some(CacheStamp::from_bytes(json.as_bytes()))); + cache.loaded_payload_stamp = CacheStamp::from_cache_payload(json.as_bytes()); + } + // Best-effort temp cleanup (ignore errors — unique name avoids clashes). + let _truncated_tmp = fs::File::create(&tmp_path).and_then(|f| f.set_len(0)); + } + + /// Default on-disk cache root: `%LOCALAPPDATA%\CodexBar` (via `dirs::cache_dir`). + pub fn default_cache_root() -> Option { + dirs::cache_dir().map(|d| d.join("CodexBar")) + } + + pub(super) fn cache_path(provider: ProviderId, cache_root: Option<&Path>) -> PathBuf { + let root = cache_root + .map(|p| p.to_path_buf()) + .or_else(Self::default_cache_root) + .unwrap_or_else(|| PathBuf::from(".")); + + // Mirror upstream layout: {cacheRoot}/cost-usage/{provider}-v1.json + root.join("cost-usage") + .join(format!("{}-v1.json", provider.cli_name())) + } + + pub(super) fn cache_stamp(cache_path: &Path) -> Option { + fs::read(cache_path) + .ok() + .map(|contents| CacheStamp::from_bytes(&contents)) + } +} From 7a24de447fae20c53bd9586841ea3243683217e1 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:28:12 +0700 Subject: [PATCH 39/41] Move the daily cost and token history out of the cost scanner --- rust/src/cost_scanner.rs | 260 +----------------------- rust/src/cost_scanner/daily_history.rs | 267 +++++++++++++++++++++++++ rust/src/cost_scanner/tests.rs | 3 +- 3 files changed, 276 insertions(+), 254 deletions(-) create mode 100644 rust/src/cost_scanner/daily_history.rs diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index ec6af23d9e..24932c60ed 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -8,7 +8,7 @@ //! debounce (default 60s; `app_driven` forces a fresh inspection), and checks //! cancel flags between files. -use chrono::{DateTime, Duration, NaiveDate, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; use serde::Deserialize; use serde_json::Value; use std::collections::{HashMap, HashSet}; @@ -40,6 +40,7 @@ mod claude_pricing; mod claude_roots; mod claude_usage; mod codex; +mod daily_history; mod read_receipt; mod stats; mod today; @@ -51,6 +52,11 @@ use claude_usage::{ ClaudeUsageDedupKey, claude_usage_dedup_key, session_id_from_entries, should_count_claude_record, }; +pub use daily_history::{ + DailyCostAndIncomplete, get_daily_cost_and_incomplete_history, get_daily_cost_history, + get_daily_token_history, has_cost_usage_sources, +}; +use daily_history::{add_claude_record_to_daily_tokens, day_slots, sorted_days}; pub use read_receipt::CodexScanReadReceipt; pub use stats::CostScanStats; pub use today::TodayUsage; @@ -1260,255 +1266,3 @@ fn zero_fill_uninitialized_claude_daily_costs( } } } - -/// Check if any cost usage sources are available -#[allow( - dead_code, - reason = "utility probe for cost-usage availability; not yet wired into all call sites" -)] -pub fn has_cost_usage_sources() -> bool { - let scanner = CostScanner::new(1); - scanner - .get_codex_sessions_dirs() - .iter() - .any(|dir| dir.exists()) - || scanner.claude_projects_roots().has_possible_roots() - || crate::pi_session_cost::pi_compatible_session_roots(crate::pi_session_cost::scan_home()) - .iter() - .any(|dir| dir.exists()) -} - -/// One `init` slot per day key, from `end` back over `days` days. -fn day_slots(end: NaiveDate, days: u32, init: T) -> HashMap { - (0..days) - .map(|days_ago| { - let date = end - Duration::days(i64::from(days_ago)); - (today::day_key(date), init.clone()) - }) - .collect() -} - -fn sorted_days(days: HashMap) -> Vec<(String, T)> { - let mut days: Vec<_> = days.into_iter().collect(); - days.sort_by(|left, right| left.0.cmp(&right.0)); - days -} - -/// One packed Codex cache day summarized on its own, with its cost. -fn codex_day_summary( - day_key: &str, - models: &HashMap>, -) -> Option<(CostSummary, f64)> { - let day = CostUsageDayRange::parse_day_key(day_key)?; - let one_day = HashMap::from([(day_key.to_string(), models.clone())]); - let mut summary = CostSummary::default(); - let (cost, _) = - add_codex_days_map_to_summary(&mut summary, &one_day, &CostUsageDayRange::new(day, day)); - Some((summary, cost)) -} - -/// Get daily cost history for the last N days -/// Returns calendar-preserving daily costs sorted by date. `None` means the day -/// is unscanned or contains unpriced Codex usage; `Some(0)` is a known zero. -pub fn get_daily_cost_history(provider: &str, days: u32) -> Vec<(String, Option)> { - get_daily_cost_and_incomplete_history(provider, days).0 -} - -/// Daily cost series plus per-day incomplete request counts. -pub type DailyCostAndIncomplete = (Vec<(String, Option)>, Vec<(String, u32)>); - -/// Daily cost history plus, for Claude, the per-day count of incomplete proxy -/// requests (upstream 0.60.5 #3688). Days with no incomplete request are -/// absent from the second vector; it is empty for every other provider. -pub fn get_daily_cost_and_incomplete_history(provider: &str, days: u32) -> DailyCostAndIncomplete { - let mut daily_incomplete = Vec::new(); - let scanner = CostScanner::new(days); - let today = cost_bucket_zone().date(Utc::now()); - let mut daily_costs = day_slots( - today, - days, - (provider != "codex" && provider != "claude" && provider != "pi").then_some(0.0), - ); - - match provider { - "codex" => { - // Warm/refresh the disk cache, then price from packed days. v0.56.1 - // preserves every calendar slot and distinguishes covered zero from - // unscanned/unpriced history. - let (_summary, _stats, cache) = scanner.scan_codex_detailed_with_cache(None); - if cache.previous_report.is_none() && !cache.codex_scan_incomplete { - for (day_key, slot) in &mut daily_costs { - if cache - .scan_since_key - .as_deref() - .is_some_and(|since| day_key.as_str() >= since) - && cache - .scan_until_key - .as_deref() - .is_some_and(|until| day_key.as_str() <= until) - { - *slot = Some(0.0); - } - } - } - for (day_key, models) in &cache.days { - let Some(slot) = daily_costs.get_mut(day_key) else { - continue; - }; - let Some((day, cost)) = codex_day_summary(day_key, models) else { - continue; - }; - *slot = (!day.model_pricing_completeness.is_partial()).then_some(cost); - } - } - "claude" => { - // Real per-day breakdown: walk the project logs once, - // de-duplicating records across files. - let cutoff = Utc::now() - Duration::days(days as i64); - let mut unknown_cost_dates = HashSet::new(); - let walk = scanner.walk_claude_records(&cutoff, None, |record| { - add_claude_record_to_daily_costs(&mut daily_costs, &mut unknown_cost_dates, record) - }); - daily_incomplete = walk.incomplete.resolve(&walk.completed_keys).daily_sorted(); - if walk.roots_present && walk.scan.is_complete() { - zero_fill_uninitialized_claude_daily_costs(&mut daily_costs, &unknown_cost_dates); - } - } - "opencodego" => { - // Per-day cost from the local OpenCode SQLite reader (upstream #2649). - // Rows are grouped by local calendar day to match Codex/Claude keying. - for (day_key, cost) in opencodego_local::daily_cost_series(Utc::now(), days) { - if let Some(slot) = daily_costs.get_mut(&day_key) { - *slot = Some(slot.unwrap_or(0.0) + cost); - } - } - } - "pi" => { - let scan = crate::pi_session_cost::scan_pi_daily(days, None); - for (day_key, cost) in &scan.costs { - if let Some(slot) = daily_costs.get_mut(day_key) { - *slot = (!scan.unpriced_days.contains(day_key)).then_some(*cost); - } - } - if scan.history_coverage_established { - for (day_key, slot) in &mut daily_costs { - if slot.is_none() && !scan.unpriced_days.contains(day_key) { - *slot = Some(0.0); - } - } - } - } - _ => {} - } - - (sorted_days(daily_costs), daily_incomplete) -} - -/// Daily token totals (provider-aware, see [`cache_is_separate_from_input`]) -/// for the Tokens chart mode, plus -/// whether local history looks incomplete at the old edge of the window -/// (Codex backfill still in progress → the chart shows a "Refreshing" -/// marker; upstream 0.50.0 #2930). -pub fn get_daily_token_history(provider: &str, days: u32) -> (Vec<(String, u64)>, bool) { - let scanner = CostScanner::new(days); - let today = cost_bucket_zone().date(Utc::now()); - let mut daily_tokens = day_slots(today, days, 0u64); - let mut covered_days: HashSet = HashSet::new(); - - match provider { - "codex" => { - // Warm/refresh the disk cache, then read exact local token totals - // from packed days through the same summary path the cost chart - // uses. - let (_summary, _stats, cache) = scanner.scan_codex_detailed_with_cache(None); - for (day_key, models) in &cache.days { - let Some(slot) = daily_tokens.get_mut(day_key) else { - continue; - }; - let Some((day, _)) = codex_day_summary(day_key, models) else { - continue; - }; - *slot = day.total_tokens_for_provider("codex"); - covered_days.insert(day_key.clone()); - } - } - "claude" => { - // Per-day token breakdown from the same de-duplicated record walk - // as the cost chart. Only a complete valid walk establishes - // authoritative coverage of the requested history window. - let cutoff = Utc::now() - Duration::days(days as i64); - let walk = scanner.walk_claude_records(&cutoff, None, |record| { - add_claude_record_to_daily_tokens(&mut daily_tokens, record) - }); - if walk.roots_present { - mark_claude_daily_token_coverage(&mut covered_days, &daily_tokens, walk.scan); - } - } - "pi" => { - let scan = crate::pi_session_cost::scan_pi_daily(days, None); - for (day_key, tokens) in scan.tokens { - if let Some(slot) = daily_tokens.get_mut(&day_key) { - *slot = tokens; - } - } - if scan.history_coverage_established { - covered_days.extend(daily_tokens.keys().cloned()); - } - } - _ => {} - } - - let result = sorted_days(daily_tokens); - - let incomplete = if matches!(provider, "claude" | "pi") { - // A complete filesystem scan covers the requested window even when - // the roots contain no sessions; failed scans leave coverage empty. - covered_days.is_empty() - } else { - // Codex catch-up may not have reached the oldest quarter of the window. - provider == "codex" - && !covered_days.is_empty() - && covered_days.len() < days as usize - && result[..(result.len() / 4).max(1)] - .iter() - .any(|(date, _)| !covered_days.contains(date)) - }; - - (result, incomplete) -} - -fn add_claude_record_to_daily_tokens( - daily_tokens: &mut HashMap, - record: &ClaudeUsageRecord, -) -> bool { - let Some(timestamp) = record.timestamp else { - return true; - }; - let date_str = today::day_key(cost_bucket_zone().date(timestamp)); - if let Some(slot) = daily_tokens.get_mut(&date_str) { - // Claude reports cache reads and writes separately from input, so the - // day total includes them (same rule as the window and model totals). - let Some(tokens) = record - .input - .checked_add(record.output) - .and_then(|tokens| tokens.checked_add(record.cache_read)) - .and_then(|tokens| tokens.checked_add(record.cache_create)) - else { - return false; - }; - return checked_add_assign(slot, tokens); - } - true -} - -fn mark_claude_daily_token_coverage( - covered_days: &mut HashSet, - daily_tokens: &HashMap, - scan_result: ClaudeFileScanResult, -) { - if scan_result.is_complete() { - covered_days.extend(daily_tokens.keys().cloned()); - } else { - covered_days.clear(); - } -} diff --git a/rust/src/cost_scanner/daily_history.rs b/rust/src/cost_scanner/daily_history.rs new file mode 100644 index 0000000000..80958e73e5 --- /dev/null +++ b/rust/src/cost_scanner/daily_history.rs @@ -0,0 +1,267 @@ +//! Per-day cost and token series for the charts, plus the day-slot helpers they share. + +use std::collections::{HashMap, HashSet}; + +use chrono::{Duration, NaiveDate, Utc}; + +use super::{ + ClaudeFileScanResult, ClaudeUsageRecord, CostScanner, CostSummary, + add_claude_record_to_daily_costs, checked_add_assign, today, + zero_fill_uninitialized_claude_daily_costs, +}; +use crate::codex_costs::add_codex_days_map_to_summary; +use crate::core::CostUsageDayRange; +use crate::cost_reporting_period::cost_bucket_zone; +use crate::providers::opencodego::local as opencodego_local; + +/// Check if any cost usage sources are available +#[allow( + dead_code, + reason = "utility probe for cost-usage availability; not yet wired into all call sites" +)] +pub fn has_cost_usage_sources() -> bool { + let scanner = CostScanner::new(1); + scanner + .get_codex_sessions_dirs() + .iter() + .any(|dir| dir.exists()) + || scanner.claude_projects_roots().has_possible_roots() + || crate::pi_session_cost::pi_compatible_session_roots(crate::pi_session_cost::scan_home()) + .iter() + .any(|dir| dir.exists()) +} + +/// One `init` slot per day key, from `end` back over `days` days. +pub(super) fn day_slots(end: NaiveDate, days: u32, init: T) -> HashMap { + (0..days) + .map(|days_ago| { + let date = end - Duration::days(i64::from(days_ago)); + (today::day_key(date), init.clone()) + }) + .collect() +} + +pub(super) fn sorted_days(days: HashMap) -> Vec<(String, T)> { + let mut days: Vec<_> = days.into_iter().collect(); + days.sort_by(|left, right| left.0.cmp(&right.0)); + days +} + +/// One packed Codex cache day summarized on its own, with its cost. +pub(super) fn codex_day_summary( + day_key: &str, + models: &HashMap>, +) -> Option<(CostSummary, f64)> { + let day = CostUsageDayRange::parse_day_key(day_key)?; + let one_day = HashMap::from([(day_key.to_string(), models.clone())]); + let mut summary = CostSummary::default(); + let (cost, _) = + add_codex_days_map_to_summary(&mut summary, &one_day, &CostUsageDayRange::new(day, day)); + Some((summary, cost)) +} + +/// Get daily cost history for the last N days +/// Returns calendar-preserving daily costs sorted by date. `None` means the day +/// is unscanned or contains unpriced Codex usage; `Some(0)` is a known zero. +pub fn get_daily_cost_history(provider: &str, days: u32) -> Vec<(String, Option)> { + get_daily_cost_and_incomplete_history(provider, days).0 +} + +/// Daily cost series plus per-day incomplete request counts. +pub type DailyCostAndIncomplete = (Vec<(String, Option)>, Vec<(String, u32)>); + +/// Daily cost history plus, for Claude, the per-day count of incomplete proxy +/// requests (upstream 0.60.5 #3688). Days with no incomplete request are +/// absent from the second vector; it is empty for every other provider. +pub fn get_daily_cost_and_incomplete_history(provider: &str, days: u32) -> DailyCostAndIncomplete { + let mut daily_incomplete = Vec::new(); + let scanner = CostScanner::new(days); + let today = cost_bucket_zone().date(Utc::now()); + let mut daily_costs = day_slots( + today, + days, + (provider != "codex" && provider != "claude" && provider != "pi").then_some(0.0), + ); + + match provider { + "codex" => { + // Warm/refresh the disk cache, then price from packed days. v0.56.1 + // preserves every calendar slot and distinguishes covered zero from + // unscanned/unpriced history. + let (_summary, _stats, cache) = scanner.scan_codex_detailed_with_cache(None); + if cache.previous_report.is_none() && !cache.codex_scan_incomplete { + for (day_key, slot) in &mut daily_costs { + if cache + .scan_since_key + .as_deref() + .is_some_and(|since| day_key.as_str() >= since) + && cache + .scan_until_key + .as_deref() + .is_some_and(|until| day_key.as_str() <= until) + { + *slot = Some(0.0); + } + } + } + for (day_key, models) in &cache.days { + let Some(slot) = daily_costs.get_mut(day_key) else { + continue; + }; + let Some((day, cost)) = codex_day_summary(day_key, models) else { + continue; + }; + *slot = (!day.model_pricing_completeness.is_partial()).then_some(cost); + } + } + "claude" => { + // Real per-day breakdown: walk the project logs once, + // de-duplicating records across files. + let cutoff = Utc::now() - Duration::days(days as i64); + let mut unknown_cost_dates = HashSet::new(); + let walk = scanner.walk_claude_records(&cutoff, None, |record| { + add_claude_record_to_daily_costs(&mut daily_costs, &mut unknown_cost_dates, record) + }); + daily_incomplete = walk.incomplete.resolve(&walk.completed_keys).daily_sorted(); + if walk.roots_present && walk.scan.is_complete() { + zero_fill_uninitialized_claude_daily_costs(&mut daily_costs, &unknown_cost_dates); + } + } + "opencodego" => { + // Per-day cost from the local OpenCode SQLite reader (upstream #2649). + // Rows are grouped by local calendar day to match Codex/Claude keying. + for (day_key, cost) in opencodego_local::daily_cost_series(Utc::now(), days) { + if let Some(slot) = daily_costs.get_mut(&day_key) { + *slot = Some(slot.unwrap_or(0.0) + cost); + } + } + } + "pi" => { + let scan = crate::pi_session_cost::scan_pi_daily(days, None); + for (day_key, cost) in &scan.costs { + if let Some(slot) = daily_costs.get_mut(day_key) { + *slot = (!scan.unpriced_days.contains(day_key)).then_some(*cost); + } + } + if scan.history_coverage_established { + for (day_key, slot) in &mut daily_costs { + if slot.is_none() && !scan.unpriced_days.contains(day_key) { + *slot = Some(0.0); + } + } + } + } + _ => {} + } + + (sorted_days(daily_costs), daily_incomplete) +} + +/// Daily token totals (provider-aware, see [`cache_is_separate_from_input`]) +/// for the Tokens chart mode, plus +/// whether local history looks incomplete at the old edge of the window +/// (Codex backfill still in progress → the chart shows a "Refreshing" +/// marker; upstream 0.50.0 #2930). +pub fn get_daily_token_history(provider: &str, days: u32) -> (Vec<(String, u64)>, bool) { + let scanner = CostScanner::new(days); + let today = cost_bucket_zone().date(Utc::now()); + let mut daily_tokens = day_slots(today, days, 0u64); + let mut covered_days: HashSet = HashSet::new(); + + match provider { + "codex" => { + // Warm/refresh the disk cache, then read exact local token totals + // from packed days through the same summary path the cost chart + // uses. + let (_summary, _stats, cache) = scanner.scan_codex_detailed_with_cache(None); + for (day_key, models) in &cache.days { + let Some(slot) = daily_tokens.get_mut(day_key) else { + continue; + }; + let Some((day, _)) = codex_day_summary(day_key, models) else { + continue; + }; + *slot = day.total_tokens_for_provider("codex"); + covered_days.insert(day_key.clone()); + } + } + "claude" => { + // Per-day token breakdown from the same de-duplicated record walk + // as the cost chart. Only a complete valid walk establishes + // authoritative coverage of the requested history window. + let cutoff = Utc::now() - Duration::days(days as i64); + let walk = scanner.walk_claude_records(&cutoff, None, |record| { + add_claude_record_to_daily_tokens(&mut daily_tokens, record) + }); + if walk.roots_present { + mark_claude_daily_token_coverage(&mut covered_days, &daily_tokens, walk.scan); + } + } + "pi" => { + let scan = crate::pi_session_cost::scan_pi_daily(days, None); + for (day_key, tokens) in scan.tokens { + if let Some(slot) = daily_tokens.get_mut(&day_key) { + *slot = tokens; + } + } + if scan.history_coverage_established { + covered_days.extend(daily_tokens.keys().cloned()); + } + } + _ => {} + } + + let result = sorted_days(daily_tokens); + + let incomplete = if matches!(provider, "claude" | "pi") { + // A complete filesystem scan covers the requested window even when + // the roots contain no sessions; failed scans leave coverage empty. + covered_days.is_empty() + } else { + // Codex catch-up may not have reached the oldest quarter of the window. + provider == "codex" + && !covered_days.is_empty() + && covered_days.len() < days as usize + && result[..(result.len() / 4).max(1)] + .iter() + .any(|(date, _)| !covered_days.contains(date)) + }; + + (result, incomplete) +} + +pub(super) fn add_claude_record_to_daily_tokens( + daily_tokens: &mut HashMap, + record: &ClaudeUsageRecord, +) -> bool { + let Some(timestamp) = record.timestamp else { + return true; + }; + let date_str = today::day_key(cost_bucket_zone().date(timestamp)); + if let Some(slot) = daily_tokens.get_mut(&date_str) { + // Claude reports cache reads and writes separately from input, so the + // day total includes them (same rule as the window and model totals). + let Some(tokens) = record + .input + .checked_add(record.output) + .and_then(|tokens| tokens.checked_add(record.cache_read)) + .and_then(|tokens| tokens.checked_add(record.cache_create)) + else { + return false; + }; + return checked_add_assign(slot, tokens); + } + true +} + +pub(super) fn mark_claude_daily_token_coverage( + covered_days: &mut HashSet, + daily_tokens: &HashMap, + scan_result: ClaudeFileScanResult, +) { + if scan_result.is_complete() { + covered_days.extend(daily_tokens.keys().cloned()); + } else { + covered_days.clear(); + } +} diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index 1915000848..f57d2ae5ff 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -2,8 +2,9 @@ use super::*; use crate::codex_costs::codex_period_start; use crate::core::test_fixtures::test_file_usage; use crate::core::{CodexSessionLineage, CostUsagePricing}; -use chrono::{FixedOffset, Local, NaiveTime, TimeZone}; +use chrono::{Duration, FixedOffset, Local, NaiveTime, TimeZone}; use claude_pricing::FALLBACK_CLAUDE_MODEL; +use daily_history::mark_claude_daily_token_coverage; use std::io::Write; use support::{ app_scanner, cached_file, cached_input_total, cached_usage_with_packed, From 5e69dc884101edc38fde1450238bcbb9f430988d Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:35:22 +0700 Subject: [PATCH 40/41] Move the Claude transcript file scan out of the cost scanner --- rust/src/cost_scanner.rs | 316 +------------------------- rust/src/cost_scanner/claude_scan.rs | 327 +++++++++++++++++++++++++++ rust/src/cost_scanner/tests.rs | 4 + 3 files changed, 340 insertions(+), 307 deletions(-) create mode 100644 rust/src/cost_scanner/claude_scan.rs diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index 24932c60ed..c84cee70d8 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -12,8 +12,7 @@ use chrono::{DateTime, NaiveDate, Utc}; use serde::Deserialize; use serde_json::Value; use std::collections::{HashMap, HashSet}; -use std::fs::{self, File}; -use std::io::{BufRead, BufReader}; +use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -29,15 +28,14 @@ use crate::core::{ CachedCostReport, CodexScanPauseReason, CostScanOptions, CostUsageCache, CostUsageDayRange, CostUsageFileUsage, JsonlScanner, ProviderId, }; -use crate::cost_reporting_period::{CostReportingPeriod, MAX_ROLLING_DAYS, cost_bucket_zone}; -use crate::providers::claude::quota_history::{ - ClaudeHistoryAttribution, ClaudeQuotaDedupKey, ClaudeQuotaHistoryRecord, -}; +use crate::cost_reporting_period::{CostReportingPeriod, MAX_ROLLING_DAYS}; +use crate::providers::claude::quota_history::ClaudeQuotaHistoryRecord; use crate::providers::opencodego::local as opencodego_local; use crate::settings::Settings; mod claude_incomplete; mod claude_pricing; mod claude_roots; +mod claude_scan; mod claude_usage; mod codex; mod daily_history; @@ -48,10 +46,12 @@ mod window; pub use claude_incomplete::ClaudeIncompleteReport; use claude_incomplete::ClaudeIncompleteTracker; use claude_pricing::ClaudeScanPricingResolver; -use claude_usage::{ - ClaudeUsageDedupKey, claude_usage_dedup_key, session_id_from_entries, - should_count_claude_record, +use claude_scan::{ + add_claude_record_to_daily_costs, add_claude_record_to_summary, checked_add_assign, + finalize_claude_summary, quota_history_record_from_usage, scan_claude_file_with_pricing, + zero_fill_uninitialized_claude_daily_costs, }; +use claude_usage::{ClaudeUsageDedupKey, session_id_from_entries}; pub use daily_history::{ DailyCostAndIncomplete, get_daily_cost_and_incomplete_history, get_daily_cost_history, get_daily_token_history, has_cost_usage_sources, @@ -968,301 +968,3 @@ impl CostScanner { read_failures } } - -fn scan_claude_file_with_pricing( - path: &Path, - cutoff: &DateTime, - seen: &mut HashSet, - cancel: Option<&AtomicBool>, - pricing: &mut ClaudeScanPricingResolver, - incomplete: &mut ClaudeIncompleteTracker, - mut on_record: F, -) -> ClaudeFileScanResult -where - F: FnMut(&ClaudeUsageRecord), -{ - let Ok(file) = File::open(path) else { - return ClaudeFileScanResult { - read_failures: 1, - ..ClaudeFileScanResult::default() - }; - }; - - let mut result = ClaudeFileScanResult::default(); - // Use read_until so a final incomplete line (no trailing newline) is still - // processed when it is valid UTF-8 JSON, and so a single bad line does not - // stop the walk the way `lines().map_while(Result::ok)` would. - let line_result = for_each_jsonl_text_line(BufReader::new(file), |line| { - if is_cancelled(cancel) { - return false; - } - if line.trim().is_empty() { - return true; - } - let Ok(event) = serde_json::from_str::(line) else { - result.malformed_lines = result.malformed_lines.saturating_add(1); - return true; - }; - if event.is_vertex_ai_usage_entry() { - return true; - } - if is_preliminary_claude_usage(&event) { - result.incomplete_requests = result.incomplete_requests.saturating_add(1); - if let Some(message) = event.message.as_ref() { - incomplete.record( - claude_usage_dedup_key( - message.id.as_deref(), - event.request_id.as_deref(), - event.session_id(), - ), - message.model.as_deref().unwrap_or("claude-3-5-sonnet"), - event.parsed_timestamp(), - cutoff, - ); - } - return true; - } - if let Some(record) = claude_usage_record_from_event_with_pricing(&event, pricing) - && should_count_claude_record(&record, cutoff, seen) - { - result.counted += 1; - on_record(&record); - } - true - }); - result.malformed_lines = result - .malformed_lines - .saturating_add(line_result.malformed_lines); - result.read_failures = line_result.read_failures; - result -} - -/// Walk JSONL text lines from `reader`, including a final incomplete line at EOF. -/// Continues past invalid UTF-8 segments. `on_line` returns `false` to stop early. -fn for_each_jsonl_text_line(mut reader: R, mut on_line: F) -> ClaudeJsonlLineResult -where - R: BufRead, - F: FnMut(&str) -> bool, -{ - let mut result = ClaudeJsonlLineResult::default(); - let mut buf = Vec::new(); - loop { - buf.clear(); - match reader.read_until(b'\n', &mut buf) { - Ok(0) => break, - Ok(_) => {} - Err(_) => { - result.read_failures = result.read_failures.saturating_add(1); - break; - } - } - while matches!(buf.last(), Some(b'\n' | b'\r')) { - buf.pop(); - } - let Ok(line) = std::str::from_utf8(&buf) else { - result.malformed_lines = result.malformed_lines.saturating_add(1); - continue; - }; - if !on_line(line) { - break; - } - } - result -} - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -struct ClaudeJsonlLineResult { - malformed_lines: u32, - read_failures: u32, -} - -fn finalize_claude_summary( - summary: &mut CostSummary, - projects_dir_exists: bool, - scan_result: ClaudeFileScanResult, - cancelled: bool, -) { - let complete = projects_dir_exists && !cancelled && scan_result.is_complete(); - summary.history_coverage_established = complete; - summary.known_zero = complete - && summary.sessions_count == 0 - && summary.input_tokens == 0 - && summary.output_tokens == 0 - && summary.cached_tokens == 0 - && summary.total_cost_usd == 0.0; -} - -fn claude_usage_record_from_event_with_pricing( - event: &ClaudeEvent, - pricing: &mut ClaudeScanPricingResolver, -) -> Option { - if event.event_type.as_deref() != Some("assistant") || is_preliminary_claude_usage(event) { - return None; - } - - let message = event.message.as_ref()?; - let usage = message.usage.as_ref()?; - let model = message.model.as_deref().unwrap_or("claude-3-5-sonnet"); - - let input = usage.input_tokens.unwrap_or(0); - let output = usage.output_tokens.unwrap_or(0); - let cache_create = usage.cache_creation_input_tokens.unwrap_or(0); - let cache_read = usage.cache_read_input_tokens.unwrap_or(0); - - if input == 0 && output == 0 && cache_create == 0 && cache_read == 0 { - return None; - } - - let cache_create_1h = usage.one_hour_cache_creation_tokens(cache_create); - let pricing_known = pricing.is_known(model); - let computed_cost = pricing.cost_usd_with_cache_ttl( - model, - input, - cache_create, - cache_create_1h, - cache_read, - output, - ); - let cost = computed_cost.is_finite().then_some(computed_cost); - - Some(ClaudeUsageRecord { - model: model.to_string(), - pricing_known, - timestamp: event.parsed_timestamp(), - dedup_key: claude_usage_dedup_key( - message.id.as_deref(), - event.request_id.as_deref(), - event.session_id(), - ), - input, - output, - cache_create, - cache_read, - cost, - }) -} - -fn add_claude_record_to_summary(summary: &mut CostSummary, record: &ClaudeUsageRecord) -> bool { - if !record.pricing_known { - summary.unknown_models.insert(record.model.clone()); - } - - let mut complete = checked_add_assign(&mut summary.input_tokens, record.input); - complete &= checked_add_assign(&mut summary.output_tokens, record.output); - let cached = record.cache_create.checked_add(record.cache_read); - complete &= cached.is_some_and(|value| checked_add_assign(&mut summary.cached_tokens, value)); - - if let Some(cost) = record.cost { - complete &= checked_add_finite(&mut summary.total_cost_usd, cost); - complete &= checked_add_finite( - summary.by_model.entry(record.model.clone()).or_insert(0.0), - cost, - ); - } else { - complete = false; - } - - let model_tokens = summary - .by_model_tokens - .entry(record.model.clone()) - .or_default(); - complete &= checked_add_assign(&mut model_tokens.input_tokens, record.input); - complete &= checked_add_assign(&mut model_tokens.output_tokens, record.output); - complete &= - cached.is_some_and(|value| checked_add_assign(&mut model_tokens.cached_tokens, value)); - complete -} - -fn checked_add_assign(total: &mut u64, value: u64) -> bool { - let Some(sum) = total.checked_add(value) else { - return false; - }; - *total = sum; - true -} - -fn checked_add_finite(total: &mut f64, value: f64) -> bool { - let sum = *total + value; - if !value.is_finite() || !sum.is_finite() { - return false; - } - *total = sum; - true -} - -fn quota_history_record_from_usage(record: &ClaudeUsageRecord) -> Option { - let timestamp = record.timestamp?; - let tokens = record - .input - .checked_add(record.output) - .and_then(|value| value.checked_add(record.cache_create)) - .and_then(|value| value.checked_add(record.cache_read)); - let dedup_key = record.dedup_key.as_ref().map(|key| match key { - ClaudeUsageDedupKey::Request { - message_id, - request_id, - } => ClaudeQuotaDedupKey::Request { - message_id: message_id.clone(), - request_id: request_id.clone(), - }, - ClaudeUsageDedupKey::Session { - session_id, - message_id, - } => ClaudeQuotaDedupKey::Session { - session_id: session_id.clone(), - message_id: message_id.clone(), - }, - }); - Some(ClaudeQuotaHistoryRecord { - timestamp, - tokens, - cost_usd: record.cost, - tokens_are_complete: tokens.is_some(), - cost_is_complete: record.pricing_known && record.cost.is_some_and(|cost| cost >= 0.0), - dedup_key, - attribution: ClaudeHistoryAttribution::Unavailable, - }) -} - -/// Add one usage record to the per-day cost buckets, keyed by the record's -/// own timestamp in the pinned bucket zone. Records outside the initialized -/// date range (or without a timestamp) are ignored. -fn add_claude_record_to_daily_costs( - daily_costs: &mut HashMap>, - unknown_cost_dates: &mut HashSet, - record: &ClaudeUsageRecord, -) -> bool { - let Some(timestamp) = record.timestamp else { - return true; - }; - let date_str = today::day_key(cost_bucket_zone().date(timestamp)); - if let Some(cost) = daily_costs.get_mut(&date_str) { - if unknown_cost_dates.contains(&date_str) { - return false; - } - let Some(record_cost) = record.cost else { - *cost = None; - unknown_cost_dates.insert(date_str); - return false; - }; - let sum = cost.unwrap_or(0.0) + record_cost; - if !sum.is_finite() { - *cost = None; - unknown_cost_dates.insert(date_str); - return false; - } - *cost = Some(sum); - } - true -} - -fn zero_fill_uninitialized_claude_daily_costs( - daily_costs: &mut HashMap>, - unknown_cost_dates: &HashSet, -) { - for (day, cost) in daily_costs { - if cost.is_none() && !unknown_cost_dates.contains(day) { - *cost = Some(0.0); - } - } -} diff --git a/rust/src/cost_scanner/claude_scan.rs b/rust/src/cost_scanner/claude_scan.rs new file mode 100644 index 0000000000..44719908ed --- /dev/null +++ b/rust/src/cost_scanner/claude_scan.rs @@ -0,0 +1,327 @@ +//! Reading one Claude transcript file into usage records, and folding those +//! records into summaries, daily buckets and quota history. + +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::sync::atomic::AtomicBool; + +use chrono::{DateTime, Utc}; + +use super::claude_incomplete::ClaudeIncompleteTracker; +use super::claude_pricing::ClaudeScanPricingResolver; +use super::claude_usage::{ + ClaudeUsageDedupKey, claude_usage_dedup_key, should_count_claude_record, +}; +use super::{ + ClaudeEvent, ClaudeFileScanResult, ClaudeUsageRecord, CostSummary, is_cancelled, + is_preliminary_claude_usage, today, +}; +use crate::cost_reporting_period::cost_bucket_zone; +use crate::providers::claude::quota_history::{ + ClaudeHistoryAttribution, ClaudeQuotaDedupKey, ClaudeQuotaHistoryRecord, +}; + +pub(super) fn scan_claude_file_with_pricing( + path: &Path, + cutoff: &DateTime, + seen: &mut HashSet, + cancel: Option<&AtomicBool>, + pricing: &mut ClaudeScanPricingResolver, + incomplete: &mut ClaudeIncompleteTracker, + mut on_record: F, +) -> ClaudeFileScanResult +where + F: FnMut(&ClaudeUsageRecord), +{ + let Ok(file) = File::open(path) else { + return ClaudeFileScanResult { + read_failures: 1, + ..ClaudeFileScanResult::default() + }; + }; + + let mut result = ClaudeFileScanResult::default(); + // Use read_until so a final incomplete line (no trailing newline) is still + // processed when it is valid UTF-8 JSON, and so a single bad line does not + // stop the walk the way `lines().map_while(Result::ok)` would. + let line_result = for_each_jsonl_text_line(BufReader::new(file), |line| { + if is_cancelled(cancel) { + return false; + } + if line.trim().is_empty() { + return true; + } + let Ok(event) = serde_json::from_str::(line) else { + result.malformed_lines = result.malformed_lines.saturating_add(1); + return true; + }; + if event.is_vertex_ai_usage_entry() { + return true; + } + if is_preliminary_claude_usage(&event) { + result.incomplete_requests = result.incomplete_requests.saturating_add(1); + if let Some(message) = event.message.as_ref() { + incomplete.record( + claude_usage_dedup_key( + message.id.as_deref(), + event.request_id.as_deref(), + event.session_id(), + ), + message.model.as_deref().unwrap_or("claude-3-5-sonnet"), + event.parsed_timestamp(), + cutoff, + ); + } + return true; + } + if let Some(record) = claude_usage_record_from_event_with_pricing(&event, pricing) + && should_count_claude_record(&record, cutoff, seen) + { + result.counted += 1; + on_record(&record); + } + true + }); + result.malformed_lines = result + .malformed_lines + .saturating_add(line_result.malformed_lines); + result.read_failures = line_result.read_failures; + result +} + +/// Walk JSONL text lines from `reader`, including a final incomplete line at EOF. +/// Continues past invalid UTF-8 segments. `on_line` returns `false` to stop early. +pub(super) fn for_each_jsonl_text_line(mut reader: R, mut on_line: F) -> ClaudeJsonlLineResult +where + R: BufRead, + F: FnMut(&str) -> bool, +{ + let mut result = ClaudeJsonlLineResult::default(); + let mut buf = Vec::new(); + loop { + buf.clear(); + match reader.read_until(b'\n', &mut buf) { + Ok(0) => break, + Ok(_) => {} + Err(_) => { + result.read_failures = result.read_failures.saturating_add(1); + break; + } + } + while matches!(buf.last(), Some(b'\n' | b'\r')) { + buf.pop(); + } + let Ok(line) = std::str::from_utf8(&buf) else { + result.malformed_lines = result.malformed_lines.saturating_add(1); + continue; + }; + if !on_line(line) { + break; + } + } + result +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(super) struct ClaudeJsonlLineResult { + pub(super) malformed_lines: u32, + pub(super) read_failures: u32, +} + +pub(super) fn finalize_claude_summary( + summary: &mut CostSummary, + projects_dir_exists: bool, + scan_result: ClaudeFileScanResult, + cancelled: bool, +) { + let complete = projects_dir_exists && !cancelled && scan_result.is_complete(); + summary.history_coverage_established = complete; + summary.known_zero = complete + && summary.sessions_count == 0 + && summary.input_tokens == 0 + && summary.output_tokens == 0 + && summary.cached_tokens == 0 + && summary.total_cost_usd == 0.0; +} + +pub(super) fn claude_usage_record_from_event_with_pricing( + event: &ClaudeEvent, + pricing: &mut ClaudeScanPricingResolver, +) -> Option { + if event.event_type.as_deref() != Some("assistant") || is_preliminary_claude_usage(event) { + return None; + } + + let message = event.message.as_ref()?; + let usage = message.usage.as_ref()?; + let model = message.model.as_deref().unwrap_or("claude-3-5-sonnet"); + + let input = usage.input_tokens.unwrap_or(0); + let output = usage.output_tokens.unwrap_or(0); + let cache_create = usage.cache_creation_input_tokens.unwrap_or(0); + let cache_read = usage.cache_read_input_tokens.unwrap_or(0); + + if input == 0 && output == 0 && cache_create == 0 && cache_read == 0 { + return None; + } + + let cache_create_1h = usage.one_hour_cache_creation_tokens(cache_create); + let pricing_known = pricing.is_known(model); + let computed_cost = pricing.cost_usd_with_cache_ttl( + model, + input, + cache_create, + cache_create_1h, + cache_read, + output, + ); + let cost = computed_cost.is_finite().then_some(computed_cost); + + Some(ClaudeUsageRecord { + model: model.to_string(), + pricing_known, + timestamp: event.parsed_timestamp(), + dedup_key: claude_usage_dedup_key( + message.id.as_deref(), + event.request_id.as_deref(), + event.session_id(), + ), + input, + output, + cache_create, + cache_read, + cost, + }) +} + +pub(super) fn add_claude_record_to_summary( + summary: &mut CostSummary, + record: &ClaudeUsageRecord, +) -> bool { + if !record.pricing_known { + summary.unknown_models.insert(record.model.clone()); + } + + let mut complete = checked_add_assign(&mut summary.input_tokens, record.input); + complete &= checked_add_assign(&mut summary.output_tokens, record.output); + let cached = record.cache_create.checked_add(record.cache_read); + complete &= cached.is_some_and(|value| checked_add_assign(&mut summary.cached_tokens, value)); + + if let Some(cost) = record.cost { + complete &= checked_add_finite(&mut summary.total_cost_usd, cost); + complete &= checked_add_finite( + summary.by_model.entry(record.model.clone()).or_insert(0.0), + cost, + ); + } else { + complete = false; + } + + let model_tokens = summary + .by_model_tokens + .entry(record.model.clone()) + .or_default(); + complete &= checked_add_assign(&mut model_tokens.input_tokens, record.input); + complete &= checked_add_assign(&mut model_tokens.output_tokens, record.output); + complete &= + cached.is_some_and(|value| checked_add_assign(&mut model_tokens.cached_tokens, value)); + complete +} + +pub(super) fn checked_add_assign(total: &mut u64, value: u64) -> bool { + let Some(sum) = total.checked_add(value) else { + return false; + }; + *total = sum; + true +} + +pub(super) fn checked_add_finite(total: &mut f64, value: f64) -> bool { + let sum = *total + value; + if !value.is_finite() || !sum.is_finite() { + return false; + } + *total = sum; + true +} + +pub(super) fn quota_history_record_from_usage( + record: &ClaudeUsageRecord, +) -> Option { + let timestamp = record.timestamp?; + let tokens = record + .input + .checked_add(record.output) + .and_then(|value| value.checked_add(record.cache_create)) + .and_then(|value| value.checked_add(record.cache_read)); + let dedup_key = record.dedup_key.as_ref().map(|key| match key { + ClaudeUsageDedupKey::Request { + message_id, + request_id, + } => ClaudeQuotaDedupKey::Request { + message_id: message_id.clone(), + request_id: request_id.clone(), + }, + ClaudeUsageDedupKey::Session { + session_id, + message_id, + } => ClaudeQuotaDedupKey::Session { + session_id: session_id.clone(), + message_id: message_id.clone(), + }, + }); + Some(ClaudeQuotaHistoryRecord { + timestamp, + tokens, + cost_usd: record.cost, + tokens_are_complete: tokens.is_some(), + cost_is_complete: record.pricing_known && record.cost.is_some_and(|cost| cost >= 0.0), + dedup_key, + attribution: ClaudeHistoryAttribution::Unavailable, + }) +} + +/// Add one usage record to the per-day cost buckets, keyed by the record's +/// own timestamp in the pinned bucket zone. Records outside the initialized +/// date range (or without a timestamp) are ignored. +pub(super) fn add_claude_record_to_daily_costs( + daily_costs: &mut HashMap>, + unknown_cost_dates: &mut HashSet, + record: &ClaudeUsageRecord, +) -> bool { + let Some(timestamp) = record.timestamp else { + return true; + }; + let date_str = today::day_key(cost_bucket_zone().date(timestamp)); + if let Some(cost) = daily_costs.get_mut(&date_str) { + if unknown_cost_dates.contains(&date_str) { + return false; + } + let Some(record_cost) = record.cost else { + *cost = None; + unknown_cost_dates.insert(date_str); + return false; + }; + let sum = cost.unwrap_or(0.0) + record_cost; + if !sum.is_finite() { + *cost = None; + unknown_cost_dates.insert(date_str); + return false; + } + *cost = Some(sum); + } + true +} + +pub(super) fn zero_fill_uninitialized_claude_daily_costs( + daily_costs: &mut HashMap>, + unknown_cost_dates: &HashSet, +) { + for (day, cost) in daily_costs { + if cost.is_none() && !unknown_cost_dates.contains(day) { + *cost = Some(0.0); + } + } +} diff --git a/rust/src/cost_scanner/tests.rs b/rust/src/cost_scanner/tests.rs index f57d2ae5ff..4b2b8dc2f6 100644 --- a/rust/src/cost_scanner/tests.rs +++ b/rust/src/cost_scanner/tests.rs @@ -2,9 +2,13 @@ use super::*; use crate::codex_costs::codex_period_start; use crate::core::test_fixtures::test_file_usage; use crate::core::{CodexSessionLineage, CostUsagePricing}; +use crate::cost_reporting_period::cost_bucket_zone; use chrono::{Duration, FixedOffset, Local, NaiveTime, TimeZone}; use claude_pricing::FALLBACK_CLAUDE_MODEL; +use claude_scan::claude_usage_record_from_event_with_pricing; +use claude_usage::{claude_usage_dedup_key, should_count_claude_record}; use daily_history::mark_claude_daily_token_coverage; +use std::fs::File; use std::io::Write; use support::{ app_scanner, cached_file, cached_input_total, cached_usage_with_packed, From d5570219bd072019d90b4e97cf5bd09509d47a71 Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:41:27 +0700 Subject: [PATCH 41/41] Move the Claude transcript row types out of the cost scanner --- rust/src/cost_scanner.rs | 238 +----------------------- rust/src/cost_scanner/claude_events.rs | 241 +++++++++++++++++++++++++ 2 files changed, 244 insertions(+), 235 deletions(-) create mode 100644 rust/src/cost_scanner/claude_events.rs diff --git a/rust/src/cost_scanner.rs b/rust/src/cost_scanner.rs index c84cee70d8..0e22883962 100755 --- a/rust/src/cost_scanner.rs +++ b/rust/src/cost_scanner.rs @@ -9,8 +9,6 @@ //! cancel flags between files. use chrono::{DateTime, NaiveDate, Utc}; -use serde::Deserialize; -use serde_json::Value; use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; @@ -32,6 +30,7 @@ use crate::cost_reporting_period::{CostReportingPeriod, MAX_ROLLING_DAYS}; use crate::providers::claude::quota_history::ClaudeQuotaHistoryRecord; use crate::providers::opencodego::local as opencodego_local; use crate::settings::Settings; +mod claude_events; mod claude_incomplete; mod claude_pricing; mod claude_roots; @@ -43,6 +42,7 @@ mod read_receipt; mod stats; mod today; mod window; +use claude_events::{ClaudeEvent, is_preliminary_claude_usage}; pub use claude_incomplete::ClaudeIncompleteReport; use claude_incomplete::ClaudeIncompleteTracker; use claude_pricing::ClaudeScanPricingResolver; @@ -51,7 +51,7 @@ use claude_scan::{ finalize_claude_summary, quota_history_record_from_usage, scan_claude_file_with_pricing, zero_fill_uninitialized_claude_daily_costs, }; -use claude_usage::{ClaudeUsageDedupKey, session_id_from_entries}; +use claude_usage::ClaudeUsageDedupKey; pub use daily_history::{ DailyCostAndIncomplete, get_daily_cost_and_incomplete_history, get_daily_cost_history, get_daily_token_history, has_cost_usage_sources, @@ -231,238 +231,6 @@ fn file_len_i64(metadata: &fs::Metadata) -> i64 { i64::try_from(metadata.len()).unwrap_or(i64::MAX) } -/// JSONL event structures for Claude transcripts. -/// -/// The flattened values retain otherwise-unknown metadata long enough to -/// distinguish Anthropic rows from Vertex AI rows. Claude's local transcript -/// format can contain both shapes, and counting Vertex rows with Anthropic -/// pricing would misstate both cost and token history. -#[derive(Debug, Deserialize)] -struct ClaudeEvent { - #[serde(rename = "type")] - event_type: Option, - timestamp: Option, - #[serde(rename = "requestId", alias = "request_id")] - request_id: Option, - #[serde(rename = "sessionId", alias = "session_id")] - session_id: Option, - message: Option, - #[serde(flatten)] - extra: HashMap, -} - -impl ClaudeEvent { - fn parsed_timestamp(&self) -> Option> { - let timestamp = self.timestamp.as_deref()?; - DateTime::parse_from_rfc3339(timestamp) - .ok() - .map(|ts| ts.with_timezone(&Utc)) - } - - fn session_id(&self) -> Option<&str> { - self.session_id - .as_deref() - .map(str::trim) - .filter(|session_id| !session_id.is_empty()) - .or_else(|| session_id_from_entries(self.extra.iter())) - .or_else(|| { - self.message - .as_ref() - .and_then(|message| session_id_from_entries(message.extra.iter())) - }) - } - - fn is_vertex_ai_usage_entry(&self) -> bool { - // Vertex AI message/request identifiers use the `_vrtx_` marker. - if self - .message - .as_ref() - .and_then(|message| message.id.as_deref()) - .is_some_and(|id| id.contains("_vrtx_")) - || self - .request_id - .as_deref() - .is_some_and(|request_id| request_id.contains("_vrtx_")) - { - return true; - } - - // Vertex AI model names use `@` as the version separator. - if self - .message - .as_ref() - .and_then(|message| message.model.as_deref()) - .is_some_and(model_name_looks_vertex) - { - return true; - } - - if contains_claude_vertex_metadata_entries(self.extra.iter()) { - return true; - } - self.message - .as_ref() - .is_some_and(ClaudeMessage::contains_vertex_metadata) - } -} - -#[derive(Debug, Deserialize)] -struct ClaudeMessage { - id: Option, - model: Option, - usage: Option, - #[serde(flatten)] - extra: HashMap, -} - -impl ClaudeMessage { - fn contains_vertex_metadata(&self) -> bool { - if contains_claude_vertex_metadata_entries(self.extra.iter()) { - return true; - } - self.usage - .as_ref() - .is_some_and(ClaudeUsage::contains_vertex_metadata) - } -} - -/// Claude Code proxies can emit a cache-unaware `message_start` estimate -/// before the final assistant response. It has a null stop reason, input -/// tokens, no output, and no cache breakdown; pricing that row would count a -/// preliminary estimate alongside the eventual final usage. -fn is_preliminary_claude_usage(event: &ClaudeEvent) -> bool { - if event.event_type.as_deref() != Some("assistant") { - return false; - } - let Some(message) = event.message.as_ref() else { - return false; - }; - let Some(usage) = message.usage.as_ref() else { - return false; - }; - - message.extra.get("stop_reason").is_some_and(Value::is_null) - && usage.input_tokens.unwrap_or(0) > 0 - && usage.output_tokens.unwrap_or(0) == 0 - && usage.cache_read_input_tokens.is_none() - && usage.cache_creation_input_tokens.is_none() -} - -#[derive(Debug, Deserialize)] -struct ClaudeUsage { - input_tokens: Option, - output_tokens: Option, - cache_creation_input_tokens: Option, - cache_read_input_tokens: Option, - cache_creation: Option, - #[serde(flatten)] - extra: HashMap, -} - -impl ClaudeUsage { - fn contains_vertex_metadata(&self) -> bool { - if contains_claude_vertex_metadata_entries(self.extra.iter()) { - return true; - } - self.cache_creation - .as_ref() - .is_some_and(ClaudeCacheCreation::contains_vertex_metadata) - } -} - -impl ClaudeUsage { - /// One-hour cache-write tokens, clamped to the total cache-write count. - fn one_hour_cache_creation_tokens(&self, total: u64) -> u64 { - self.cache_creation - .as_ref() - .and_then(|cache_creation| cache_creation.ephemeral_1h_input_tokens) - .unwrap_or(0) - .min(total) - } -} - -/// TTL breakdown of cache writes reported by the API. -#[derive(Debug, Deserialize)] -struct ClaudeCacheCreation { - ephemeral_1h_input_tokens: Option, - #[serde(flatten)] - extra: HashMap, -} - -impl ClaudeCacheCreation { - fn contains_vertex_metadata(&self) -> bool { - contains_claude_vertex_metadata_entries(self.extra.iter()) - } -} - -const CLAUDE_VERTEX_PROVIDER_KEYS: &[&str] = &[ - "provider", - "platform", - "backend", - "api_provider", - "apiprovider", - "api_type", - "apitype", - "source", - "vendor", - "client", -]; - -fn model_name_looks_vertex(model: &str) -> bool { - model.starts_with("claude-") && model.contains('@') -} - -/// Match the upstream Claude classifier's recursive metadata rules. Marker -/// keys (`vertex`/`gcp`) classify regardless of value; provider-key values -/// classify only when their text contains `vertex` (not merely `gcp`). -fn contains_claude_vertex_metadata(value: &Value) -> bool { - match value { - Value::Object(object) => contains_claude_vertex_metadata_entries(object.iter()), - Value::Array(array) => array.iter().any(contains_claude_vertex_metadata), - Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => false, - } -} - -fn contains_claude_vertex_metadata_entries<'a, I>(entries: I) -> bool -where - I: IntoIterator, -{ - entries.into_iter().any(|(key, value)| { - contains_claude_vertex_marker(key, true) - || (CLAUDE_VERTEX_PROVIDER_KEYS - .iter() - .any(|candidate| key.eq_ignore_ascii_case(candidate)) - && value - .as_str() - .is_some_and(|text| contains_claude_vertex_marker(text, false))) - || contains_claude_vertex_metadata(value) - }) -} - -fn contains_claude_vertex_marker(value: &str, include_gcp: bool) -> bool { - let bytes = value.as_bytes(); - let has_marker = |marker: &[u8]| { - bytes.windows(marker.len()).any(|window| { - window - .iter() - .zip(marker) - .all(|(byte, expected)| byte.to_ascii_lowercase() == *expected) - }) - }; - - if has_marker(b"vertex") || (include_gcp && has_marker(b"gcp")) { - return true; - } - - // ASCII folding above is enough for the common path. Unicode lowercasing - // preserves the historical classifier's behavior for non-ASCII strings. - if value.is_ascii() { - return false; - } - let lower = value.to_lowercase(); - lower.contains("vertex") || (include_gcp && lower.contains("gcp")) -} - #[derive(Debug)] struct ClaudeUsageRecord { model: String, diff --git a/rust/src/cost_scanner/claude_events.rs b/rust/src/cost_scanner/claude_events.rs new file mode 100644 index 0000000000..0fc6ac09a3 --- /dev/null +++ b/rust/src/cost_scanner/claude_events.rs @@ -0,0 +1,241 @@ +//! Claude transcript JSONL rows and the Vertex AI metadata classifier. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use serde_json::Value; + +use super::claude_usage::session_id_from_entries; + +/// JSONL event structures for Claude transcripts. +/// +/// The flattened values retain otherwise-unknown metadata long enough to +/// distinguish Anthropic rows from Vertex AI rows. Claude's local transcript +/// format can contain both shapes, and counting Vertex rows with Anthropic +/// pricing would misstate both cost and token history. +#[derive(Debug, Deserialize)] +pub(super) struct ClaudeEvent { + #[serde(rename = "type")] + pub(super) event_type: Option, + pub(super) timestamp: Option, + #[serde(rename = "requestId", alias = "request_id")] + pub(super) request_id: Option, + #[serde(rename = "sessionId", alias = "session_id")] + pub(super) session_id: Option, + pub(super) message: Option, + #[serde(flatten)] + pub(super) extra: HashMap, +} + +impl ClaudeEvent { + pub(super) fn parsed_timestamp(&self) -> Option> { + let timestamp = self.timestamp.as_deref()?; + DateTime::parse_from_rfc3339(timestamp) + .ok() + .map(|ts| ts.with_timezone(&Utc)) + } + + pub(super) fn session_id(&self) -> Option<&str> { + self.session_id + .as_deref() + .map(str::trim) + .filter(|session_id| !session_id.is_empty()) + .or_else(|| session_id_from_entries(self.extra.iter())) + .or_else(|| { + self.message + .as_ref() + .and_then(|message| session_id_from_entries(message.extra.iter())) + }) + } + + pub(super) fn is_vertex_ai_usage_entry(&self) -> bool { + // Vertex AI message/request identifiers use the `_vrtx_` marker. + if self + .message + .as_ref() + .and_then(|message| message.id.as_deref()) + .is_some_and(|id| id.contains("_vrtx_")) + || self + .request_id + .as_deref() + .is_some_and(|request_id| request_id.contains("_vrtx_")) + { + return true; + } + + // Vertex AI model names use `@` as the version separator. + if self + .message + .as_ref() + .and_then(|message| message.model.as_deref()) + .is_some_and(model_name_looks_vertex) + { + return true; + } + + if contains_claude_vertex_metadata_entries(self.extra.iter()) { + return true; + } + self.message + .as_ref() + .is_some_and(ClaudeMessage::contains_vertex_metadata) + } +} + +#[derive(Debug, Deserialize)] +pub(super) struct ClaudeMessage { + pub(super) id: Option, + pub(super) model: Option, + pub(super) usage: Option, + #[serde(flatten)] + pub(super) extra: HashMap, +} + +impl ClaudeMessage { + pub(super) fn contains_vertex_metadata(&self) -> bool { + if contains_claude_vertex_metadata_entries(self.extra.iter()) { + return true; + } + self.usage + .as_ref() + .is_some_and(ClaudeUsage::contains_vertex_metadata) + } +} + +/// Claude Code proxies can emit a cache-unaware `message_start` estimate +/// before the final assistant response. It has a null stop reason, input +/// tokens, no output, and no cache breakdown; pricing that row would count a +/// preliminary estimate alongside the eventual final usage. +pub(super) fn is_preliminary_claude_usage(event: &ClaudeEvent) -> bool { + if event.event_type.as_deref() != Some("assistant") { + return false; + } + let Some(message) = event.message.as_ref() else { + return false; + }; + let Some(usage) = message.usage.as_ref() else { + return false; + }; + + message.extra.get("stop_reason").is_some_and(Value::is_null) + && usage.input_tokens.unwrap_or(0) > 0 + && usage.output_tokens.unwrap_or(0) == 0 + && usage.cache_read_input_tokens.is_none() + && usage.cache_creation_input_tokens.is_none() +} + +#[derive(Debug, Deserialize)] +pub(super) struct ClaudeUsage { + pub(super) input_tokens: Option, + pub(super) output_tokens: Option, + pub(super) cache_creation_input_tokens: Option, + pub(super) cache_read_input_tokens: Option, + pub(super) cache_creation: Option, + #[serde(flatten)] + pub(super) extra: HashMap, +} + +impl ClaudeUsage { + pub(super) fn contains_vertex_metadata(&self) -> bool { + if contains_claude_vertex_metadata_entries(self.extra.iter()) { + return true; + } + self.cache_creation + .as_ref() + .is_some_and(ClaudeCacheCreation::contains_vertex_metadata) + } +} + +impl ClaudeUsage { + /// One-hour cache-write tokens, clamped to the total cache-write count. + pub(super) fn one_hour_cache_creation_tokens(&self, total: u64) -> u64 { + self.cache_creation + .as_ref() + .and_then(|cache_creation| cache_creation.ephemeral_1h_input_tokens) + .unwrap_or(0) + .min(total) + } +} + +/// TTL breakdown of cache writes reported by the API. +#[derive(Debug, Deserialize)] +pub(super) struct ClaudeCacheCreation { + pub(super) ephemeral_1h_input_tokens: Option, + #[serde(flatten)] + pub(super) extra: HashMap, +} + +impl ClaudeCacheCreation { + pub(super) fn contains_vertex_metadata(&self) -> bool { + contains_claude_vertex_metadata_entries(self.extra.iter()) + } +} + +pub(super) const CLAUDE_VERTEX_PROVIDER_KEYS: &[&str] = &[ + "provider", + "platform", + "backend", + "api_provider", + "apiprovider", + "api_type", + "apitype", + "source", + "vendor", + "client", +]; + +pub(super) fn model_name_looks_vertex(model: &str) -> bool { + model.starts_with("claude-") && model.contains('@') +} + +/// Match the upstream Claude classifier's recursive metadata rules. Marker +/// keys (`vertex`/`gcp`) classify regardless of value; provider-key values +/// classify only when their text contains `vertex` (not merely `gcp`). +pub(super) fn contains_claude_vertex_metadata(value: &Value) -> bool { + match value { + Value::Object(object) => contains_claude_vertex_metadata_entries(object.iter()), + Value::Array(array) => array.iter().any(contains_claude_vertex_metadata), + Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => false, + } +} + +pub(super) fn contains_claude_vertex_metadata_entries<'a, I>(entries: I) -> bool +where + I: IntoIterator, +{ + entries.into_iter().any(|(key, value)| { + contains_claude_vertex_marker(key, true) + || (CLAUDE_VERTEX_PROVIDER_KEYS + .iter() + .any(|candidate| key.eq_ignore_ascii_case(candidate)) + && value + .as_str() + .is_some_and(|text| contains_claude_vertex_marker(text, false))) + || contains_claude_vertex_metadata(value) + }) +} + +pub(super) fn contains_claude_vertex_marker(value: &str, include_gcp: bool) -> bool { + let bytes = value.as_bytes(); + let has_marker = |marker: &[u8]| { + bytes.windows(marker.len()).any(|window| { + window + .iter() + .zip(marker) + .all(|(byte, expected)| byte.to_ascii_lowercase() == *expected) + }) + }; + + if has_marker(b"vertex") || (include_gcp && has_marker(b"gcp")) { + return true; + } + + // ASCII folding above is enough for the common path. Unicode lowercasing + // preserves the historical classifier's behavior for non-ASCII strings. + if value.is_ascii() { + return false; + } + let lower = value.to_lowercase(); + lower.contains("vertex") || (include_gcp && lower.contains("gcp")) +}