diff --git a/apps/maple-agent/crates/maple-agent/src/agent/provider.rs b/apps/maple-agent/crates/maple-agent/src/agent/provider.rs index 36f8b2479..ee8b2732b 100644 --- a/apps/maple-agent/crates/maple-agent/src/agent/provider.rs +++ b/apps/maple-agent/crates/maple-agent/src/agent/provider.rs @@ -42,8 +42,17 @@ const MAPLE_BASE_URL: &str = "/v1"; const CHAT_COMPLETIONS_PATH: &str = "/v1/chat/completions"; /// Where Pi's System One classifier posts: `{base_url}/systemone`. const SYSTEM_ONE_PATH: &str = "/v1/systemone"; -/// The model System One answers with, the only one Maple's API serves. +/// The model System One answers with by default. pub(crate) const SYSTEM_ONE_MODEL_ID: &str = "glm-5-3-flash"; +/// The decision models Maple's System One API also serves (Tinfoil's), with +/// their names: text only, at most 26 options per choice, and a 32,768-token +/// context per question. +const SYSTEM_ONE_DECISION_MODELS: [(&str, &str); 3] = [ + ("decision-small", "Decision Small"), + ("decision-medium", "Decision Medium"), + ("decision-large", "Decision Large"), +]; +const DECISION_MODEL_CONTEXT_WINDOW: u64 = 32_768; pub(crate) const AUTHENTICATION_ERROR_MESSAGE: &str = "Maple authentication failed"; pub(crate) const CREDITS_EXHAUSTED_MESSAGE: &str = "Maple credits are exhausted"; @@ -144,7 +153,9 @@ pub(crate) fn maple_model_registry( system_one::API, Arc::new(SystemOne::new(Arc::new(MapleTransport { transport }))), ); - registry.register_classifier_models([system_one_model()]); + registry.register_classifier_models( + std::iter::once(system_one_model()).chain(system_one_decision_models()), + ); registry } @@ -165,6 +176,23 @@ pub(crate) fn system_one_model() -> ClassifierModel { } } +/// The decision models as Pi classifier models: they judge text only. +fn system_one_decision_models() -> impl Iterator { + SYSTEM_ONE_DECISION_MODELS + .into_iter() + .map(|(id, name)| ClassifierModel { + id: id.to_string(), + name: name.to_string(), + api: system_one::API.to_string(), + provider: MAPLE_PROVIDER.to_string(), + base_url: MAPLE_BASE_URL.to_string(), + input: vec![InputModality::Text], + cost: Default::default(), + context_window: DECISION_MODEL_CONTEXT_WINDOW, + headers: Default::default(), + }) +} + /// OpenSecret authenticates the account itself; Pi passes no key. struct NoKeys; diff --git a/apps/maple-agent/crates/maple-agent/src/agent/provider/tests.rs b/apps/maple-agent/crates/maple-agent/src/agent/provider/tests.rs index 825a013bd..a3dc8ef9a 100644 --- a/apps/maple-agent/crates/maple-agent/src/agent/provider/tests.rs +++ b/apps/maple-agent/crates/maple-agent/src/agent/provider/tests.rs @@ -273,6 +273,30 @@ async fn system_one_classifies_over_the_accounts_transport() { assert_eq!(requests[1].body["questions"]["urgent"]["type"], "noul"); } +#[tokio::test] +async fn decision_models_classify_text_through_the_same_api() { + let transport = FakeTransport::new(vec![json_reply(200, Vec::new(), SYSTEM_ONE_REPLY)]); + let registry = maple_model_registry(transport.clone(), []); + for id in ["decision-small", "decision-medium", "decision-large"] { + let model = registry + .find_classifier(MAPLE_PROVIDER, id) + .expect("decision model is registered"); + assert!(!model.supports_images(), "{id}"); + assert_eq!(model.context_window, 32_768, "{id}"); + } + let model = registry + .find_classifier(MAPLE_PROVIDER, "decision-medium") + .unwrap(); + let result = registry + .classify(&model, urgent_ticket(), pi_ai::ClassifierOptions::default()) + .await; + assert!(!result.is_failure(), "{result:?}"); + let requests = transport.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].uri, SYSTEM_ONE_PATH); + assert_eq!(requests[0].body["model"], "decision-medium"); +} + #[tokio::test] async fn a_failed_system_one_request_keeps_its_body_out_of_the_error() { let transport = FakeTransport::new(vec![json_reply( diff --git a/proxy/README.md b/proxy/README.md index 33be79540..14add6736 100644 --- a/proxy/README.md +++ b/proxy/README.md @@ -268,10 +268,13 @@ curl http://localhost:8080/v1/systemone \ }' ``` -Typed questions (`noul`, `choice`, `score`) answered with calibrated -probabilities on Continuum GLM-5.3-Flash; request and response follow the -TypeSafe System One schema, with optional `images` (`data:` URLs) and -`temperature` fields. Rejections keep the backend's status and +Typed questions (`noul`, `choice`, `score`) answered with probabilities +instead of text; request and response follow the TypeSafe System One schema, +with optional `images` (`data:` URLs) and `temperature` fields. `model` picks +the engine: `glm-5-3-flash` (the default, on Continuum, calibrated, up to 255 +options per choice) or Tinfoil's `decision-small`, `decision-medium` and +`decision-large` (text only, up to 26 options per choice, one upstream request +per call). Rejections keep the backend's status and `x-opensecret-error-code` header, for example `422` with `system_one_too_many_questions`, or `system_one_unknown_field` naming a misspelled key. diff --git a/sdk/README.md b/sdk/README.md index 98b10f5f0..b524e4f96 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -192,9 +192,11 @@ different callback. Native Apple sign-in and the Rust SDK are unchanged. ### System One decisions `systemOne` answers typed questions (`noul`, `choice`, `score`) about a JSON -state with calibrated probabilities instead of text (`POST /v1/systemone`, -Continuum GLM-5.3-Flash). It uses the signed-in session, or an API key through -`options.apiKey`; `useOpenSecret().systemOne` is the same function: +state with probabilities instead of text (`POST /v1/systemone`). `model` is +`glm-5-3-flash` by default (Continuum, calibrated, takes `images`) or one of +Tinfoil's `decision-small`, `decision-medium` and `decision-large` (text only, +up to 26 options per choice). It uses the signed-in session, or an API key +through `options.apiKey`; `useOpenSecret().systemOne` is the same function: ```ts import { systemOne, SystemOneError } from "@mapleai/sdk"; diff --git a/sdk/rust/README.md b/sdk/rust/README.md index 95fb93f52..6c5ffca6a 100644 --- a/sdk/rust/README.md +++ b/sdk/rust/README.md @@ -167,10 +167,13 @@ logical request and is not installed as mutable client state. ### System One decisions -`system_one` answers typed questions about a JSON state with one masked -single-token decision each, returning calibrated probabilities instead of text -(`POST /v1/systemone`, Continuum GLM-5.3-Flash). Questions and choice options -keep the order you give them: +`system_one` answers typed questions about a JSON state with probabilities +instead of text (`POST /v1/systemone`). The default model, `glm-5-3-flash`, +reads one masked single-token decision per question on Continuum and +calibrates it; `with_model("decision-medium")` (or `decision-small`, +`decision-large`) uses Tinfoil's decision models, which read text only and take +up to 26 options per choice. Questions and choice options keep the order you +give them: ```rust use maple_sdk::{SystemOneAnswer, SystemOneQuestion, SystemOneRequest}; @@ -198,8 +201,9 @@ if let Some(SystemOneAnswer::Choice { choice, confidence, .. }) = response.answe The client's configured credential is used (API key or signed-in session); `system_one_with_api_key` encrypts an explicit key inside one request. Optional -request fields are `images` (`data:` URLs shown with the state) and -`temperature` (`1` returns raw probabilities). A rejected request is +request fields are `images` (`data:` URLs shown with the state, +`glm-5-3-flash` only) and `temperature` (`1`, the decision models' default, +returns raw probabilities). A rejected request is `Error::Api` with the backend's status (422 for schema problems, 413 for size limits, chat's statuses for quota and capacity) and body; `Error::api_error_code` returns its `system_one_*` code. The example diff --git a/sdk/rust/src/system_one.rs b/sdk/rust/src/system_one.rs index d0b6740dc..50d823794 100644 --- a/sdk/rust/src/system_one.rs +++ b/sdk/rust/src/system_one.rs @@ -169,13 +169,15 @@ impl SystemOneQuestion { /// per-type limits. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SystemOneRequest { - /// Defaults to the only supported model, `glm-5-3-flash`. + /// `glm-5-3-flash` when unset. `decision-small`, `decision-medium` and + /// `decision-large` read text only and take at most 26 options per choice. #[serde(default, skip_serializing_if = "Option::is_none")] pub model: Option, /// Any JSON; embedded in the prompt as given. pub state: Value, pub questions: OrderedEntries, - /// Up to 4 `data:` image URLs, 4 MiB each and 8 MiB in total, shown with the state. + /// Up to 4 `data:` image URLs, 4 MiB each and 8 MiB in total, shown with the state + /// (`glm-5-3-flash` only). #[serde(default, skip_serializing_if = "Vec::is_empty")] pub images: Vec, /// Calibration temperature override; `1` returns the raw probabilities. @@ -218,7 +220,9 @@ impl SystemOneRequest { /// The answer to one question. Every variant reports the calibration /// `temperature` applied and `option_mass`, the share of the model's next-token /// probability that fell on the offered options (low values mean the prompt did -/// not fit the question well). +/// not fit the question well). The decision models report temperature 1 unless +/// the request sets one, and an `option_mass` of 1: they score only the offered +/// options. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "lowercase")] pub enum SystemOneAnswer { @@ -260,7 +264,8 @@ pub struct SystemOneUsage { pub input_tokens: u64, pub output_tokens: u64, pub cached_tokens: u64, - /// Upstream inference requests made: one per question, more for wide choices. + /// Upstream inference requests made: one per question on `glm-5-3-flash` (more for + /// wide choices), one per call on the decision models. pub requests: u64, } diff --git a/sdk/src/lib/systemOne.ts b/sdk/src/lib/systemOne.ts index cb45dd353..577dafcf4 100644 --- a/sdk/src/lib/systemOne.ts +++ b/sdk/src/lib/systemOne.ts @@ -44,13 +44,19 @@ export type SystemOneQuestion = SystemOneNoulQuestion | SystemOneChoiceQuestion | SystemOneScoreQuestion; export type SystemOneRequest = { - /** Defaults to the only supported model, `glm-5-3-flash`. */ + /** + * `glm-5-3-flash` when unset. `decision-small`, `decision-medium` and + * `decision-large` read text only and take at most 26 options per choice. + */ model?: string; /** Any JSON; embedded in the prompt as given. */ state: unknown; /** Up to 64 questions, answered in the order given. */ questions: Record; - /** Up to 4 `data:` image URLs, 4 MiB each and 8 MiB in total, shown with the state. */ + /** + * Up to 4 `data:` image URLs, 4 MiB each and 8 MiB in total, shown with the + * state (`glm-5-3-flash` only). + */ images?: string[]; /** Calibration temperature override; `1` returns the raw probabilities. */ temperature?: number; @@ -59,7 +65,9 @@ export type SystemOneRequest = { /** * Every answer reports the calibration `temperature` applied and `option_mass`, * the share of the model's next-token probability that fell on the offered - * options (low values mean the prompt did not fit the question well). + * options (low values mean the prompt did not fit the question well). The + * decision models report temperature 1 unless the request sets one, and an + * `option_mass` of 1: they score only the offered options. */ export type SystemOneNoulAnswer = { type: "noul"; @@ -107,7 +115,10 @@ export type SystemOneUsage = { input_tokens: number; output_tokens: number; cached_tokens: number; - /** Upstream inference requests made: one per question, more for wide choices. */ + /** + * Upstream inference requests made: one per question on `glm-5-3-flash` (more + * for wide choices), one per call on the decision models. + */ requests: number; }; diff --git a/services/opensecret/src/inference/health.rs b/services/opensecret/src/inference/health.rs index 2749a5c52..aec7dbe48 100644 --- a/services/opensecret/src/inference/health.rs +++ b/services/opensecret/src/inference/health.rs @@ -260,16 +260,26 @@ impl ShadowHealthState { let mut route_health = HashMap::new(); let mut capacity = HashMap::new(); - for model in registry.completion_models() { - for route in model.routes { - let route_key = RouteKey { - provider: route.provider, - provider_model_id: route.provider_model_id.to_string(), - }; - let pool = CapacityPoolKey::ProviderModel(route_key.clone()); - route_health.entry(route_key.clone()).or_default(); - capacity.entry(pool).or_default(); - } + // Decision-model routes serve System One only, but their health and capacity are + // tracked the same way. + let routes = registry + .completion_models() + .iter() + .flat_map(|model| model.routes) + .chain( + registry + .decision_models() + .iter() + .flat_map(|model| model.routes), + ); + for route in routes { + let route_key = RouteKey { + provider: route.provider, + provider_model_id: route.provider_model_id.to_string(), + }; + let pool = CapacityPoolKey::ProviderModel(route_key.clone()); + route_health.entry(route_key.clone()).or_default(); + capacity.entry(pool).or_default(); } Self { @@ -2452,6 +2462,21 @@ mod tests { assert_eq!(state.cardinality(), cardinality); } + #[test] + fn decision_model_routes_are_tracked_like_completion_routes() { + let state = ShadowHealthState::new(&crate::provider_registry::PROVIDER_REGISTRY); + for model in crate::provider_registry::PROVIDER_REGISTRY.decision_models() { + let id = model.public_model_id; + let route = route(ProviderId::Tinfoil, id, id); + match state.try_claim_probe(&route.route_key()) { + ProbeClaimResult::Ready(_) => {} + other => panic!("decision route {id} must be known, got {other:?}"), + } + let report = state.observe_terminal(&completed(route), ShadowObservationMode::Update); + assert_ne!(report.signal, ShadowSignal::UnknownRoute, "{id}"); + } + } + #[test] fn maple_replica_probe_leases_and_recovery_do_not_cross_instance_boundaries() { let state = ShadowHealthState::new(&DUAL_INSTANCE_TEST_REGISTRY); diff --git a/services/opensecret/src/model_config.rs b/services/opensecret/src/model_config.rs index 3ecdb5eb7..0f05523f1 100644 --- a/services/opensecret/src/model_config.rs +++ b/services/opensecret/src/model_config.rs @@ -439,6 +439,51 @@ pub const POWERFUL_MODEL_ID: &str = GLM_5_3_MODEL_ID; pub const KIMI_K3_MODEL_ID: &str = "kimi-k3"; pub const DEEPSEEK_V4_1_FLASH_MODEL_ID: &str = "deepseek-v4-1-flash"; pub const DEEPSEEK_V4_1_FLASH_UNCENSORED_MODEL_ID: &str = "deepseek-v4-1-flash-1017-uncensored"; +pub const DECISION_SMALL_MODEL_ID: &str = "decision-small"; +pub const DECISION_MEDIUM_MODEL_ID: &str = "decision-medium"; +pub const DECISION_LARGE_MODEL_ID: &str = "decision-large"; + +/// A typed-decision model: it answers System One questions and nothing else, so it is in +/// neither chat catalog and never resolves as a completion model. The limits are those of +/// the serving deployment (Tinfoil's vLLM d3 family behind `/v1/decisions`): text only, at +/// most 26 lettered options per choice, and one question's prompt within the context window. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DecisionModel { + pub(crate) id: &'static str, + pub(crate) access: ModelAccessTier, + /// Tokens one question's prompt (state, question and options) may take. + pub(crate) context_window: usize, + pub(crate) max_choice_options: usize, +} + +impl DecisionModel { + const fn pro(id: &'static str) -> Self { + Self { + id, + access: ModelAccessTier::Pro, + context_window: 32_768, + max_choice_options: 26, + } + } +} + +const DECISION_MODELS: &[DecisionModel] = &[ + DecisionModel::pro(DECISION_SMALL_MODEL_ID), + DecisionModel::pro(DECISION_MEDIUM_MODEL_ID), + DecisionModel::pro(DECISION_LARGE_MODEL_ID), +]; + +pub(crate) fn decision_model(model: &str) -> Option { + DECISION_MODELS + .iter() + .find(|entry| entry.id == model) + .copied() +} + +#[cfg(test)] +pub(crate) fn decision_models() -> &'static [DecisionModel] { + DECISION_MODELS +} const FREE_MODEL_ALIAS_TARGETS: ModelAliasTargets = ModelAliasTargets { quick: QUICK_MODEL_ID, @@ -709,6 +754,9 @@ impl ModelPlan { pub(crate) fn allows_model(self, model: &str) -> bool { let canonical = alias_target(model).unwrap_or(model); + if let Some(decision) = decision_model(canonical) { + return decision.access.is_available_to(self); + } model_entry(canonical).is_none_or(|entry| entry.access.is_available_to(self)) } } @@ -1729,6 +1777,35 @@ mod tests { } } + #[test] + fn decision_models_are_pro_and_never_chat_models() { + let ids: Vec<&str> = decision_models().iter().map(|model| model.id).collect(); + assert_eq!( + ids, + [ + DECISION_SMALL_MODEL_ID, + DECISION_MEDIUM_MODEL_ID, + DECISION_LARGE_MODEL_ID + ] + ); + let catalog = model_catalog_response(ModelPlan::Paid, true); + let openai = openai_models_response(true); + for model in decision_models() { + assert_eq!(decision_model(model.id), Some(*model)); + assert!(!ModelPlan::Free.allows_model(model.id), "{}", model.id); + assert!(ModelPlan::Paid.allows_model(model.id), "{}", model.id); + assert_eq!(model.context_window, 32_768); + assert_eq!(model.max_choice_options, 26); + // Not a completion model: chat and Responses cannot resolve it, and neither + // catalog lists it. + assert_eq!(resolve_public_model_id(model.id), None); + assert!(model_capabilities(model.id).is_none()); + assert!(!has_model(&catalog, model.id)); + assert!(!has_model(&openai, model.id)); + } + assert_eq!(decision_model(GLM_5_3_FLASH_MODEL_ID), None); + } + #[test] fn test_catalog_hides_api_only_models_and_includes_aliases() { let response = model_catalog_response(ModelPlan::Free, false); diff --git a/services/opensecret/src/provider_registry.rs b/services/opensecret/src/provider_registry.rs index 289b78536..6bfd56137 100644 --- a/services/opensecret/src/provider_registry.rs +++ b/services/opensecret/src/provider_registry.rs @@ -5,6 +5,7 @@ //! completion model. use crate::model_config::{ + DECISION_LARGE_MODEL_ID, DECISION_MEDIUM_MODEL_ID, DECISION_SMALL_MODEL_ID, DEEPSEEK_V4_1_FLASH_MODEL_ID, DEEPSEEK_V4_1_FLASH_UNCENSORED_MODEL_ID, GLM_5_3_FLASH_MODEL_ID, GLM_5_3_MODEL_ID, KIMI_K3_MODEL_ID, QUICK_MODEL_ID, }; @@ -107,10 +108,19 @@ const fn completion_model( } } +/// The routes of a typed-decision model. These serve System One only and never take part in +/// completion routing; they are here so route health covers them like every other route. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DecisionModelSpec { + pub(crate) public_model_id: &'static str, + pub(crate) routes: &'static [ModelRouteSpec], +} + #[derive(Debug)] pub(crate) struct ProviderRegistry { providers: &'static [ProviderSpec], completion_models: &'static [CompletionModelSpec], + decision_models: &'static [DecisionModelSpec], } impl ProviderRegistry { @@ -131,6 +141,16 @@ impl ProviderRegistry { pub(crate) fn completion_models(&self) -> &'static [CompletionModelSpec] { self.completion_models } + + pub(crate) fn decision_model(&self, public_model_id: &str) -> Option<&DecisionModelSpec> { + self.decision_models + .iter() + .find(|model| model.public_model_id == public_model_id) + } + + pub(crate) fn decision_models(&self) -> &'static [DecisionModelSpec] { + self.decision_models + } } const PROVIDERS: &[ProviderSpec] = &[ @@ -276,6 +296,7 @@ const DUAL_INSTANCE_TEST_MODELS: &[CompletionModelSpec] = &[completion_model( pub(crate) static DUAL_INSTANCE_TEST_REGISTRY: ProviderRegistry = ProviderRegistry { providers: PROVIDERS, completion_models: DUAL_INSTANCE_TEST_MODELS, + decision_models: &[], }; const LLAMA3_3_70B_ROUTES: &[ModelRouteSpec] = &[ModelRouteSpec { @@ -309,9 +330,42 @@ const COMPLETION_MODELS: &[CompletionModelSpec] = &[ completion_model("gpt-oss-safeguard-120b", GPT_OSS_SAFEGUARD_120B_ROUTES), ]; +// Tinfoil serves the decision models under the same names as our public IDs. They have no +// other provider. +const fn tinfoil_decision_route(model: &'static str) -> ModelRouteSpec { + ModelRouteSpec { + provider: ProviderId::Tinfoil, + provider_model_id: model, + response_model_id: model, + weight: 100, + enabled: true, + } +} + +const DECISION_SMALL_ROUTES: &[ModelRouteSpec] = &[tinfoil_decision_route(DECISION_SMALL_MODEL_ID)]; +const DECISION_MEDIUM_ROUTES: &[ModelRouteSpec] = + &[tinfoil_decision_route(DECISION_MEDIUM_MODEL_ID)]; +const DECISION_LARGE_ROUTES: &[ModelRouteSpec] = &[tinfoil_decision_route(DECISION_LARGE_MODEL_ID)]; + +const DECISION_MODELS: &[DecisionModelSpec] = &[ + DecisionModelSpec { + public_model_id: DECISION_SMALL_MODEL_ID, + routes: DECISION_SMALL_ROUTES, + }, + DecisionModelSpec { + public_model_id: DECISION_MEDIUM_MODEL_ID, + routes: DECISION_MEDIUM_ROUTES, + }, + DecisionModelSpec { + public_model_id: DECISION_LARGE_MODEL_ID, + routes: DECISION_LARGE_ROUTES, + }, +]; + pub(crate) static PROVIDER_REGISTRY: ProviderRegistry = ProviderRegistry { providers: PROVIDERS, completion_models: COMPLETION_MODELS, + decision_models: DECISION_MODELS, }; #[cfg(test)] @@ -593,4 +647,30 @@ mod tests { [GLM_5_3_MODEL_ID, KIMI_K3_MODEL_ID] ); } + + #[test] + fn decision_models_have_one_tinfoil_route_each_and_are_not_completion_models() { + use crate::model_config::decision_models; + let registered: Vec<&str> = PROVIDER_REGISTRY + .decision_models() + .iter() + .map(|model| model.public_model_id) + .collect(); + let configured: Vec<&str> = decision_models().iter().map(|model| model.id).collect(); + assert_eq!(registered, configured); + for id in configured { + let model = PROVIDER_REGISTRY.decision_model(id).expect("registered"); + assert_eq!(model.routes.len(), 1, "{id}"); + let route = model.routes[0]; + assert_eq!(route.provider, ProviderId::Tinfoil); + assert_eq!(route.provider_model_id, id); + assert_eq!(route.response_model_id, id); + assert!(route.enabled); + assert!(PROVIDER_REGISTRY.completion_model(id).is_none(), "{id}"); + } + assert!(PROVIDER_REGISTRY + .decision_model(GLM_5_3_FLASH_MODEL_ID) + .is_none()); + assert!(DUAL_INSTANCE_TEST_REGISTRY.decision_models().is_empty()); + } } diff --git a/services/opensecret/src/web/system_one.rs b/services/opensecret/src/web/system_one.rs index 07ef1ac54..56e9c6927 100644 --- a/services/opensecret/src/web/system_one.rs +++ b/services/opensecret/src/web/system_one.rs @@ -1,35 +1,45 @@ //! System One decisions: `POST /v1/systemone`. //! //! A TypeSafe-compatible subset (state + typed `noul` / `choice` / `score` questions in, -//! typed answers with probabilities out) implemented as one masked, single-token chat -//! completion per question against Continuum's GLM-5.3-Flash. The model never generates -//! text: the prompt numbers the options, the assistant turn is prefilled with `answer=`, -//! sampling is restricted to the option-number tokens, and the distribution is read from -//! their log-probabilities. Prompt layout, prefill, and calibration follow Privatemode's -//! reference implementation (edgelesssys/privatemode-decisions at 45f4273, 2026-10-05) so -//! its benchmark and calibration numbers apply here. +//! typed answers with probabilities out). The request's `model` picks one of two engines +//! behind the same request and response shape: //! -//! First-pass MVP: the provider and model are fixed, the request bypasses the completion -//! router on purpose (only this deployment honors the masked logprob read), and nothing -//! here touches the chat API. Route health is still consulted: every upstream send claims a -//! probe on the Continuum route and reports its outcome, so an open circuit breaker rejects +//! - `glm-5-3-flash` (the default): one masked, single-token chat completion per question +//! against Continuum's GLM-5.3-Flash, described below. +//! - `decision-small`, `decision-medium`, `decision-large`: Tinfoil's decision models, one +//! `/v1/decisions` request for all questions (see `decisions`). They read text only and +//! take at most 26 options per choice; their probabilities come from a trained +//! classification head, so they are reported raw (temperature 1) with an `option_mass` of 1. +//! +//! GLM-5.3-Flash never generates text: the prompt numbers the options, the assistant turn is +//! prefilled with `answer=`, sampling is restricted to the option-number tokens, and the +//! distribution is read from their log-probabilities. Prompt layout, prefill, and +//! calibration follow Privatemode's reference implementation +//! (edgelesssys/privatemode-decisions at 45f4273, 2026-10-05) so its benchmark and +//! calibration numbers apply here. +//! +//! Each model has fixed routes: the request bypasses the completion router on purpose (only +//! Continuum honors the masked logprob read, only Tinfoil serves the decision models), and +//! nothing here touches the chat API. Route health is still consulted: every upstream send +//! claims a probe on its route and reports its outcome, so an open circuit breaker rejects //! requests up front and our failures feed the shared health state. //! -//! Probabilities are calibrated by default: raw one-token probabilities are overconfident, -//! so the log-probabilities are divided by the reference benchmark's temperature for the -//! number of options (`temperature: 1` in the request keeps them raw). Each answer reports +//! GLM-5.3-Flash's probabilities are calibrated by default: raw one-token probabilities are +//! overconfident, so the log-probabilities are divided by the reference benchmark's +//! temperature for the number of options (`temperature: 1` in the request keeps them raw). Each answer reports //! the temperature used and `option_mass`, the probability the model put on the option //! tokens before the mask (low means the prompt did not fit the model). //! //! Limits mirror TypeSafe's where the backing model allows: 64 questions per request, 255 -//! options per choice (labelled with single-token numbers, see `OPTION_NUMBERS`), 10 score -//! levels, 32k-token state. +//! options per choice (labelled with single-token numbers, see `OPTION_NUMBERS`; 26 on the +//! decision models), 10 score levels, 32k-token state. //! //! Billing and failure policy: usage is published per upstream request as it completes, -//! exactly as chat does. The first failing question stops the remaining ones and fails the +//! exactly as chat does (one request per question on GLM-5.3-Flash, one per call on the +//! decision models). The first failing question stops the remaining ones and fails the //! whole request; usage the provider already consumed stays billed, and no partial answers -//! are returned. A `noul` is asked as a plain two-option choice with `true` listed first, so -//! any position bias of the model always leans the same way. +//! are returned. On GLM-5.3-Flash a `noul` is asked as a plain two-option choice with `true` +//! listed first, so any position bias of the model always leans the same way. use crate::inference::health::{ProbeClaimResult, ProbeLease, ShadowObservationMode}; use crate::inference::{ @@ -37,7 +47,7 @@ use crate::inference::{ InferenceAttempt, InferenceExecution, InferenceIntent, InferenceSurface, ReplaySafety, RouteIdentity, WorkloadClass, }; -use crate::model_config::GLM_5_3_FLASH_MODEL_ID; +use crate::model_config::{decision_model, DecisionModel, GLM_5_3_FLASH_MODEL_ID}; use crate::models::users::User; use crate::provider_cache::CacheNamespaceRoot; use crate::provider_client::{ProviderRequest, ProviderRequestError}; @@ -76,11 +86,14 @@ use std::time::{Duration, Instant}; use tokio::sync::Semaphore; use tracing::{debug, error, warn}; +mod decisions; + const SYSTEM_ONE_PATH: &str = "/v1/systemone"; -/// The only public model id accepted in requests (and echoed in responses). The provider's -/// own id for it comes from the registry route, never from the request. -const PUBLIC_MODEL_ID: &str = GLM_5_3_FLASH_MODEL_ID; +/// The default model, answered by the masked-logprob engine on Continuum. The decision +/// models are in `model_config`. A provider's own id for a model comes from the registry +/// route, never from the request. +const GLM_MODEL_ID: &str = GLM_5_3_FLASH_MODEL_ID; // Request limits. The state and question limits follow TypeSafe's (state plus the longest // question within 32k tokens; a 255-option choice with descriptions is several thousand @@ -101,6 +114,10 @@ const MAX_QUESTION_BYTES: usize = 512 * 1024; /// Prompt tokens the request may send upstream in total, summed over every read of every /// question (preamble excluded). const MAX_UPSTREAM_PROMPT_TOKENS: usize = 200_000; +/// Room left in a decision model's context for its own prompt around the state and the +/// question. The token counts here are cl100k estimates of a different tokenizer, so the +/// check is conservative; the provider still rejects anything over its window. +const DECISION_PROMPT_MARGIN: usize = 1_024; /// Privatemode reports at most this many `logprob_token_ids` per response; longer reads /// are split into several requests over the same masked forward pass and merged. @@ -238,7 +255,8 @@ pub enum SystemOneRequestError { EmptyQuestionName, DuplicateQuestion, QuestionTooLarge, - BadOptionCount, + /// The most options the requested model takes per choice. + BadOptionCount(usize), EmptyOptionLabel, DuplicateOption, BadLevelCount, @@ -247,6 +265,10 @@ pub enum SystemOneRequestError { ImageTooLarge, PromptBudgetExceeded, BadTemperature, + /// The requested model reads text only. + ImagesUnsupported, + /// The state plus one question exceed the requested model's context window (in tokens). + ContextExceeded(usize), } impl SystemOneRequestError { @@ -255,7 +277,8 @@ impl SystemOneRequestError { Self::StateTooLarge | Self::QuestionTooLarge | Self::ImageTooLarge - | Self::PromptBudgetExceeded => StatusCode::PAYLOAD_TOO_LARGE, + | Self::PromptBudgetExceeded + | Self::ContextExceeded(_) => StatusCode::PAYLOAD_TOO_LARGE, _ => StatusCode::UNPROCESSABLE_ENTITY, } } @@ -274,7 +297,7 @@ impl SystemOneRequestError { Self::EmptyQuestionName => "system_one_empty_question_name", Self::DuplicateQuestion => "system_one_duplicate_question", Self::QuestionTooLarge => "system_one_question_too_large", - Self::BadOptionCount => "system_one_bad_option_count", + Self::BadOptionCount(_) => "system_one_bad_option_count", Self::EmptyOptionLabel => "system_one_empty_option_label", Self::DuplicateOption => "system_one_duplicate_option", Self::BadLevelCount => "system_one_bad_level_count", @@ -283,6 +306,8 @@ impl SystemOneRequestError { Self::ImageTooLarge => "system_one_image_too_large", Self::PromptBudgetExceeded => "system_one_prompt_budget_exceeded", Self::BadTemperature => "system_one_bad_temperature", + Self::ImagesUnsupported => "system_one_images_unsupported", + Self::ContextExceeded(_) => "system_one_context_exceeded", } } @@ -293,7 +318,9 @@ impl SystemOneRequestError { Self::MissingField(detail) => Cow::Owned(format!("Missing field {detail}.")), Self::UnknownQuestionType => fixed("Question type must be `noul`, `choice` or `score`."), Self::MalformedBody => fixed("The request body does not match the System One schema."), - Self::UnsupportedModel => fixed("Only the glm-5-3-flash model is supported."), + Self::UnsupportedModel => fixed( + "model must be glm-5-3-flash, decision-small, decision-medium or decision-large.", + ), Self::MissingState => fixed("A non-null state is required."), Self::StateTooLarge => fixed("The state exceeds 32,000 tokens or 1 MiB."), Self::NoQuestions => fixed("At least one question is required."), @@ -301,7 +328,9 @@ impl SystemOneRequestError { Self::EmptyQuestionName => fixed("Question names must be non-empty strings."), Self::DuplicateQuestion => fixed("Question names must be unique."), Self::QuestionTooLarge => fixed("A question with its criteria exceeds 16,000 tokens or 512 KiB."), - Self::BadOptionCount => fixed("A choice needs between 2 and 255 options."), + Self::BadOptionCount(max) => { + Cow::Owned(format!("A choice needs between 2 and {max} options.")) + } Self::EmptyOptionLabel => fixed("Option labels must be non-empty strings."), Self::DuplicateOption => fixed("Option labels within a question must be unique."), Self::BadLevelCount => fixed("A score needs between 2 and 10 levels."), @@ -312,6 +341,12 @@ impl SystemOneRequestError { "The questions and state together exceed the 200,000 upstream prompt tokens one request may use. Send fewer questions or a smaller state.", ), Self::BadTemperature => fixed("temperature must be a positive finite number."), + Self::ImagesUnsupported => { + fixed("This model reads text only; send images to glm-5-3-flash.") + } + Self::ContextExceeded(window) => Cow::Owned(format!( + "The state and one question together exceed this model's {window}-token context." + )), } } } @@ -492,8 +527,43 @@ impl Lowered { } } +/// The model a request asked for, which decides the engine that answers it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SystemOneModel { + /// GLM-5.3-Flash's masked logprob read (`GLM_MODEL_ID`, the default). + Glm, + /// One of Tinfoil's decision models. + Decision(DecisionModel), +} + +impl SystemOneModel { + fn resolve(model: Option<&str>) -> Result { + match model { + None | Some(GLM_MODEL_ID) => Ok(Self::Glm), + Some(model) => decision_model(model) + .map(Self::Decision) + .ok_or(SystemOneRequestError::UnsupportedModel), + } + } + + fn id(self) -> &'static str { + match self { + Self::Glm => GLM_MODEL_ID, + Self::Decision(model) => model.id, + } + } + + fn max_choice_options(self) -> usize { + match self { + Self::Glm => MAX_CHOICE_OPTIONS, + Self::Decision(model) => model.max_choice_options, + } + } +} + /// Everything the handler has checked before any billing lookup or upstream request. struct Prepared { + model: SystemOneModel, questions: Vec, temperature: Option, } @@ -516,18 +586,15 @@ fn render_description(value: &Value) -> Option { fn prepare_request(request: &SystemOneRequest) -> Result { use SystemOneRequestError as E; - if request - .model - .as_deref() - .is_some_and(|model| model != PUBLIC_MODEL_ID) - { - return Err(E::UnsupportedModel); - } + let model = SystemOneModel::resolve(request.model.as_deref())?; if let Some(temperature) = request.temperature { if !(temperature.is_finite() && temperature > 0.0) { return Err(E::BadTemperature); } } + if !request.images.is_empty() && matches!(model, SystemOneModel::Decision(_)) { + return Err(E::ImagesUnsupported); + } if request.images.len() > MAX_IMAGES { return Err(E::TooManyImages); } @@ -557,7 +624,10 @@ fn prepare_request(request: &SystemOneRequest) -> Result MAX_STATE_TOKENS { return Err(E::StateTooLarge); } - let questions = lower_questions(&request.questions)?; + let questions = lower_questions(&request.questions, model.max_choice_options())?; + if let SystemOneModel::Decision(_) = model { + check_decision_levels(&questions)?; + } let mut budget = 0usize; for question in &questions { @@ -570,21 +640,54 @@ fn prepare_request(request: &SystemOneRequest) -> Result MAX_QUESTION_TOKENS { return Err(E::QuestionTooLarge); } - // Each read re-sends the whole prompt: the question block leads it and closes it. - let per_read = state_tokens + 2 * question_tokens; - budget = budget - .saturating_add(per_read.saturating_mul(plan_reads(question.options.len()).len())); + let prompt_tokens = match model { + // Each read re-sends the whole prompt: the question block leads it and closes it. + SystemOneModel::Glm => (state_tokens + 2 * question_tokens) + .saturating_mul(plan_reads(question.options.len()).len()), + // One read per question, over the state and the question; the provider counts + // (and bills) the state again for every question. + SystemOneModel::Decision(decision) => { + let prompt = state_tokens + question_tokens; + if prompt + DECISION_PROMPT_MARGIN > decision.context_window { + return Err(E::ContextExceeded(decision.context_window)); + } + prompt + } + }; + budget = budget.saturating_add(prompt_tokens); } if budget > MAX_UPSTREAM_PROMPT_TOKENS { return Err(E::PromptBudgetExceeded); } Ok(Prepared { + model, questions, temperature: request.temperature, }) } -fn lower_questions(questions: &Entries) -> Result, SystemOneRequestError> { +/// A decision model shows each score level by its text alone, so, as d3 requires, every +/// level needs a distinct non-blank text. GLM numbers its levels and takes any text. +fn check_decision_levels(questions: &[Lowered]) -> Result<(), SystemOneRequestError> { + for question in questions.iter().filter(|q| q.kind == Kind::Score) { + let mut seen = std::collections::HashSet::with_capacity(question.options.len()); + for (_, text) in &question.options { + let text = text.as_deref().unwrap_or_default(); + if text.trim().is_empty() { + return Err(SystemOneRequestError::EmptyOptionLabel); + } + if !seen.insert(text) { + return Err(SystemOneRequestError::DuplicateOption); + } + } + } + Ok(()) +} + +fn lower_questions( + questions: &Entries, + max_choice_options: usize, +) -> Result, SystemOneRequestError> { use SystemOneRequestError as E; if questions.0.is_empty() { @@ -633,8 +736,8 @@ fn lower_questions(questions: &Entries) -> Result, System instructions, criteria, } => { - if criteria.0.len() < MIN_OPTIONS || criteria.0.len() > MAX_CHOICE_OPTIONS { - return Err(E::BadOptionCount); + if criteria.0.len() < MIN_OPTIONS || criteria.0.len() > max_choice_options { + return Err(E::BadOptionCount(max_choice_options)); } let mut labels = std::collections::HashSet::with_capacity(criteria.0.len()); for (label, _) in &criteria.0 { @@ -1092,8 +1195,9 @@ fn answer_body(lowered: &Lowered, probabilities: &[f64], temperature: f64, mass: // Upstream calls // --------------------------------------------------------------------------- -/// What every question of one request shares. -struct SystemOneCall<'a> { +/// What every upstream request of one System One request shares: the caller, how usage is +/// billed and cached, and the route whose health each send claims and reports. +struct Upstream<'a> { state: &'a Arc, user: &'a User, billing_context: &'a BillingContext, @@ -1101,6 +1205,11 @@ struct SystemOneCall<'a> { proxy: &'a ProxyConfig, route: &'a RouteIdentity, execution: InferenceExecution, +} + +/// What every question of one GLM-5.3-Flash request shares. +struct SystemOneCall<'a> { + upstream: Upstream<'a>, provider_model_id: &'a str, state_raw: &'a RawValue, images: &'a [String], @@ -1194,7 +1303,7 @@ impl SystemOneCall<'_> { match decode_read(&outcome.body, read_ids) { Ok(values) => { logprobs[read].copy_from_slice(&values); - self.observe( + self.upstream.observe( AttemptTerminal::Completed { attempt: outcome.attempt, evidence: CompletionEvidence::NonStreamingResponse, @@ -1213,7 +1322,7 @@ impl SystemOneCall<'_> { AttemptFailureKind::InvalidResponse, AttemptStage::ResponseBody, ); - self.observe( + self.upstream.observe( AttemptTerminal::Failed { attempt: outcome.attempt, failure, @@ -1240,9 +1349,7 @@ impl SystemOneCall<'_> { Ok((question.index, question.name, answer, totals)) } - /// One masked single-token completion: claims the route's health probe, sends, bills - /// the usage, and returns the parsed body. Failures are reported to route health here; - /// successes are reported by the caller once the body has decoded. + /// One masked single-token completion, sent and billed through `Upstream::send`. async fn send_read( &self, question: &Lowered, @@ -1250,6 +1357,33 @@ impl SystemOneCall<'_> { allowed: &[u32], read: &[u32], totals: &mut UsageTotals, + ) -> Result { + let body = build_upstream_body(self.provider_model_id, prompt, self.images, allowed, read); + self.upstream + .send( + "/v1/chat/completions", + body, + question.index, + totals, + extract_usage, + ) + .await + } +} + +impl Upstream<'_> { + /// One upstream POST of `body` to `path`: claims the route's health probe, sends with + /// the provider-managed fields applied, bills the usage `usage_of` reads from the + /// response, and returns the parsed body. Failures are reported to route health here; + /// successes are reported by the caller once the body has decoded. `index` is the + /// question the logs name. + async fn send( + &self, + path: &str, + mut body: Map, + index: usize, + totals: &mut UsageTotals, + usage_of: fn(&Value) -> Option, ) -> Result { let _permit = UPSTREAM_PERMITS .acquire() @@ -1267,15 +1401,13 @@ impl SystemOneCall<'_> { } => { debug!( "system one route unavailable before send: question={} reason={:?}", - question.index, reason + index, reason ); return Err(probe_api_error(retry_after)); } }; let attempt = self.execution.begin_attempt(self.route.clone()); - let mut body = - build_upstream_body(self.provider_model_id, prompt, self.images, allowed, read); apply_provider_managed_request_fields( &mut body, &self.proxy.provider_name, @@ -1291,7 +1423,7 @@ impl SystemOneCall<'_> { .provider_client .send( self.proxy, - ProviderRequest::new(Method::POST, "/v1/chat/completions", READ_TIMEOUT) + ProviderRequest::new(Method::POST, path, READ_TIMEOUT) .content_type("application/json") .body(payload), ) @@ -1303,7 +1435,7 @@ impl SystemOneCall<'_> { Err(e) => { error!( "Failed to read system one upstream response: question={}: {e}", - question.index + index ); let (failure, api_error) = local_failure( AttemptFailureKind::ResponseBody, @@ -1315,7 +1447,7 @@ impl SystemOneCall<'_> { }; match serde_json::from_slice::(&bytes) { Ok(body) => { - let usage = extract_usage(&body).unwrap_or(CompletionUsage { + let usage = usage_of(&body).unwrap_or(CompletionUsage { prompt_tokens: 0, completion_tokens: 0, cached_prompt_tokens: None, @@ -1338,7 +1470,7 @@ impl SystemOneCall<'_> { Err(e) => { error!( "Failed to parse system one upstream response: question={}: {}", - question.index, + index, crate::log_redaction::JsonErrorSummary(&e) ); let (failure, api_error) = local_failure( @@ -1356,7 +1488,7 @@ impl SystemOneCall<'_> { Err(error) => { error!( "system one upstream request failed: question={}: {error:?}", - question.index + index ); error } @@ -1385,7 +1517,7 @@ impl SystemOneCall<'_> { /// logprob read. fn continuum_route(state: &AppState) -> Result<(&'static ModelRouteSpec, ProxyConfig), ApiError> { let route = PROVIDER_REGISTRY - .completion_model(PUBLIC_MODEL_ID) + .completion_model(GLM_MODEL_ID) .and_then(|model| { model .routes @@ -1393,7 +1525,7 @@ fn continuum_route(state: &AppState) -> Result<(&'static ModelRouteSpec, ProxyCo .find(|route| route.provider == ProviderId::Continuum && route.enabled) }); let Some(route) = route else { - error!("system one unavailable: no enabled Continuum route for {PUBLIC_MODEL_ID}"); + error!("system one unavailable: no enabled Continuum route for {GLM_MODEL_ID}"); return Err(ApiError::ServiceUnavailable); }; let Some(proxy) = state.proxy_router.continuum_proxy() else { @@ -1403,6 +1535,27 @@ fn continuum_route(state: &AppState) -> Result<(&'static ModelRouteSpec, ProxyCo Ok((route, proxy)) } +/// The registry's enabled Tinfoil route for a decision model, and the attested in-process +/// Tinfoil client's proxy configuration. Like Continuum's, the route is fixed per model. +fn decision_route( + state: &AppState, + model: DecisionModel, +) -> Result<(&'static ModelRouteSpec, ProxyConfig), ApiError> { + let route = PROVIDER_REGISTRY.decision_model(model.id).and_then(|spec| { + spec.routes + .iter() + .find(|route| route.provider == ProviderId::Tinfoil && route.enabled) + }); + let Some(route) = route else { + error!( + "system one unavailable: no enabled Tinfoil route for {}", + model.id + ); + return Err(ApiError::ServiceUnavailable); + }; + Ok((route, state.proxy_router.get_tinfoil_proxy())) +} + /// Shared outcome of checking `NUMBER_TOKEN_IDS` against the serving tokenizer. Its lock is /// held only to read or store a verdict, never across the probe: callers that arrive while /// a probe is in flight go by the last verdict instead of queueing behind it. @@ -1667,54 +1820,89 @@ async fn system_one( "system one", ) .await?; - ensure_completion_model_access(PUBLIC_MODEL_ID, model_plan)?; - - let (route_spec, proxy) = continuum_route(&state)?; - verify_number_tokens(&state, &proxy, route_spec.provider_model_id).await?; - let route = RouteIdentity::new( - ProviderId::Continuum, - PUBLIC_MODEL_ID, - route_spec.provider_model_id, - route_spec.response_model_id, - RouteSelectionSource::StaticSplit, - None, - ); - let billing_context = BillingContext::new(auth_method, PUBLIC_MODEL_ID.to_string()); - let call = SystemOneCall { - state: &state, - user: &user, - billing_context: &billing_context, - cache_policy: &cache_policy, - proxy: &proxy, - route: &route, - execution: InferenceIntent::new( - user.uuid, - PUBLIC_MODEL_ID, - PUBLIC_MODEL_ID, - model_plan, - InferenceSurface::Internal, - WorkloadClass::Interactive, - ) - .begin_execution(), - provider_model_id: route_spec.provider_model_id, - state_raw: &request.state, - images: &request.images, - temperature: prepared.temperature, - }; - let (answers, totals) = - tokio::time::timeout(REQUEST_DEADLINE, call.answer_all(prepared.questions)) + let model_id = prepared.model.id(); + ensure_completion_model_access(model_id, model_plan)?; + let billing_context = BillingContext::new(auth_method, model_id.to_string()); + let execution = InferenceIntent::new( + user.uuid, + model_id, + model_id, + model_plan, + InferenceSurface::Internal, + WorkloadClass::Interactive, + ) + .begin_execution(); + + let answered = match prepared.model { + SystemOneModel::Glm => { + let (route_spec, proxy) = continuum_route(&state)?; + verify_number_tokens(&state, &proxy, route_spec.provider_model_id).await?; + let route = RouteIdentity::new( + ProviderId::Continuum, + GLM_MODEL_ID, + route_spec.provider_model_id, + route_spec.response_model_id, + RouteSelectionSource::StaticSplit, + None, + ); + let call = SystemOneCall { + upstream: Upstream { + state: &state, + user: &user, + billing_context: &billing_context, + cache_policy: &cache_policy, + proxy: &proxy, + route: &route, + execution, + }, + provider_model_id: route_spec.provider_model_id, + state_raw: &request.state, + images: &request.images, + temperature: prepared.temperature, + }; + tokio::time::timeout(REQUEST_DEADLINE, call.answer_all(prepared.questions)).await + } + SystemOneModel::Decision(model) => { + let (route_spec, proxy) = decision_route(&state, model)?; + let route = RouteIdentity::new( + ProviderId::Tinfoil, + model.id, + route_spec.provider_model_id, + route_spec.response_model_id, + RouteSelectionSource::StaticSplit, + None, + ); + let call = decisions::DecisionCall { + upstream: Upstream { + state: &state, + user: &user, + billing_context: &billing_context, + cache_policy: &cache_policy, + proxy: &proxy, + route: &route, + execution, + }, + provider_model_id: route_spec.provider_model_id, + temperature: prepared.temperature, + }; + tokio::time::timeout( + REQUEST_DEADLINE, + call.answer_all(&request, prepared.questions), + ) .await - .map_err(|_| { - error!( - "system one request exceeded its {}s deadline", - REQUEST_DEADLINE.as_secs() - ); - deadline_error() - })??; + } + }; + let (answers, totals) = answered.map_err(|_| { + error!( + "system one request exceeded its {}s deadline", + REQUEST_DEADLINE.as_secs() + ); + deadline_error() + })??; let response = ResponseBody { id: format!("so_{}", uuid::Uuid::new_v4()), - model: PUBLIC_MODEL_ID, + model: model_id, answers, usage: totals, }; @@ -1843,7 +2031,7 @@ mod tests { ), ( json!({"state": "s", "questions": {"q": {"type": "choice", "instructions": "x", "criteria": {"only": null}}}}), - E::BadOptionCount, + E::BadOptionCount(255), ), ( json!({"state": "s", "questions": {"q": {"type": "choice", "instructions": "x", "criteria": {" ": null, "b": null}}}}), @@ -2223,7 +2411,7 @@ mod tests { }; let body = ResponseBody { id: "so_test".into(), - model: PUBLIC_MODEL_ID, + model: GLM_MODEL_ID, answers: Entries(vec![ ("zebra".into(), answer_body(&choice, &[0.6, 0.4], 1.0, 0.9)), ("apple".into(), answer_body(&choice, &[0.1, 0.9], 1.0, 0.9)), diff --git a/services/opensecret/src/web/system_one/decisions.rs b/services/opensecret/src/web/system_one/decisions.rs new file mode 100644 index 000000000..265bc1823 --- /dev/null +++ b/services/opensecret/src/web/system_one/decisions.rs @@ -0,0 +1,846 @@ +//! Tinfoil's decision models behind `/v1/systemone`: `decision-small`, `decision-medium` and +//! `decision-large` (vLLM-SR d3 checkpoints in Tinfoil's attested enclaves, served as OpenAI's +//! `POST /v1/decisions`). The System One request and response keep their shape; this module +//! translates both ways. One upstream request answers every question, with no prompt, prefill, +//! token table or calibration of ours: d3 lays out its own prompt and reports the +//! probabilities of a trained classification head. +//! +//! Where the two shapes differ, the translation follows d3's own reading of a System One +//! question (d3_format.py and vLLM's `D3Strategy`): +//! - a `noul` without criteria is a `predicate`; one with criteria is a boolean `choice` +//! listing `false` first, each option described by its criterion or by d3's default +//! (`No / false`, `Yes / true`), as d3 lays out a described noul; +//! - a `choice` keeps its labels as values and its descriptions; a `score` sends its level +//! texts as labels, in order; +//! - a non-string state, instruction or description is sent as JSON text with Python's +//! separators, the way d3 renders one; a null instruction is sent empty so d3 asks its +//! default question, and an empty state is sent as d3's `(empty)`. +//! +//! The answers then take the same shapes as GLM-5.3-Flash's: probabilities in option order, +//! the `temperature` (1 unless the request sets one, applied as `softmax(ln p / T)`), +//! TypeSafe's confidence, and an `option_mass` of 1 (a classification head puts all of its +//! probability on the offered options; vLLM reports the same `label_mass`). + +use super::{ + answer_body, calibrated_probabilities, local_failure, render_description, to_python_json, + Answer, Entries, Lowered, Question, SystemOneRequest, Upstream, UsageTotals, +}; +use crate::inference::{AttemptFailureKind, AttemptStage, AttemptTerminal, CompletionEvidence}; +use crate::web::openai::CompletionUsage; +use crate::ApiError; +use serde_json::value::RawValue; +use serde_json::{json, Map, Value}; +use tracing::{debug, error}; + +const DECISIONS_PATH: &str = "/v1/decisions"; + +/// d3's texts for the options of a `noul` whose criteria do not describe them. +const NOUL_FALSE: &str = "No / false"; +const NOUL_TRUE: &str = "Yes / true"; +/// d3's text for an empty state. +const EMPTY_STATE: &str = "(empty)"; + +/// How a question was asked upstream, to read its answer back in `Lowered` option order. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Asked { + /// A `noul` without criteria: one probability that it is true. + Predicate, + /// A `noul` with criteria: a boolean choice, `false` first. + Noul, + Choice, + Score, +} + +/// What one decision-model request shares. +pub(super) struct DecisionCall<'a> { + pub(super) upstream: Upstream<'a>, + pub(super) provider_model_id: &'a str, + pub(super) temperature: Option, +} + +impl DecisionCall<'_> { + /// Answers every question of `request` (lowered and validated as `questions`) with one + /// upstream request. Usage is billed once the provider answers, even if the answers + /// cannot be read; no partial answers are returned. + pub(super) async fn answer_all( + &self, + request: &SystemOneRequest, + questions: Vec, + ) -> Result<(Entries, UsageTotals), ApiError> { + let (body, asked) = build_request(self.provider_model_id, request).map_err(|e| { + error!( + "Failed to render system one decisions request: {}", + crate::log_redaction::JsonErrorSummary(&e) + ); + ApiError::InternalServerError + })?; + let mut totals = UsageTotals::default(); + let outcome = self + .upstream + .send(DECISIONS_PATH, body, 0, &mut totals, decision_usage) + .await?; + let distributions = match read_answers(&outcome.body, &asked, &questions) { + Ok(distributions) => { + self.upstream.observe( + AttemptTerminal::Completed { + attempt: outcome.attempt, + evidence: CompletionEvidence::NonStreamingResponse, + }, + outcome.probe, + ); + distributions + } + Err(read_error) => { + error!("system one decisions answer unusable: {read_error:?}"); + let (failure, api_error) = local_failure( + AttemptFailureKind::InvalidResponse, + AttemptStage::ResponseBody, + ); + self.upstream.observe( + AttemptTerminal::Failed { + attempt: outcome.attempt, + failure, + }, + outcome.probe, + ); + return Err(api_error); + } + }; + let temperature = self.temperature.unwrap_or(1.0); + let answers = questions + .iter() + .zip(distributions) + .map(|(question, probabilities)| { + ( + question.name.clone(), + answer(question, &probabilities, temperature), + ) + }) + .collect(); + debug!( + "system one answered {} questions with {}", + questions.len(), + self.provider_model_id + ); + Ok((Entries(answers), totals)) + } +} + +/// The System One answer for one question from the provider's probabilities (in `Lowered` +/// option order), with the temperature applied as `softmax(ln p / T)`: `T = 1` renormalizes +/// the provider's distribution and leaves it as it is. +fn answer(question: &Lowered, probabilities: &[f64], temperature: f64) -> Answer { + let log_probabilities: Vec = probabilities.iter().map(|p| p.ln()).collect(); + let probabilities = calibrated_probabilities(&log_probabilities, temperature); + answer_body(question, &probabilities, temperature, 1.0) +} + +// --------------------------------------------------------------------------- +// Request +// --------------------------------------------------------------------------- + +/// The `/v1/decisions` body for `request`, whose questions `prepare_request` has validated, +/// and how each question was asked. +fn build_request( + provider_model_id: &str, + request: &SystemOneRequest, +) -> Result<(Map, Vec), serde_json::Error> { + let mut questions = Vec::with_capacity(request.questions.0.len()); + let mut asked = Vec::with_capacity(request.questions.0.len()); + for (name, question) in &request.questions.0 { + let (value, how) = decision_question(name, question)?; + questions.push(value); + asked.push(how); + } + let mut body = Map::new(); + body.insert("model".into(), json!(provider_model_id)); + body.insert("input".into(), json!(state_input(&request.state))); + body.insert("questions".into(), Value::Array(questions)); + Ok((body, asked)) +} + +fn decision_question(name: &str, question: &Question) -> Result<(Value, Asked), serde_json::Error> { + Ok(match question { + Question::Noul { + instructions, + criteria, + } => { + let criteria = criteria.as_ref(); + let yes = criterion(criteria.and_then(|c| c.yes.as_ref()))?; + let no = criterion(criteria.and_then(|c| c.no.as_ref()))?; + if yes.is_none() && no.is_none() { + ( + json!({ + "type": "predicate", + "name": name, + "instructions": instructions_text(instructions)?, + }), + Asked::Predicate, + ) + } else { + ( + json!({ + "type": "choice", + "name": name, + "instructions": instructions_text(instructions)?, + "choices": [ + {"value": false, "description": no.as_deref().unwrap_or(NOUL_FALSE)}, + {"value": true, "description": yes.as_deref().unwrap_or(NOUL_TRUE)}, + ], + }), + Asked::Noul, + ) + } + } + Question::Choice { + instructions, + criteria, + } => { + let choices = criteria + .0 + .iter() + .map(|(label, description)| { + let mut choice = json!({ "value": label }); + if let Some(description) = criterion(Some(description))? { + choice["description"] = Value::String(description); + } + Ok(choice) + }) + .collect::, serde_json::Error>>()?; + ( + json!({ + "type": "choice", + "name": name, + "instructions": instructions_text(instructions)?, + "choices": choices, + }), + Asked::Choice, + ) + } + Question::Score { + instructions, + criteria, + } => { + let levels = criteria + .iter() + .map(|level| describe(level).map(|label| json!({ "label": label }))) + .collect::, serde_json::Error>>()?; + ( + json!({ + "type": "score", + "name": name, + "instructions": instructions_text(instructions)?, + "levels": levels, + }), + Asked::Score, + ) + } + }) +} + +/// d3's text for a value: a string as it is, anything else as JSON with Python's separators. +fn describe(value: &Value) -> Result { + match value { + Value::String(text) => Ok(text.clone()), + other => to_python_json(other), + } +} + +/// A question's instructions; empty for null, which d3 reads as its default question. +fn instructions_text(value: &Value) -> Result { + match value { + Value::Null => Ok(String::new()), + other => describe(other), + } +} + +/// A criterion's text, or `None` when the caller left it out (null or blank), by the same +/// rule GLM-5.3-Flash's lowering uses. +fn criterion(value: Option<&Value>) -> Result, serde_json::Error> { + match value { + Some(value) if render_description(value).is_some() => describe(value).map(Some), + _ => Ok(None), + } +} + +/// The state as d3 renders it: a JSON string as its text (`(empty)` when empty), anything +/// else as the caller's JSON in the caller's key order with Python's separators (`", "` and +/// `": "`) in place of insignificant whitespace. +fn state_input(state: &RawValue) -> String { + let text = state.get().trim(); + if text.starts_with('"') { + if let Ok(text) = serde_json::from_str::(text) { + return if text.is_empty() { + EMPTY_STATE.to_string() + } else { + text + }; + } + } + let mut out = String::with_capacity(text.len() + text.len() / 8); + let mut in_string = false; + let mut escaped = false; + for c in text.chars() { + if in_string { + out.push(c); + if escaped { + escaped = false; + } else if c == '\\' { + escaped = true; + } else if c == '"' { + in_string = false; + } + continue; + } + match c { + '"' => { + in_string = true; + out.push(c); + } + ',' => out.push_str(", "), + ':' => out.push_str(": "), + c if c.is_ascii_whitespace() => {} + c => out.push(c), + } + } + out +} + +// --------------------------------------------------------------------------- +// Response +// --------------------------------------------------------------------------- + +#[derive(Debug, PartialEq, Eq)] +enum ReadError { + /// No `answers` array, or not one answer per question. + Answers, + /// The model declined the question at this position. + Refused(usize), + /// The answer at this position does not have the shape of what was asked. + Shape(usize), +} + +/// The probabilities of each answer, in `Lowered` option order (a `noul` is true, then +/// false). Answers are matched by position and must echo what was asked: the type, the +/// option values in order, and one finite non-negative probability per option. +fn read_answers( + body: &Value, + asked: &[Asked], + questions: &[Lowered], +) -> Result>, ReadError> { + let answers = body + .get("answers") + .and_then(Value::as_array) + .filter(|answers| answers.len() == asked.len() && answers.len() == questions.len()) + .ok_or(ReadError::Answers)?; + answers + .iter() + .zip(asked) + .zip(questions) + .enumerate() + .map(|(index, ((answer, how), question))| { + read_answer(answer, *how, question).map_err(|refused| { + if refused { + ReadError::Refused(index) + } else { + ReadError::Shape(index) + } + }) + }) + .collect() +} + +/// One answer's probabilities; `Err(true)` for a refusal, `Err(false)` for any other shape. +fn read_answer(answer: &Value, how: Asked, question: &Lowered) -> Result, bool> { + let kind = answer.get("type").and_then(Value::as_str); + if kind == Some("refusal") { + return Err(true); + } + let expected = match how { + Asked::Predicate => "predicate", + Asked::Noul | Asked::Choice => "choice", + Asked::Score => "score", + }; + if kind != Some(expected) { + return Err(false); + } + let probabilities = match how { + Asked::Predicate => { + let p = answer + .get("probability") + .and_then(Value::as_f64) + .filter(|p| (0.0..=1.0).contains(p)) + .ok_or(false)?; + vec![p, 1.0 - p] + } + Asked::Noul => { + // Asked as [false, true]; the answer reads true first. + let read = option_probabilities(answer, &[json!(false), json!(true)])?; + vec![read[1], read[0]] + } + Asked::Choice => { + let values: Vec = question + .options + .iter() + .map(|(label, _)| json!(label)) + .collect(); + option_probabilities(answer, &values)? + } + Asked::Score => { + let values: Vec = (0..question.options.len()).map(|i| json!(i)).collect(); + option_probabilities(answer, &values)? + } + }; + if probabilities.len() != question.options.len() + || probabilities.iter().any(|p| !p.is_finite() || *p < 0.0) + || probabilities.iter().sum::() <= 0.0 + { + return Err(false); + } + Ok(probabilities) +} + +/// The `probabilities` of a choice or score answer, which must list exactly `values` in order. +fn option_probabilities(answer: &Value, values: &[Value]) -> Result, bool> { + let entries = answer + .get("probabilities") + .and_then(Value::as_array) + .filter(|entries| entries.len() == values.len()) + .ok_or(false)?; + entries + .iter() + .zip(values) + .map(|(entry, value)| { + if entry.get("value") != Some(value) { + return Err(false); + } + entry + .get("probability") + .and_then(Value::as_f64) + .ok_or(false) + }) + .collect() +} + +/// Usage as Tinfoil reports it: every question's prompt counted (each one re-reads the +/// state) and no output tokens. +fn decision_usage(body: &Value) -> Option { + let usage = body.get("usage")?; + let tokens = |value: Option<&Value>| { + value + .and_then(Value::as_i64) + .filter(|tokens| *tokens >= 0) + .map(|tokens| tokens.min(i64::from(i32::MAX)) as i32) + }; + let prompt_tokens = tokens(usage.get("input_tokens"))?; + Some(CompletionUsage { + prompt_tokens, + completion_tokens: tokens(usage.get("output_tokens")).unwrap_or(0), + cached_prompt_tokens: tokens( + usage + .get("input_tokens_details") + .and_then(|details| details.get("cached_tokens")), + ) + .map(|cached| cached.min(prompt_tokens)), + }) +} + +#[cfg(test)] +mod tests { + use super::super::{lower_questions, parse_request, prepare_request, Kind}; + use super::*; + use crate::provider_client::{ProviderClient, ProviderRequest}; + use crate::proxy_config::ProxyConfig; + use axum::http::HeaderMap; + use axum::routing::post; + use axum::{Json, Router}; + use reqwest::Method; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + fn request(json: Value) -> SystemOneRequest { + parse_request(&json.to_string()).unwrap_or_else(|e| panic!("{e}")) + } + + /// A request from JSON text, keeping its key order (`json!` would sort the keys). + fn request_text(text: &str) -> SystemOneRequest { + parse_request(text).unwrap_or_else(|e| panic!("{e}")) + } + + fn lowered(request: &SystemOneRequest) -> Vec { + lower_questions(&request.questions, 26).unwrap() + } + + fn answer_json(answer: &Answer) -> Value { + serde_json::to_value(answer).unwrap() + } + + #[test] + fn questions_are_asked_as_d3_reads_them() { + let request = request_text( + r#"{ + "model": "decision-medium", + "state": {"ticket": "Charged twice", "meta": {"b": 1, "a": [1, 2]}}, + "questions": { + "urgent": {"type": "noul", "instructions": "Is it urgent?"}, + "churn": { + "type": "noul", + "instructions": "Will they leave?", + "criteria": {"true": "they say they will cancel", "false": ""} + }, + "intent": { + "type": "choice", + "instructions": {"ask": "What do they want?"}, + "criteria": {"refund": "money back", "question": null, "praise": {"tone": "thanks"}} + }, + "severity": {"type": "score", "instructions": null, "criteria": ["low", {"level": "high"}]} + } + }"#, + ); + let (body, asked) = build_request("decision-medium", &request).unwrap(); + assert_eq!( + asked, + [Asked::Predicate, Asked::Noul, Asked::Choice, Asked::Score] + ); + assert_eq!( + Value::Object(body), + json!({ + "model": "decision-medium", + "input": "{\"ticket\": \"Charged twice\", \"meta\": {\"b\": 1, \"a\": [1, 2]}}", + "questions": [ + {"type": "predicate", "name": "urgent", "instructions": "Is it urgent?"}, + {"type": "choice", "name": "churn", "instructions": "Will they leave?", + "choices": [ + {"value": false, "description": "No / false"}, + {"value": true, "description": "they say they will cancel"} + ]}, + {"type": "choice", "name": "intent", "instructions": "{\"ask\": \"What do they want?\"}", + "choices": [ + {"value": "refund", "description": "money back"}, + {"value": "question"}, + {"value": "praise", "description": "{\"tone\": \"thanks\"}"} + ]}, + {"type": "score", "name": "severity", "instructions": "", + "levels": [{"label": "low"}, {"label": "{\"level\": \"high\"}"}]} + ] + }) + ); + } + + #[test] + fn state_keeps_caller_order_and_takes_python_separators() { + let raw = |text: &str| RawValue::from_string(text.to_string()).unwrap(); + assert_eq!( + state_input(&raw( + "{\n \"z\" : 1,\n \"a\":[true ,null],\"s\":\"x, y: z\\\" ,:\"\n}" + )), + "{\"z\": 1, \"a\": [true, null], \"s\": \"x, y: z\\\" ,:\"}" + ); + assert_eq!( + state_input(&raw("\"plain text\\nline\"")), + "plain text\nline" + ); + assert_eq!(state_input(&raw("\"\"")), "(empty)"); + assert_eq!(state_input(&raw("[ ]")), "[]"); + assert_eq!(state_input(&raw("42")), "42"); + } + + /// Four questions, one of each way of asking, in this order. + const FOUR_QUESTIONS: &str = r#"{ + "model": "decision-small", + "state": "x", + "questions": { + "plain": {"type": "noul", "instructions": "a?"}, + "described": {"type": "noul", "instructions": "b?", "criteria": {"true": "yes it is"}}, + "pick": {"type": "choice", "instructions": "c?", "criteria": {"x": null, "y": null, "z": null}}, + "rate": {"type": "score", "instructions": "d?", "criteria": ["lo", "mid", "hi"]} + } + }"#; + + fn noul_answers() -> (Vec, Vec) { + let request = request_text(FOUR_QUESTIONS); + let (_, asked) = build_request("decision-small", &request).unwrap(); + (asked, lowered(&request)) + } + + fn provider_answers() -> Value { + json!({ + "model": "decision-small", + "answers": [ + {"type": "predicate", "name": "plain", "probability": 0.75}, + {"type": "choice", "name": "described", "choice": true, "confidence": 0.9, + "probabilities": [{"value": false, "probability": 0.1}, {"value": true, "probability": 0.9}]}, + {"type": "choice", "name": "pick", "choice": "y", "confidence": 0.5, + "probabilities": [{"value": "x", "probability": 0.2}, {"value": "y", "probability": 0.5}, + {"value": "z", "probability": 0.3}]}, + {"type": "score", "name": "rate", "score": 1.4, "confidence": 0.5, + "probabilities": [{"value": 0, "label": "lo", "probability": 0.1}, + {"value": 1, "label": "mid", "probability": 0.4}, + {"value": 2, "label": "hi", "probability": 0.5}]} + ], + "usage": {"input_tokens": 452, "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 452} + }) + } + + #[test] + fn answers_take_the_system_one_shapes() { + let (asked, questions) = noul_answers(); + let distributions = read_answers(&provider_answers(), &asked, &questions).unwrap(); + assert_eq!( + distributions, + [ + vec![0.75, 0.25], + vec![0.9, 0.1], + vec![0.2, 0.5, 0.3], + vec![0.1, 0.4, 0.5] + ] + ); + let answers: Vec = questions + .iter() + .zip(&distributions) + .map(|(question, probabilities)| answer_json(&answer(question, probabilities, 1.0))) + .collect(); + assert_eq!( + answers[0], + json!({"type": "noul", "noul": 0.75, "temperature": 1.0, "option_mass": 1.0}) + ); + assert_eq!(answers[1]["noul"], 0.9); + assert_eq!( + answers[2], + json!({"type": "choice", "choice": "y", "probabilities": {"x": 0.2, "y": 0.5, "z": 0.3}, + "confidence": 0.25, "temperature": 1.0, "option_mass": 1.0}) + ); + assert_eq!(answers[3]["score"], 1.4); + assert_eq!( + answers[3]["legend"], + json!({"0": "lo", "1": "mid", "2": "hi"}) + ); + assert_eq!( + answers[3]["probabilities"], + json!({"0": 0.1, "1": 0.4, "2": 0.5}) + ); + // TypeSafe's score confidence, not the provider's top probability. + assert_ne!(answers[3]["confidence"], 0.5); + } + + #[test] + fn a_requested_temperature_reshapes_without_changing_the_winner() { + let (asked, questions) = noul_answers(); + let distributions = read_answers(&provider_answers(), &asked, &questions).unwrap(); + let sharp = answer_json(&answer(&questions[2], &distributions[2], 0.5)); + assert_eq!(sharp["choice"], "y"); + assert_eq!(sharp["temperature"], 0.5); + // softmax(ln p / 0.5) = p^2 / sum p^2 = 0.25 / 0.38 + assert_eq!(sharp["probabilities"]["y"], 0.6579); + let raw = answer_json(&answer(&questions[0], &[0.6, 0.4], 2.0)); + // sqrt(0.6) / (sqrt(0.6) + sqrt(0.4)) + assert_eq!(raw["noul"], 0.5505); + } + + #[test] + fn answers_must_echo_what_was_asked() { + let (asked, questions) = noul_answers(); + let read = |body: Value| read_answers(&body, &asked, &questions); + let mut body = provider_answers(); + body["answers"].as_array_mut().unwrap().pop(); + assert_eq!(read(body), Err(ReadError::Answers)); + assert_eq!(read(json!({"usage": {}})), Err(ReadError::Answers)); + + let mut body = provider_answers(); + body["answers"][1] = json!({"type": "refusal", "name": "described"}); + assert_eq!(read(body), Err(ReadError::Refused(1))); + + let mut body = provider_answers(); + body["answers"][0]["type"] = json!("choice"); + assert_eq!(read(body), Err(ReadError::Shape(0))); + + let mut body = provider_answers(); + body["answers"][0]["probability"] = json!(1.5); + assert_eq!(read(body), Err(ReadError::Shape(0))); + + let mut body = provider_answers(); + body["answers"][2]["probabilities"][1]["value"] = json!("z"); + assert_eq!(read(body), Err(ReadError::Shape(2))); + + let mut body = provider_answers(); + body["answers"][3]["probabilities"][0]["value"] = json!(2); + assert_eq!(read(body), Err(ReadError::Shape(3))); + + let mut body = provider_answers(); + body["answers"][2]["probabilities"][0]["probability"] = json!(-0.1); + assert_eq!(read(body), Err(ReadError::Shape(2))); + + let mut body = provider_answers(); + for entry in body["answers"][3]["probabilities"].as_array_mut().unwrap() { + entry["probability"] = json!(0.0); + } + assert_eq!(read(body), Err(ReadError::Shape(3))); + } + + #[test] + fn usage_counts_every_questions_prompt() { + let usage = |body: Value| { + decision_usage(&body).map(|usage| { + ( + usage.prompt_tokens, + usage.completion_tokens, + usage.cached_prompt_tokens, + ) + }) + }; + assert_eq!(usage(provider_answers()), Some((452, 0, Some(0)))); + assert_eq!(usage(json!({"answers": []})), None); + assert_eq!( + usage( + json!({"usage": {"input_tokens": 10, "input_tokens_details": {"cached_tokens": 99}}}) + ), + Some((10, 0, Some(10))) + ); + } + + #[test] + fn decision_models_take_text_and_at_most_26_options() { + use super::super::SystemOneRequestError as E; + let prepare = |json: Value| prepare_request(&request(json)).err(); + let options = |n: usize| -> Value { + Value::Object((0..n).map(|i| (format!("o{i}"), json!(null))).collect()) + }; + let choice = |n: usize| json!({"q": {"type": "choice", "instructions": "?", "criteria": options(n)}}); + for model in ["decision-small", "decision-medium", "decision-large"] { + assert_eq!( + prepare(json!({"model": model, "state": "s", "questions": choice(26)})), + None + ); + assert_eq!( + prepare(json!({"model": model, "state": "s", "questions": choice(27)})), + Some(E::BadOptionCount(26)) + ); + assert_eq!( + prepare(json!({"model": model, "state": "s", "questions": choice(2), + "images": ["data:image/png;base64,AAAA"]})), + Some(E::ImagesUnsupported) + ); + assert_eq!( + prepare(json!({"model": model, "state": "s", + "questions": {"q": {"type": "score", "instructions": "?", "criteria": ["a", " "]}}})), + Some(E::EmptyOptionLabel) + ); + assert_eq!( + prepare(json!({"model": model, "state": "s", + "questions": {"q": {"type": "score", "instructions": "?", "criteria": ["a", "a"]}}})), + Some(E::DuplicateOption) + ); + // A state that fits the 32,000-token state limit but not, with a question, the + // model's 32,768-token context. + let state = "word ".repeat(31_900); + assert_eq!( + prepare(json!({"model": model, "state": state, "questions": choice(2)})), + Some(E::ContextExceeded(32_768)) + ); + } + // The same requests stay valid on GLM-5.3-Flash. + for questions in [ + choice(255), + json!({"q": {"type": "score", "instructions": "?", "criteria": ["a", "a"]}}), + ] { + assert_eq!(prepare(json!({"state": "s", "questions": questions})), None); + } + assert_eq!( + prepare( + json!({"state": "s", "questions": choice(2), "images": ["data:image/png;base64,AAAA"]}) + ), + None + ); + assert_eq!( + prepare(json!({"model": "decision-xl", "state": "s", "questions": choice(2)})), + Some(E::UnsupportedModel) + ); + } + + #[tokio::test] + async fn sends_one_decisions_request_through_the_tinfoil_transport() { + type Seen = Arc>>; + let seen: Seen = Arc::default(); + let recorded = Arc::clone(&seen); + let router = Router::new().route( + DECISIONS_PATH, + post( + move |headers: HeaderMap, Json(body): Json| async move { + recorded.lock().unwrap().push((headers, body)); + Json(provider_answers()) + }, + ), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let client = ProviderClient::for_test(base_url.clone()).unwrap(); + let proxy = ProxyConfig { + base_url, + api_key: None, + provider_name: "tinfoil".to_owned(), + }; + + let request = request_text(FOUR_QUESTIONS); + let (mut body, asked) = build_request("decision-small", &request).unwrap(); + // As `Upstream::send` does before every send. + crate::web::openai::apply_provider_managed_request_fields( + &mut body, + &proxy.provider_name, + uuid::Uuid::nil(), + &crate::web::openai::CompletionCachePolicy::LegacyV1, + ); + let response = client + .send( + &proxy, + ProviderRequest::new(Method::POST, DECISIONS_PATH, Duration::from_secs(5)) + .content_type("application/json") + .body(serde_json::to_vec(&Value::Object(body.clone())).unwrap()), + ) + .await + .unwrap(); + assert!(response.is_success()); + let answered: Value = serde_json::from_slice(&response.bytes().await.unwrap()).unwrap(); + assert_eq!( + read_answers(&answered, &asked, &lowered(&request)) + .unwrap() + .len(), + 4 + ); + assert_eq!( + decision_usage(&answered).map(|usage| usage.prompt_tokens), + Some(452) + ); + + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 1); + let (headers, sent) = &seen[0]; + assert_eq!(headers["authorization"], "Bearer test-tinfoil-key"); + assert_eq!(headers["content-type"], "application/json"); + assert_eq!(sent, &Value::Object(body)); + // Tinfoil's decision enclave rejects fields it does not know. Its router consumes + // `user_cache_secret` (the account's cache namespace, as for chat) before forwarding; + // nothing else is sent. + let mut keys: Vec<&str> = sent + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!(keys, ["input", "model", "questions", "user_cache_secret"]); + task.abort(); + } + + #[test] + fn noul_kinds_lower_in_the_order_answers_are_read() { + let request = request( + json!({"state": "x", "questions": {"n": {"type": "noul", "instructions": "?"}}}), + ); + let questions = lowered(&request); + assert_eq!(questions[0].kind, Kind::Noul); + assert_eq!(questions[0].options[0].0, "true"); + assert_eq!(questions[0].options[1].0, "false"); + } +}