From 842f8d5070ec90489a4e932246be869262d92cac Mon Sep 17 00:00:00 2001 From: hachall Date: Thu, 22 Oct 2026 11:37:39 +0100 Subject: [PATCH 1/2] feat: overlay targets are scheduling hints, not a serving class A serving class is where a request is routed. Overlay `targets` used to make a request resolve as a `custom` class, which changed its analytics class and its price lookup: an account's standard-class price was skipped whenever its overlay carried targets. Targets now travel exactly like account-file hints: the priority and latency targets are sent to dynamo members, and the request keeps its routed class (standard) and its price. - onwards: `ServingClass` has only `standard`; the separate targets path and `ServingTargets` are gone. Hints are not sent to a named-pool member that refuses scheduling fields, matching how its `nvext` is already stripped. The request span records the hinted priority and whether targets were sent. - dwctl: the sync turns stored overlay targets into hints. Overlay `class_pricing` prices one of the model's classes (`standard`); `custom` is refused, and rows the catalog owns under it are retired on the next apply. - dashboard: pricing shows the model's classes only; overlay details label targets as scheduling hints. --- .../models/manage/ModelPricing.test.tsx | 29 +- .../features/models/manage/ModelPricing.tsx | 22 +- .../features/serving/OverlayDetails.tsx | 6 +- docs/model-serving-class-identity.md | 2 +- .../src/how-to/organization-model-overlays.md | 22 +- docs/src/reference/configuration.md | 2 +- dwctl/src/account_limits.rs | 1 - dwctl/src/org_overlays.rs | 54 +-- dwctl/src/sync/onwards_config/mod.rs | 22 +- dwctl/src/sync/onwards_config/tests.rs | 13 +- dwctl/src/test/serving_classes.rs | 11 +- dwctl/tests/org_catalog_validator.rs | 14 +- onwards/src/handlers.rs | 83 +++-- onwards/src/lib.rs | 1 - onwards/src/serving.rs | 315 ++++++------------ 15 files changed, 244 insertions(+), 353 deletions(-) diff --git a/dashboard/src/components/features/models/manage/ModelPricing.test.tsx b/dashboard/src/components/features/models/manage/ModelPricing.test.tsx index 895ba492f..a0bf5794e 100644 --- a/dashboard/src/components/features/models/manage/ModelPricing.test.tsx +++ b/dashboard/src/components/features/models/manage/ModelPricing.test.tsx @@ -53,7 +53,7 @@ const serving: OrganizationServing = { }), tariff("custom", { organization_id: "org-a", - serving_class: "custom", + serving_class: "standard", input_price_per_token: "0", output_price_per_token: "0", }), @@ -62,7 +62,7 @@ const serving: OrganizationServing = { { deployed_model_id: "model", alias: "example-model", - serving_class: "custom", + serving_class: "standard", read_multiplier: "0.25", write_multiplier_5m: "1", write_multiplier_1h: "2", @@ -188,9 +188,10 @@ describe("model pricing selector", () => { it("shows effective playground prices when only realtime is configured", () => { const { container } = mount(true, "org-a"); const playground = within(within(container).getByRole("region", { name: "Playground prices" })); - expect(playground.getByText("Bespoke · custom · realtime fallback")).toBeInTheDocument(); + expect(playground.getByText("Bespoke · standard · realtime fallback")).toBeInTheDocument(); expect(playground.getAllByText("$0")).toHaveLength(2); - expect(playground.getAllByText("$2.00")).toHaveLength(1); + // The standard-class deal wins over the organisation's all-class price. + expect(playground.queryByText("$2.00")).not.toBeInTheDocument(); }); it("starts general then shows effective class prices and inherited batch/cache values", async () => { @@ -212,14 +213,14 @@ describe("model pricing selector", () => { }), ); const rt = within(page.getByRole("region", { name: "Realtime prices" })); - expect(rt.getByText("Bespoke · custom")).toBeInTheDocument(); + expect(rt.getByText("Bespoke · standard")).toBeInTheDocument(); expect(rt.getAllByText("$0")).toHaveLength(2); - expect(rt.getAllByText("$2.00")).toHaveLength(1); + expect(rt.queryByText("$2.00")).not.toBeInTheDocument(); const batch = within( page.getByRole("region", { name: "Batch and flex prices" }), ); expect(batch.getByText("Inherited · general model")).toBeInTheDocument(); - expect(batch.getByText("Bespoke · all classes · realtime fallback")).toBeInTheDocument(); + expect(batch.getByText("Bespoke · standard · realtime fallback")).toBeInTheDocument(); expect( page.queryByRole("button", { name: "Manage Tariffs" }), ).not.toBeInTheDocument(); @@ -228,7 +229,7 @@ describe("model pricing selector", () => { ).not.toBeInTheDocument(); const cache = within(page.getByRole("region", { name: "Cache pricing" })); expect(cache.getByText("0.25×")).toBeInTheDocument(); - expect(cache.getAllByText("Inherited · general model")).toHaveLength(1); + expect(cache.queryByText("Inherited · general model")).not.toBeInTheDocument(); const sections = [...container.querySelectorAll("section[aria-label]")].map( (x) => x.getAttribute("aria-label"), ); @@ -271,7 +272,7 @@ describe("model pricing selector", () => { expect(rt.queryByText("interactive")).not.toBeInTheDocument(); expect(rt.getByText("standard")).toBeInTheDocument(); }); - it("shows custom target pricing even without a class-specific tariff", () => { + it("never adds a pricing class for overlay targets", () => { vi.mocked(useOrganizationServing).mockReturnValue({ data: { ...serving, @@ -286,11 +287,11 @@ describe("model pricing selector", () => { isError: false, } as unknown as ReturnType); const { container } = mount(true, "org-a"); - expect( - within( - within(container).getByRole("region", { name: "Realtime prices" }), - ).getByText("custom"), - ).toBeInTheDocument(); + const rt = within( + within(container).getByRole("region", { name: "Realtime prices" }), + ); + expect(rt.queryByText("custom")).not.toBeInTheDocument(); + expect(rt.getByText("standard")).toBeInTheDocument(); }); it.each(["loading", "error"])( "never disguises %s organisation data as general prices", diff --git a/dashboard/src/components/features/models/manage/ModelPricing.tsx b/dashboard/src/components/features/models/manage/ModelPricing.tsx index a18570431..53dee38ca 100644 --- a/dashboard/src/components/features/models/manage/ModelPricing.tsx +++ b/dashboard/src/components/features/models/manage/ModelPricing.tsx @@ -89,25 +89,9 @@ export function ModelPricing({ const orgCache = (organization ? (own.data?.cache_tariffs ?? []) : []).filter( (row) => row.deployed_model_id === model.id, ); - const selectedOverlay = own.data?.overlays.find( - (row) => row.deployed_model_id === model.id, - ); - // Billing resolves to `standard`, or `custom` when the overlay's explicit - // targets apply; prices under any other class name are never selected. - const classes = organization - ? [ - "standard", - ...(selectedOverlay?.targets || - rows.some( - (row) => - row.organization_id === organization && - row.serving_class === "custom", - ) || - orgCache.some((row) => row.serving_class === "custom") - ? ["custom"] - : []), - ] - : ["standard"]; + // Requests are priced by the class they are routed to; every model has + // `standard`, and prices under any other class name are never selected. + const classes = ["standard"]; const windows = [ ...new Set([ ...(model.allowed_batch_completion_windows ?? []), diff --git a/dashboard/src/components/features/serving/OverlayDetails.tsx b/dashboard/src/components/features/serving/OverlayDetails.tsx index ba023e769..57a294c38 100644 --- a/dashboard/src/components/features/serving/OverlayDetails.tsx +++ b/dashboard/src/components/features/serving/OverlayDetails.tsx @@ -24,8 +24,8 @@ export function OverlayDetails({
-
Targets
-
{overlay?.targets ? "Custom targets" : "Standard"}
+
Scheduling hints
+
{overlay?.targets ? "Set (see below)" : "None"}
Self-hosted only
@@ -43,7 +43,7 @@ export function OverlayDetails({
{overlay?.targets && (
-
Explicit targets
+
Priority and latency targets
TTFT {overlay.targets.ttft_ms} ms · ITL {overlay.targets.itl_ms}{" "} ms · priority {overlay.targets.priority ?? 0} diff --git a/docs/model-serving-class-identity.md b/docs/model-serving-class-identity.md index 060f3b1f4..83ac747b5 100644 --- a/docs/model-serving-class-identity.md +++ b/docs/model-serving-class-identity.md @@ -99,7 +99,7 @@ resolvers indefinitely: | --- | --- | | `deployed_models.serving_classes` JSON presets; `model_provisioning.rs` | Class catalog configuration replaces numeric target presets; retain shared model metadata. | | `users.granted_serving_classes`, `default_serving_class`; sync account snapshots | Explicit class selection replaces grants and implicit defaults. Preserve routing preferences such as `self_hosted_only`. | -| `model_overlays.default_serving_class`, `targets`; `org_overlays.rs`, serving API | Replace target/default selection with class overrides. Retain account/model ownership, catalog versioning and prices. | +| `model_overlays.default_serving_class`, `targets`; `org_overlays.rs`, serving API | Default selection is retired. `targets` are scheduling hints (priority, latency targets) that never select a class; overlay and account prices override one of the model's classes. Retain account/model ownership, catalog versioning and prices. | | `inference/middleware.rs`, `onwards/src/serving.rs`, `onwards/src/handlers.rs` | Replace suffix/default resolution and numeric target injection; preserve ingress scrubbing and trusted priority handling. | | `sync/onwards_config` and composite member rows | Keep legacy routing until all live consumers have migrated; new classes remain dormant until explicit activation. | | Tariff ledgers, analytics, queued billing records | Retain historical class keys and compatible readers; introducing class UUIDs does not rewrite history. | diff --git a/docs/src/how-to/organization-model-overlays.md b/docs/src/how-to/organization-model-overlays.md index 8f09c3a98..002f8cf40 100644 --- a/docs/src/how-to/organization-model-overlays.md +++ b/docs/src/how-to/organization-model-overlays.md @@ -73,7 +73,7 @@ retires all catalog-owned entries; a missing/unconfigured catalog is not a clean request. Future-dated prices must be resolved before catalog adoption or editing: startup reconciliation applies immediately and does not author schedules. -## Example: batch/flex and custom-target prices +## Example: batch/flex prices, a standard-class deal and scheduling hints The organization, model alias and prices below are fictional. Token prices are expressed per million tokens: @@ -104,9 +104,9 @@ models: write_multiplier_24h: "2" read_multiplier: "0.10" class_pricing: - custom: + standard: tariffs: - - name: custom-realtime + - name: standard-realtime purpose: realtime input_per_million_tokens: "3.00" output_per_million_tokens: "9.00" @@ -117,11 +117,17 @@ models: read_multiplier: "0.15" ``` -Realtime requests to which the overlay's explicit `targets` apply resolve as -`custom`; everything else resolves as `standard`. Batch and flex use standard; -flex uses the batch `1h` tariff. Prefer putting batch prices in the all-class -`tariffs` list. `class_pricing.standard` also supports batch; `custom` batch -prices are rejected by validation. +A request's serving class is where it is routed, and `class_pricing` prices one +of the model's classes (`standard` until models carry more). A class price wins +over the all-class `tariffs` for requests in that class. Batch and flex use +standard; flex uses the batch `1h` tariff. Prefer putting batch prices in the +all-class `tariffs` list. + +`targets` are scheduling hints for this organisation on this model: the priority +and the time-to-first-token and inter-token-latency targets are sent to +self-hosted serving members on realtime requests. They change neither the +request's class nor its price. The targets are labels the serving stack may use +for alerting; nothing routes on them. ## Billing, display and admission diff --git a/docs/src/reference/configuration.md b/docs/src/reference/configuration.md index 9372814ce..2489cd9fd 100644 --- a/docs/src/reference/configuration.md +++ b/docs/src/reference/configuration.md @@ -79,7 +79,7 @@ model_provisioning: |-------|------|---------|-------------| | `enabled` | boolean | `false` | Apply the declarative model catalog during startup. | | `directory` | path | `/app/model-provisioning.d` | Directory containing one `.yaml` or `.yml` document per canonical model. | -| `org_overlays_directory` | path | `/app/org-overlays.d` | Directory containing one `.yaml` or `.yml` document per organisation with per-model serving overrides (explicit `targets`, `self_hosted_only`). Applied after the model catalog when `enabled` is set. A missing directory is an empty catalog, so a deployment that does not mount one applies no overlays; an existing path that is not a directory fails startup. | +| `org_overlays_directory` | path | `/app/org-overlays.d` | Directory containing one `.yaml` or `.yml` document per organisation with per-model serving overrides (`targets`, which are scheduling hints and never change the class or price; `self_hosted_only`; prices). Applied after the model catalog when `enabled` is set. A missing directory is an empty catalog, so a deployment that does not mount one applies no overlays; an existing path that is not a directory fails startup. | | `account_limits_directory` | path | `/app/account-limits.d` | Directory containing one `.yaml` or `.yml` document per account with its own realtime in-flight limits, prices, scheduling priority and latency targets on virtual models. Applied after the model catalog and organisation overlays when `enabled` is set; limits replace every stored per-account limit, and the other settings are owned row by row. A missing directory changes nothing; an empty one clears what the files own. See [Account limits](./model-provisioning.md#account-limits). | When enabled, the directory must exist. Startup fails before any provisioning diff --git a/dwctl/src/account_limits.rs b/dwctl/src/account_limits.rs index 61a19984f..86668f2b0 100644 --- a/dwctl/src/account_limits.rs +++ b/dwctl/src/account_limits.rs @@ -1095,7 +1095,6 @@ mod tests { (overlay.priority, overlay.ttft_ms, overlay.itl_ms), (Some(300), Some(1000), Some(30)) ); - assert!(overlay.targets.is_none(), "policy never becomes billed custom targets"); assert_eq!(gateway_limit(&pool, "org/model", org).await, 900); // Removing the files clears what this catalog owns, keeping history. diff --git a/dwctl/src/org_overlays.rs b/dwctl/src/org_overlays.rs index 7042ad7b5..fb77f41fb 100644 --- a/dwctl/src/org_overlays.rs +++ b/dwctl/src/org_overlays.rs @@ -62,28 +62,27 @@ pub struct OrgDocument { pub models: Vec, } -/// Billing follows the resolved outcome: `custom` when the overlay's explicit -/// targets applied, otherwise `standard`. +/// A serving class the model routes to. An override prices one of the model's +/// classes; until models carry more, that is `standard`. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum PricingClass { Standard, - Custom, } impl PricingClass { - pub(crate) const ALL: [PricingClass; 2] = [PricingClass::Standard, PricingClass::Custom]; + pub(crate) const ALL: [PricingClass; 1] = [PricingClass::Standard]; pub(crate) fn as_str(self) -> &'static str { match self { Self::Standard => "standard", - Self::Custom => "custom", } } } -/// Class names this catalog may have priced before the serving-class presets -/// were retired. Rows it owns under them are retired on the next apply. -const RETIRED_PRICING_CLASSES: [&str; 2] = ["interactive", "throughput"]; +/// Class names this catalog may have priced before they were retired: the +/// serving-class presets, and `custom`, which overlay targets used to select. +/// Rows it owns under them are retired on the next apply. +const RETIRED_PRICING_CLASSES: [&str; 3] = ["interactive", "throughput", "custom"]; pub use crate::model_provisioning::CachePrices; @@ -102,8 +101,9 @@ pub struct ClassPrices { pub struct OrgModelOverlay { /// A virtual model alias from the model catalog. pub alias: String, - /// Explicit targets for a bespoke deal on this model, sent as-is to - /// self-hosted serving members on realtime requests. + /// Scheduling hints for this organisation on this model: priority and + /// latency targets sent to self-hosted serving members on realtime + /// requests. They change neither the request's class nor its price. #[serde(default)] pub targets: Option, /// Overrides the account's `self_hosted_only` setting on this model. @@ -119,7 +119,8 @@ pub struct OrgModelOverlay { /// changes the multipliers this organisation pays. #[serde(default)] pub cache_tariff: Option, - /// Optional prices for a resolved class, falling back to the general deal. + /// Optional prices for one of the model's serving classes, falling back to + /// the general deal. #[serde(default)] pub class_pricing: BTreeMap, } @@ -227,14 +228,6 @@ impl OrgCatalog { class.as_str() ); validate_tariffs(&prices.tariffs, &context)?; - ensure!( - *class == PricingClass::Standard - || prices - .tariffs - .iter() - .all(|t| !matches!(t.purpose, crate::model_provisioning::TariffPurpose::Batch)), - "{source}: batch prices can only specialize standard" - ); if let Some(cache) = &prices.cache_tariff { validate_cache_tariff(&cache.as_tariff(), &context)?; } @@ -465,13 +458,24 @@ mod tests { "input_per_million_tokens":"1", "output_per_million_tokens":"2"}); for model in [ serde_json::json!({"alias":"m", "tariffs":[tariff.clone()]}), - serde_json::json!({"alias":"m", "class_pricing":{"custom":{"tariffs":[tariff.clone()]}}}), + serde_json::json!({"alias":"m", "class_pricing":{"standard":{"tariffs":[tariff.clone()]}}}), ] { assert!(serde_json::from_value::(serde_json::json!({"org":"acme","models":[model]})).is_err()); } } } + #[test] + fn custom_is_not_a_pricing_class() { + let tariff = serde_json::json!({"name":"deal", "purpose":"realtime", + "input_per_million_tokens":"1", "output_per_million_tokens":"2"}); + let model = serde_json::json!({"alias":"m", "class_pricing":{"custom":{"tariffs":[tariff]}}}); + let error = serde_json::from_value::(serde_json::json!({"org":"acme","models":[model]})) + .unwrap_err() + .to_string(); + assert!(error.contains("custom"), "{error}"); + } + #[test] fn missing_directory_is_an_empty_catalog_but_a_file_is_an_error() { let directory = tempdir().unwrap(); @@ -755,7 +759,7 @@ mod tests { models: - alias: class-model class_pricing: - custom: + standard: tariffs: - {name: custom, purpose: realtime, input_per_million_tokens: '0', output_per_million_tokens: '1'} cache_tariff: {write_multiplier_5m: '1', write_multiplier_1h: '1', write_multiplier_24h: '1', read_multiplier: '0'} @@ -775,7 +779,7 @@ models: .fetch_one(&pool) .await .unwrap(); - assert_eq!(class, "custom"); + assert_eq!(class, "standard"); // A missing mount is a no-op, distinct from an explicitly empty mounted catalog. apply(&pool, &OrgCatalog::load(directory.path().join("missing")).unwrap()) .await @@ -815,7 +819,7 @@ models: async fn catalog_rejects_future_manual_price_collisions_atomically(pool: PgPool) { let org: Uuid = sqlx::query_scalar("INSERT INTO users (username,email,auth_source,user_type) VALUES ('future-org','future@example.com','test','organization') RETURNING id").fetch_one(&pool).await.unwrap(); let directory = tempdir().unwrap(); - for class in [None, Some("custom")] { + for class in [None, Some("standard")] { for finite in [false, true] { for cache in [false, true] { let alias = format!("future-{}-{finite}-{cache}", class.unwrap_or("all")); @@ -886,7 +890,7 @@ models: "tariffs:\n - {name: replacement, purpose: realtime, input_per_million_tokens: '3', output_per_million_tokens: '4'}".to_owned() }; let scope = if class.is_some() { - " class_pricing:\n custom:\n" + " class_pricing:\n standard:\n" } else { "" }; @@ -1086,7 +1090,7 @@ models: directory.path(), "org.yaml", &format!( - "org: review-org\nmodels:\n - alias: review-model\n class_pricing:\n custom:\n tariffs:\n - {{name: deal, purpose: realtime, input_per_million_tokens: '{input}', output_per_million_tokens: '2'}}\n" + "org: review-org\nmodels:\n - alias: review-model\n class_pricing:\n standard:\n tariffs:\n - {{name: deal, purpose: realtime, input_per_million_tokens: '{input}', output_per_million_tokens: '2'}}\n" ), ); apply(&pool, &OrgCatalog::load(directory.path()).unwrap()).await.unwrap(); diff --git a/dwctl/src/sync/onwards_config/mod.rs b/dwctl/src/sync/onwards_config/mod.rs index 844a8ff33..0bcace49a 100644 --- a/dwctl/src/sync/onwards_config/mod.rs +++ b/dwctl/src/sync/onwards_config/mod.rs @@ -17,7 +17,7 @@ use onwards::target::{ JitterStrategy as OnwardsJitterStrategy, KeyDefinition, LoadBalanceStrategy as OnwardsLoadBalanceStrategy, PoolSpec, PoolsSpec, ProviderSpec, RoutingAction, RoutingRule, TargetSpecOrList, Targets, WatchTargetsStream, inheritable_routing_rules, }; -use onwards::{AccountServing, ProviderKind, ServingOverlay, ServingTargets}; +use onwards::{AccountServing, ProviderKind, ServingOverlay}; use sqlx::{PgPool, postgres::PgListener}; use tokio::sync::{mpsc, watch}; use tokio_util::sync::CancellationToken; @@ -143,8 +143,18 @@ fn parse_provider_kind(value: &str) -> ProviderKind { }) } -/// Parse an overlay's stored explicit targets (`model_overlays.targets`). -fn parse_stored_targets(value: Option) -> Option { +/// An organisation overlay's stored targets (`model_overlays.targets`). They +/// are scheduling hints: they set the request's priority and latency targets +/// but never its class. +#[derive(serde::Deserialize)] +struct StoredTargets { + ttft_ms: u32, + itl_ms: u32, + #[serde(default)] + priority: i32, +} + +fn parse_stored_targets(value: Option) -> Option { let value = value?; match serde_json::from_value(value) { Ok(targets) => Some(targets), @@ -172,12 +182,14 @@ async fn load_overlays_from_db(db: &PgPool) -> Result( gen_ai.request.model = tracing::field::Empty, http.response.status_code = tracing::field::Empty, onwards.serving.resolved = tracing::field::Empty, + onwards.serving.priority = tracing::field::Empty, + onwards.serving.targets = tracing::field::Empty, ); // Extract W3C trace context (traceparent + tracestate) from inbound headers @@ -732,7 +734,17 @@ pub async fn target_message_handler( overlay.as_ref(), key_purpose.as_deref(), ); - tracing::Span::current().record("onwards.serving.resolved", resolution.resolved.as_str()); + let span = tracing::Span::current(); + span.record("onwards.serving.resolved", resolution.resolved.as_str()); + // Customer scheduling hints, recorded apart from the class: the + // priority, and whether latency targets were sent. + if let Some(priority) = resolution.hints.priority { + span.record("onwards.serving.priority", priority); + } + span.record( + "onwards.serving.targets", + resolution.hints.ttft_ms.is_some() || resolution.hints.itl_ms.is_some(), + ); metrics::counter!( "onwards_serving_class_requests_total", "model" => model_name.to_string(), @@ -1314,49 +1326,32 @@ pub async fn target_message_handler( }; } - // Send the resolved serving targets to a member of kind `dynamo`, the - // only serving stack that reads them, as `nvext.router` targets plus - // (when the preset carries one) `nvext.agent_hints.priority`. Every - // JSON request to a `dynamo` member also carries a priority (unless - // the capability strip above refused scheduling fields for it): when - // neither the body nor the preset sets one, realtime's 0, so the - // dynamo frontend is never left to pick a value for a missing one. - // Account and model policy hints (priority, latency targets) travel - // the same way but leave the request billed as its own class. - // A priority already present is kept, and targets overwrite rather - // than remove: this same crate runs again as the hop inside the - // serving namespace, with no key policy of its own, and must forward - // what the first hop set. Client-supplied targets are scrubbed at - // dwctl's ingress, alongside the body priority (the same perimeter - // the priority strip relies on). - if target.kind == ProviderKind::Dynamo && !attempt_body.is_empty() { - match serde_json::from_slice::(&attempt_body) { - Ok(mut body) => { - if let Some(object) = body.as_object_mut() { - if let Some(targets) = serving_resolution.targets { - targets.stamp(object); - } - if !scheduling_fields_refused { - serving_resolution.hints.stamp(object); - serving::stamp_default_priority(object); - } - attempt_body = match serde_json::to_vec(&body) { - Ok(bytes) => axum::body::Bytes::from(bytes), - Err(_) => return LoopAction::Done(Err(OnwardsErrorResponse::internal())), - }; - } - } - // Targets must reach the member; a body that cannot carry - // them is refused. Without targets a non-JSON body (a - // multipart upload) is forwarded as it came. - Err(_) if serving_resolution.targets.is_some() => { - return LoopAction::Done(Err(OnwardsErrorResponse::bad_request( - "Request body must be valid JSON.", - None, - ))) - } - Err(_) => {} - } + // Send the resolved scheduling hints (priority, latency targets) to a + // member of kind `dynamo`, the only serving stack that reads them. + // They never change the request's class. Every JSON request to a + // `dynamo` member also carries a priority (unless the capability strip + // above refused scheduling fields for it): when neither the body nor + // the hints set one, realtime's 0, so the dynamo frontend is never left + // to pick a value for a missing one. A priority already present is + // kept, and hints overwrite rather than remove: this same crate runs + // again as the hop inside the serving namespace, with no key policy of + // its own, and must forward what the first hop set. Client-supplied + // values are scrubbed at dwctl's ingress, alongside the body priority + // (the same perimeter the priority strip relies on). + // A non-JSON body (a multipart upload) carries no hints and is + // forwarded as it came. + if target.kind == ProviderKind::Dynamo + && !scheduling_fields_refused + && !attempt_body.is_empty() + && let Ok(mut body) = serde_json::from_slice::(&attempt_body) + && let Some(object) = body.as_object_mut() + { + serving_resolution.hints.stamp(object); + serving::stamp_default_priority(object); + attempt_body = match serde_json::to_vec(&body) { + Ok(bytes) => axum::body::Bytes::from(bytes), + Err(_) => return LoopAction::Done(Err(OnwardsErrorResponse::internal())), + }; } // Build the upstream URI for this target diff --git a/onwards/src/lib.rs b/onwards/src/lib.rs index d79c2a43f..b9eb85a34 100644 --- a/onwards/src/lib.rs +++ b/onwards/src/lib.rs @@ -76,7 +76,6 @@ use models::ExtractedModel; pub use rejections::RequestIdentity; pub use serving::{ AccountServing, ProviderKind, ServingClass, ServingClassOutcome, ServingHints, ServingOverlay, - ServingTargets, }; /// Type alias for response transformation function diff --git a/onwards/src/serving.rs b/onwards/src/serving.rs index 502059bf9..76fa63a71 100644 --- a/onwards/src/serving.rs +++ b/onwards/src/serving.rs @@ -1,30 +1,28 @@ -//! Per-request serving policy: overlay targets, scheduling priority and the -//! self-hosted restriction. +//! Per-request serving policy: scheduling hints and the self-hosted +//! restriction. //! -//! An organisation's **overlay** on an alias may carry explicit objective -//! targets for a bespoke deal: a time-to-first-token target, an -//! inter-token-latency target and a scheduling priority. When present they are -//! sent to members of kind `dynamo` and the request resolves as -//! [`ServingClass::Custom`]; otherwise it resolves as -//! [`ServingClass::Standard`] and sends no targets. +//! A request's **serving class** is where it is routed: today every request +//! resolves as [`ServingClass::Standard`]. A class is a destination on the +//! model, never a customer setting. //! -//! Separately, an account may carry a scheduling **priority** (account-wide, -//! or per model on an overlay) and per-model latency targets. These are sent -//! the same way but leave the request [`ServingClass::Standard`]: they change -//! scheduling, not the price. +//! On top of the class, an account may carry **scheduling hints**: a priority +//! (account-wide, or per alias) and per-alias time-to-first-token and +//! inter-token-latency targets, from an organisation overlay or the account +//! catalog. They are sent to members of kind `dynamo` but change neither the +//! class nor the price. The targets are labels the serving stack may use for +//! alerting and diagnostics; nothing routes on them. //! //! Daemon legs (batch and flex on the batch-purpose key, continuation resume -//! legs) never receive overlay targets or priorities and keep their own: -//! batch and flex carry deadline-derived priorities, background dispatch the -//! reserved `i32::MIN`, and continuation resumes their fixed priority. +//! legs) never receive hints and keep their own priorities: batch and flex +//! carry deadline-derived priorities, background dispatch the reserved +//! `i32::MIN`, and continuation resumes their fixed priority. //! //! The targets travel in the request body as `nvext.router.{ttft_target, -//! itl_target}` and, when non-zero, the priority as -//! `nvext.agent_hints.priority`, on members of kind `dynamo` only. A request -//! to a `dynamo` member that carries no priority is sent -//! [`DEFAULT_PRIORITY`], so every priority is relative to realtime's 0. -//! Everything a client could send to steer this itself is scrubbed at dwctl's -//! ingress; only the resolver sets them. +//! itl_target}` and the priority as `nvext.agent_hints.priority`, on members +//! of kind `dynamo` only. A request to a `dynamo` member that carries no +//! priority is sent [`DEFAULT_PRIORITY`], so every priority is relative to +//! realtime's 0. Everything a client could send to steer this itself is +//! scrubbed at dwctl's ingress; only the resolver sets them. use std::fmt; use std::str::FromStr; @@ -56,22 +54,18 @@ pub const ACCOUNT_NAME_LABEL: &str = "account_name"; /// [`ACCOUNT_NAME_LABEL`] is used when it is absent. pub const ACCOUNT_DISPLAY_NAME_LABEL: &str = "account_display_name"; -/// How a request was served, for analytics and tariff selection. +/// Where a request was routed, for analytics and tariff selection. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] #[serde(rename_all = "lowercase")] pub enum ServingClass { - /// No overlay targets applied. + /// The model's default destination. Standard, - /// Explicit targets from an overlay: a bespoke deal on one alias. An - /// outcome only, never configuration. - Custom, } impl ServingClass { pub fn as_str(self) -> &'static str { match self { ServingClass::Standard => "standard", - ServingClass::Custom => "custom", } } } @@ -82,64 +76,10 @@ impl fmt::Display for ServingClass { } } -/// The objective targets one request is served to. An overlay may carry them -/// explicitly for a bespoke deal. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct ServingTargets { - /// Time-to-first-token target, milliseconds. - pub ttft_ms: u32, - /// Inter-token-latency target, milliseconds. - pub itl_ms: u32, - /// Scheduling priority on the serving stack (higher wins; 0 is the - /// realtime value, sent by [`stamp_default_priority`] instead). - #[serde(default)] - pub priority: i32, -} - -impl ServingTargets { - /// Write the targets into a request body: `nvext.router.{ttft_target, - /// itl_target}` always, `nvext.agent_hints.priority` only when non-zero. - /// Existing values are overwritten; the rest of `nvext` is kept. - pub fn stamp(&self, body: &mut serde_json::Map) { - let nvext = body - .entry(NVEXT_FIELD) - .or_insert_with(|| serde_json::Value::Object(Default::default())); - if !nvext.is_object() { - *nvext = serde_json::Value::Object(Default::default()); - } - let nvext = nvext - .as_object_mut() - .expect("nvext was just made an object"); - let router = nvext - .entry(NVEXT_ROUTER_FIELD) - .or_insert_with(|| serde_json::Value::Object(Default::default())); - if !router.is_object() { - *router = serde_json::Value::Object(Default::default()); - } - let router = router - .as_object_mut() - .expect("router was just made an object"); - router.insert(TTFT_TARGET_FIELD.to_string(), self.ttft_ms.into()); - router.insert(ITL_TARGET_FIELD.to_string(), self.itl_ms.into()); - if self.priority != 0 { - let hints = nvext - .entry(NVEXT_AGENT_HINTS_FIELD) - .or_insert_with(|| serde_json::Value::Object(Default::default())); - if !hints.is_object() { - *hints = serde_json::Value::Object(Default::default()); - } - hints - .as_object_mut() - .expect("agent_hints was just made an object") - .insert(PRIORITY_FIELD.to_string(), self.priority.into()); - } - } -} - /// Write [`DEFAULT_PRIORITY`] as `nvext.agent_hints.priority` unless the body /// already carries a priority. One already present was set by someone -/// entitled to: the daemon (batch deadline, continuation resume), resolved overlay -/// targets, or the first hop when this crate runs again inside the serving +/// entitled to: the daemon (batch deadline, continuation resume), the +/// resolved hints, or the first hop when this crate runs again inside the serving /// namespace; callers' own are scrubbed at dwctl's ingress. The rest of /// `nvext` is kept. pub fn stamp_default_priority(body: &mut serde_json::Map) { @@ -185,7 +125,7 @@ pub fn scrub_router_targets(nvext: &mut serde_json::Map, /// Overrides the account's `self_hosted_only` on this alias. #[serde(default)] pub self_hosted_only: Option, @@ -280,8 +217,8 @@ pub struct ServingOverlay { pub itl_ms: Option, } -/// Scheduling hints for one request that do not change how it is billed: -/// each field is written into the body only when set. +/// Scheduling hints for one request. They change neither its class nor its +/// price; each field is written into the body only when set. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct ServingHints { pub priority: Option, @@ -343,17 +280,14 @@ impl ServingHints { /// Key purposes whose requests are daemon legs: the ordering already travels /// as a deadline-derived (batch, flex) or fixed (continuation resume) -/// priority in the body, and no targets are sent. +/// priority in the body, and no hints are sent. const DAEMON_PURPOSES: [&str; 2] = ["batch", "continuation"]; /// The outcome of resolving one request. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ServingResolution { pub resolved: ServingClass, - /// What a `dynamo` member is told to serve to; `None` sends nothing. - pub targets: Option, - /// Priority and latency targets that leave the class unchanged. Empty - /// whenever `targets` is set. + /// Priority and latency targets sent to `dynamo` members. pub hints: ServingHints, /// Restrict the composite to its non-external members. pub self_hosted_only: bool, @@ -372,7 +306,7 @@ impl ServingResolution { /// /// * `account` — the owning organisation's settings, if it has any. /// * `overlay` — the organisation's overrides on this alias, if any. -/// * `purpose` — the key's purpose label; daemon purposes get no targets. +/// * `purpose` — the key's purpose label; daemon purposes get no hints. pub fn resolve( account: Option<&AccountServing>, overlay: Option<&ServingOverlay>, @@ -383,29 +317,20 @@ pub fn resolve( .unwrap_or_else(|| account.is_some_and(|a| a.self_hosted_only)); let daemon = purpose.is_some_and(|p| DAEMON_PURPOSES.contains(&p)); - match overlay.and_then(|o| o.targets) { - Some(targets) if !daemon => ServingResolution { - resolved: ServingClass::Custom, - targets: Some(targets), - hints: ServingHints::default(), - self_hosted_only, - }, - _ => ServingResolution { - resolved: ServingClass::Standard, - targets: None, - hints: if daemon { - ServingHints::default() - } else { - ServingHints { - priority: overlay - .and_then(|o| o.priority) - .or(account.and_then(|a| a.priority)), - ttft_ms: overlay.and_then(|o| o.ttft_ms), - itl_ms: overlay.and_then(|o| o.itl_ms), - } - }, - self_hosted_only, + ServingResolution { + resolved: ServingClass::Standard, + hints: if daemon { + ServingHints::default() + } else { + ServingHints { + priority: overlay + .and_then(|o| o.priority) + .or(account.and_then(|a| a.priority)), + ttft_ms: overlay.and_then(|o| o.ttft_ms), + itl_ms: overlay.and_then(|o| o.itl_ms), + } }, + self_hosted_only, } } @@ -413,45 +338,52 @@ pub fn resolve( mod tests { use super::*; - const DEAL: ServingTargets = ServingTargets { - ttft_ms: 500, - itl_ms: 20, - priority: 200, - }; - - fn overlay_with(targets: ServingTargets) -> ServingOverlay { + fn deal() -> ServingOverlay { ServingOverlay { - targets: Some(targets), + priority: Some(200), + ttft_ms: Some(500), + itl_ms: Some(20), ..Default::default() } } #[test] - fn nothing_set_resolves_to_standard_with_no_targets() { + fn nothing_set_resolves_to_standard_with_no_hints() { let r = resolve(None, None, Some("realtime")); assert_eq!(r.resolved, ServingClass::Standard); - assert_eq!(r.targets, None); + assert!(r.hints.is_empty()); assert!(!r.self_hosted_only); assert_eq!(r.outcome().requested, None); } #[test] - fn overlay_targets_resolve_as_custom_for_realtime() { - let o = overlay_with(DEAL); + fn overlay_targets_are_hints_on_the_standard_class() { + let o = deal(); let r = resolve(None, Some(&o), Some("realtime")); - assert_eq!(r.resolved, ServingClass::Custom); - assert_eq!(r.targets, Some(DEAL)); + assert_eq!( + r.resolved, + ServingClass::Standard, + "targets never change the class" + ); + assert_eq!( + r.hints, + ServingHints { + priority: Some(200), + ttft_ms: Some(500), + itl_ms: Some(20) + } + ); // a key without a purpose label is treated as realtime - assert_eq!(resolve(None, Some(&o), None).targets, Some(DEAL)); + assert_eq!(resolve(None, Some(&o), None).hints, r.hints); } #[test] - fn daemon_legs_never_receive_overlay_targets() { - let o = overlay_with(DEAL); + fn daemon_legs_never_receive_hints() { + let o = deal(); for purpose in ["batch", "continuation"] { let r = resolve(None, Some(&o), Some(purpose)); assert_eq!(r.resolved, ServingClass::Standard, "{purpose}"); - assert_eq!(r.targets, None, "{purpose}"); + assert!(r.hints.is_empty(), "{purpose}"); } } @@ -472,12 +404,14 @@ mod tests { } #[test] - fn targets_are_stamped_into_nvext_and_scrubbed_from_it() { + fn hints_are_stamped_into_nvext_and_targets_scrubbed_from_it() { let mut body = serde_json::json!({ "model": "m", "nvext": {"cache_control": {"enabled": true}, "router": {"ttft_target": 1}, "agent_hints": {"max_batch_size": 8}} }); - DEAL.stamp(body.as_object_mut().unwrap()); + resolve(None, Some(&deal()), Some("realtime")) + .hints + .stamp(body.as_object_mut().unwrap()); assert_eq!(body["nvext"]["router"]["ttft_target"], 500, "overwritten"); assert_eq!(body["nvext"]["router"]["itl_target"], 20); assert_eq!(body["nvext"]["agent_hints"]["priority"], 200); @@ -487,12 +421,11 @@ mod tests { ); assert_eq!(body["nvext"]["cache_control"]["enabled"], true); - // priority 0 sends no priority at all; a body with no nvext gets one let mut body = serde_json::json!({"model": "m"}); - ServingTargets { - ttft_ms: 1, - itl_ms: 2, - priority: 0, + ServingHints { + ttft_ms: Some(1), + itl_ms: Some(2), + ..Default::default() } .stamp(body.as_object_mut().unwrap()); assert_eq!( @@ -574,7 +507,7 @@ mod tests { assert_eq!( r.resolved, ServingClass::Standard, - "hints never change the billed class" + "hints never change the class" ); assert_eq!( r.hints, @@ -591,16 +524,6 @@ mod tests { .hints .is_empty() ); - - // Explicit overlay targets still win and carry their own priority. - let legacy = ServingOverlay { - targets: Some(DEAL), - priority: Some(1), - ..Default::default() - }; - let r = resolve(Some(&account), Some(&legacy), Some("realtime")); - assert_eq!(r.resolved, ServingClass::Custom); - assert!(r.hints.is_empty()); } #[test] @@ -646,8 +569,8 @@ mod tests { assert!("cloud".parse::().is_err()); assert_eq!(ServingClass::Standard.as_str(), "standard"); assert_eq!( - serde_json::to_string(&ServingClass::Custom).unwrap(), - "\"custom\"" + serde_json::to_string(&ServingClass::Standard).unwrap(), + "\"standard\"" ); } @@ -661,14 +584,10 @@ mod tests { .unwrap(); assert!(a.self_hosted_only); let o: ServingOverlay = serde_json::from_str( - r#"{"default_class": "interactive", "targets": {"ttft_ms": 800, "itl_ms": 30}}"#, + r#"{"default_class": "interactive", "targets": {"ttft_ms": 800, "itl_ms": 30}, "priority": 5}"#, ) .unwrap(); - assert_eq!( - o.targets.map(|t| (t.ttft_ms, t.itl_ms, t.priority)), - Some((800, 30, 0)), - "priority defaults to 0" - ); + assert_eq!(o.priority, Some(5)); assert_eq!( serde_json::to_string(&ProviderKind::Dynamo).unwrap(), "\"dynamo\"" @@ -723,16 +642,12 @@ mod handler_tests { } } - const DEAL: ServingTargets = ServingTargets { - ttft_ms: 500, - itl_ms: 20, - priority: 200, - }; - fn with_deal(members: &[Member]) -> Setup { let mut s = setup(members); s.overlay = Some(ServingOverlay { - targets: Some(DEAL), + priority: Some(200), + ttft_ms: Some(500), + itl_ms: Some(20), ..Default::default() }); s @@ -846,7 +761,7 @@ mod handler_tests { } #[tokio::test] - async fn strict_handlers_send_overlay_targets() { + async fn strict_handlers_send_overlay_hints() { for (path, payload) in [ ("/chat/completions", json!({"model":ALIAS,"messages":[]})), ("/completions", json!({"model":ALIAS,"prompt":"hi"})), @@ -897,13 +812,13 @@ mod handler_tests { assert_eq!(error.status, StatusCode::SERVICE_UNAVAILABLE); assert_eq!( error.serving_outcome.unwrap().resolved, - ServingClass::Custom + ServingClass::Standard ); assert!(mock.get_requests().is_empty()); } #[tokio::test] - async fn only_dynamo_members_receive_the_targets() { + async fn only_dynamo_members_receive_the_hints() { for member in [EXTERNAL, HOSTED] { let s = with_deal(&[member]); let mock = MockHttpClient::new(StatusCode::OK, OK_BODY); @@ -966,29 +881,6 @@ mod handler_tests { assert!(body(&mock.get_requests()[0]).get("nvext").is_none()); } - #[tokio::test] - async fn explicit_overlay_targets_are_sent_as_is() { - let mut s = setup(&[DYNAMO]); - s.overlay = Some(ServingOverlay { - targets: Some(ServingTargets { - ttft_ms: 800, - itl_ms: 30, - priority: 50, - }), - ..Default::default() - }); - let mock = MockHttpClient::new(StatusCode::OK, OK_BODY); - let srv = server(&s, &mock); - assert_eq!(post(&srv, ALIAS, json!({})).await.status_code(), 200); - let req = &mock.get_requests()[0]; - assert_eq!(sent(req), Some((800, 30, Some(50)))); - assert_eq!( - body(req)["messages"][0]["content"], - "hi", - "the rest of the body is untouched" - ); - } - #[tokio::test] async fn no_policy_at_all_sends_only_realtimes_priority() { let s = setup(&[DYNAMO]); @@ -1002,8 +894,8 @@ mod handler_tests { #[tokio::test] async fn a_non_json_body_reaches_a_dynamo_member_untouched() { - // Multipart uploads carry no hints; without targets to deliver they - // are forwarded as they came, never refused. + // Multipart uploads carry no hints: they are forwarded as they came, + // never refused. let s = setup(&[DYNAMO]); let mock = MockHttpClient::new(StatusCode::OK, OK_BODY); let srv = server(&s, &mock); @@ -1054,12 +946,12 @@ mod handler_tests { assert_eq!( sent(&mock.get_requests()[0]), Some((500, 20, Some(200))), - "the targets are overwritten" + "the hints are overwritten" ); } #[tokio::test] - async fn daemon_purpose_keys_get_no_overlay_targets() { + async fn daemon_purpose_keys_get_no_hints() { let mut s = with_deal(&[DYNAMO]); s.purpose = "batch"; let mock = MockHttpClient::new(StatusCode::OK, OK_BODY); @@ -1105,7 +997,7 @@ mod handler_tests { .copied() .expect("the outcome rides on error responses too, for analytics"); assert_eq!(outcome.requested, None); - assert_eq!(outcome.resolved, ServingClass::Custom); + assert_eq!(outcome.resolved, ServingClass::Standard); } #[tokio::test] @@ -1201,6 +1093,7 @@ mod handler_tests { Target::builder() .url("https://dynamo-completions.example.com/".parse().unwrap()) .kind(ProviderKind::Dynamo) + .accepts_scheduling_priority(true) .build(), 1, )], @@ -1284,8 +1177,8 @@ mod handler_tests { #[test] fn accounts_and_overlays_round_trip_through_config_json() { - // Retired class fields (`serving_classes`, `granted`, `default_class`) - // in a stored config are ignored rather than rejected. + // Retired fields (`serving_classes`, `granted`, `default_class`, + // overlay `targets`) in a stored config are ignored rather than rejected. let json = json!({ "targets": { "gpt-4": { @@ -1293,7 +1186,7 @@ mod handler_tests { "serving_classes": {"interactive": {"ttft_ms": 500, "itl_ms": 20, "priority": 200}}, "overlays": { "org-1": {"default_class": "interactive", "self_hosted_only": false}, - "org-2": {"targets": {"ttft_ms": 800, "itl_ms": 30}} + "org-2": {"targets": {"ttft_ms": 800, "itl_ms": 30}, "priority": 300, "ttft_ms": 800, "itl_ms": 30} } } }, @@ -1308,10 +1201,12 @@ mod handler_tests { let overlays = pools.default_pool().overlays(); assert_eq!(overlays["org-1"].self_hosted_only, Some(false)); assert_eq!( - overlays["org-2"] - .targets - .map(|t| (t.ttft_ms, t.itl_ms, t.priority)), - Some((800, 30, 0)) + ( + overlays["org-2"].priority, + overlays["org-2"].ttft_ms, + overlays["org-2"].itl_ms + ), + (Some(300), Some(800), Some(30)) ); assert_eq!( pools.default_pool().providers()[0].target.kind, From e53b7153caf0552c6b2bad645997400f28409f66 Mon Sep 17 00:00:00 2001 From: hachall Date: Thu, 22 Oct 2026 12:15:15 +0100 Subject: [PATCH 2/2] fix: record delivered hints, keep overlay targets under priority-only policies Name the request span's targets field for what it is (configured) and record onwards.hints_sent on the attempt that actually carried hints. A priority-only account policy no longer clears an overlay's latency targets in the sync. The overlay docs name dynamo members that accept scheduling fields, and a test covers retiring catalog-owned custom prices on a still-declared model. --- .../src/how-to/organization-model-overlays.md | 4 +- dwctl/src/org_overlays.rs | 52 ++++++++++++++++++- dwctl/src/sync/onwards_config/mod.rs | 10 +++- dwctl/src/sync/onwards_config/tests.rs | 37 +++++++++++++ onwards/src/handlers.rs | 13 +++-- 5 files changed, 106 insertions(+), 10 deletions(-) diff --git a/docs/src/how-to/organization-model-overlays.md b/docs/src/how-to/organization-model-overlays.md index 002f8cf40..a06935996 100644 --- a/docs/src/how-to/organization-model-overlays.md +++ b/docs/src/how-to/organization-model-overlays.md @@ -124,8 +124,8 @@ standard; flex uses the batch `1h` tariff. Prefer putting batch prices in the all-class `tariffs` list. `targets` are scheduling hints for this organisation on this model: the priority -and the time-to-first-token and inter-token-latency targets are sent to -self-hosted serving members on realtime requests. They change neither the +and the time-to-first-token and inter-token-latency targets are sent on realtime +requests to members of kind `dynamo` that accept scheduling fields. They change neither the request's class nor its price. The targets are labels the serving stack may use for alerting; nothing routes on them. diff --git a/dwctl/src/org_overlays.rs b/dwctl/src/org_overlays.rs index fb77f41fb..7aa833313 100644 --- a/dwctl/src/org_overlays.rs +++ b/dwctl/src/org_overlays.rs @@ -102,8 +102,9 @@ pub struct OrgModelOverlay { /// A virtual model alias from the model catalog. pub alias: String, /// Scheduling hints for this organisation on this model: priority and - /// latency targets sent to self-hosted serving members on realtime - /// requests. They change neither the request's class nor its price. + /// latency targets sent on realtime requests to `dynamo` members that + /// accept scheduling fields. They change neither the request's class nor + /// its price. #[serde(default)] pub targets: Option, /// Overrides the account's `self_hosted_only` setting on this model. @@ -1014,6 +1015,53 @@ models: (org, model) } + #[dwctl_test_macros::test] + async fn legacy_custom_class_prices_are_retired_on_a_still_declared_model(pool: PgPool) { + let (org, model) = review_org_model(&pool).await; + // Prices this catalog wrote under `custom` before it was retired. + for table in ["model_tariffs", "model_cache_tariffs"] { + let insert = if table == "model_tariffs" { + "INSERT INTO model_tariffs (deployed_model_id,user_id,serving_class,name,input_price_per_token,output_price_per_token,api_key_purpose,provisioning_source) VALUES ($1,$2,'custom','legacy',1,1,'realtime','org-overlays')" + } else { + "INSERT INTO model_cache_tariffs (deployed_model_id,user_id,serving_class,read_multiplier,write_multiplier_5m,write_multiplier_1h,write_multiplier_24h,min_prefix_tokens,provisioning_source) VALUES ($1,$2,'custom',0.1,1,1,1,1024,'org-overlays')" + }; + sqlx::query(insert).bind(model).bind(org).execute(&pool).await.unwrap(); + } + let directory = tempdir().unwrap(); + write( + directory.path(), + "org.yaml", + "org: review-org\nmodels:\n - alias: review-model\n class_pricing:\n standard:\n tariffs:\n - {name: deal, purpose: realtime, input_per_million_tokens: '1', output_per_million_tokens: '2'}\n", + ); + apply(&pool, &OrgCatalog::load(directory.path()).unwrap()).await.unwrap(); + for table in ["model_tariffs", "model_cache_tariffs"] { + let active: Vec> = sqlx::query_scalar(&format!( + "SELECT serving_class FROM {table} WHERE user_id = $1 AND valid_until IS NULL ORDER BY serving_class" + )) + .bind(org) + .fetch_all(&pool) + .await + .unwrap(); + let expected: Vec> = if table == "model_tariffs" { + vec![Some("standard".into())] + } else { + vec![] + }; + assert_eq!( + active, expected, + "{table}: the legacy custom price is retired, the declared one kept" + ); + let retired: i64 = sqlx::query_scalar(&format!( + "SELECT count(*) FROM {table} WHERE user_id = $1 AND serving_class = 'custom' AND valid_until IS NOT NULL" + )) + .bind(org) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(retired, 1, "{table}: history is kept"); + } + } + #[dwctl_test_macros::test] async fn waiting_replica_uses_time_after_org_catalog_lock(pool: PgPool) { review_org_model(&pool).await; diff --git a/dwctl/src/sync/onwards_config/mod.rs b/dwctl/src/sync/onwards_config/mod.rs index 0bcace49a..9c6ec3566 100644 --- a/dwctl/src/sync/onwards_config/mod.rs +++ b/dwctl/src/sync/onwards_config/mod.rs @@ -216,9 +216,15 @@ async fn load_overlays_from_db(db: &PgPool) -> Result( http.response.status_code = tracing::field::Empty, onwards.serving.resolved = tracing::field::Empty, onwards.serving.priority = tracing::field::Empty, - onwards.serving.targets = tracing::field::Empty, + onwards.serving.targets_configured = tracing::field::Empty, ); // Extract W3C trace context (traceparent + tracestate) from inbound headers @@ -736,13 +736,14 @@ pub async fn target_message_handler( ); let span = tracing::Span::current(); span.record("onwards.serving.resolved", resolution.resolved.as_str()); - // Customer scheduling hints, recorded apart from the class: the - // priority, and whether latency targets were sent. + // The customer's configured scheduling hints, recorded apart from + // the class. Whether an attempt actually carried them is recorded on + // its `onwards.provider_attempt` span (`onwards.hints_sent`). if let Some(priority) = resolution.hints.priority { span.record("onwards.serving.priority", priority); } span.record( - "onwards.serving.targets", + "onwards.serving.targets_configured", resolution.hints.ttft_ms.is_some() || resolution.hints.itl_ms.is_some(), ); metrics::counter!( @@ -1181,6 +1182,7 @@ pub async fn target_message_handler( http.response.status_code = tracing::field::Empty, onwards.fallback = tracing::field::Empty, onwards.pool = tracing::field::Empty, + onwards.hints_sent = tracing::field::Empty, ); // Recorded only when a non-default pool served the request, so the // field's presence answers "did per-class routing engage here?". @@ -1347,6 +1349,9 @@ pub async fn target_message_handler( && let Some(object) = body.as_object_mut() { serving_resolution.hints.stamp(object); + if !serving_resolution.hints.is_empty() { + tracing::Span::current().record("onwards.hints_sent", true); + } serving::stamp_default_priority(object); attempt_body = match serde_json::to_vec(&body) { Ok(bytes) => axum::body::Bytes::from(bytes),