diff --git a/.sqlx/query-c2c9cd3d6e383666f056cc686f46b971dcfc39d39f0a98ec42a3b7913d2421d1.json b/.sqlx/query-056f692525815bb06d25a97b61384e22c1121ba776c803b558e4fa17e7a9a42c.json similarity index 54% rename from .sqlx/query-c2c9cd3d6e383666f056cc686f46b971dcfc39d39f0a98ec42a3b7913d2421d1.json rename to .sqlx/query-056f692525815bb06d25a97b61384e22c1121ba776c803b558e4fa17e7a9a42c.json index a79eb31c8..2fc21bb92 100644 --- a/.sqlx/query-c2c9cd3d6e383666f056cc686f46b971dcfc39d39f0a98ec42a3b7913d2421d1.json +++ b/.sqlx/query-056f692525815bb06d25a97b61384e22c1121ba776c803b558e4fa17e7a9a42c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n cm.id as composite_model_id,\n ak.id as api_key_id,\n ak.secret as api_key_secret,\n ak.purpose as api_key_purpose,\n ak.user_zero_data_retention,\n ak.user_id,\n ak.user_username,\n ak.user_display_name as \"user_display_name?\"\n FROM deployed_models cm\n CROSS JOIN LATERAL (\n SELECT DISTINCT\n ak.id,\n ak.secret,\n ak.purpose,\n ak.user_id,\n u.zero_data_retention as user_zero_data_retention,\n u.username as user_username,\n u.display_name as user_display_name\n FROM api_keys ak\n JOIN users u ON u.id = ak.user_id\n WHERE (\n -- System user always has access\n ak.user_id = '00000000-0000-0000-0000-000000000000'\n -- OR user is in a group assigned to this composite model (via deployment_groups)\n OR EXISTS (\n SELECT 1 FROM user_groups ug\n INNER JOIN deployment_groups dg ON ug.group_id = dg.group_id\n WHERE dg.deployment_id = cm.id\n AND ug.user_id = ak.user_id\n )\n -- OR composite model is in public group (nil UUID)\n OR EXISTS (\n SELECT 1 FROM deployment_groups dg\n WHERE dg.deployment_id = cm.id\n AND dg.group_id = '00000000-0000-0000-0000-000000000000'\n )\n -- OR this is a batch API key and composite model is an escalation target\n OR (\n ak.purpose = 'batch'\n AND cm.alias = ANY($1::text[])\n )\n )\n -- General free-model access is preserved; a zero customer deal does not make a paid model free.\n AND (\n ak.user_id = '00000000-0000-0000-0000-000000000000'\n -- Positive balance read directly from the total\n -- user_balance_checkpoints read model (kept current by the\n -- writers folding synchronously with each charge). The\n -- is_deleted guard mirrors the old balance CTE, which only\n -- contained non-deleted users; key deletion is not implied by\n -- user deletion, so this check is load-bearing.\n -- Keep this lookup inline so PostgreSQL can build a hashed\n -- set of enabled accounts once instead of calling a function\n -- for every model/key pair. The outer guard excludes deleted users.\n OR (u.is_deleted = false AND (EXISTS (\n SELECT 1 FROM user_feature_flags f\n WHERE f.user_id = u.id\n AND f.feature_flag = 'ALLOW_NEGATIVE_BALANCE'\n AND f.enabled\n ) OR EXISTS (\n SELECT 1 FROM user_balance_checkpoints ub\n WHERE ub.user_id = ak.user_id AND ub.balance > 0\n )))\n OR (\n -- Keep the legacy general-tariff rule; only this account's\n -- positive deal can require credit on an otherwise free model.\n NOT EXISTS (\n SELECT 1 FROM model_tariffs mt\n WHERE mt.deployed_model_id = cm.id AND mt.user_id IS NULL AND mt.serving_class IS NULL\n AND mt.valid_until IS NULL\n AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0)\n ) AND NOT EXISTS (\n SELECT 1 FROM model_tariffs mt\n WHERE mt.deployed_model_id = cm.id AND mt.user_id = ak.user_id\n AND mt.valid_from <= NOW() AND (mt.valid_until IS NULL OR mt.valid_until > NOW())\n AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0)\n )\n )\n )\n -- Account deletion revokes access independently of key deletion.\n AND (ak.user_id = '00000000-0000-0000-0000-000000000000' OR u.is_deleted = false)\n AND ak.is_deleted = false\n -- Spending-cap gate: exclude every key of a cap scope (the capped\n -- root and its hidden batch child alike) once the scope's\n -- CALENDAR-ALIGNED (UTC, non-rolling) window spend has reached the\n -- root's limit. Guarded so the uncapped majority short-circuits on\n -- the first branch; the subquery is two PK probes. Generally free\n -- models remain usable, but zero customer deals do not bypass caps\n -- on paid models. Un-capping needs no job or traffic: the\n -- window-membership check (shared function, migration 123) turns\n -- false at the calendar boundary and the periodic fallback sync\n -- readmits the keys.\n AND (\n (ak.spend_limit IS NULL AND ak.parent_api_key_id IS NULL)\n OR NOT EXISTS (\n SELECT 1\n FROM api_keys root\n LEFT JOIN api_key_spend_checkpoints ck ON ck.api_key_id = root.id\n WHERE root.id = COALESCE(ak.parent_api_key_id, ak.id)\n AND root.spend_limit IS NOT NULL\n AND api_key_cap_window_current(ck.window_started_at, root.spend_limit_interval)\n AND ck.window_spend >= root.spend_limit\n AND (EXISTS (\n SELECT 1 FROM model_tariffs mt\n WHERE mt.deployed_model_id = cm.id AND mt.user_id IS NULL AND mt.serving_class IS NULL\n AND mt.valid_until IS NULL\n AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0)\n ) OR EXISTS (\n SELECT 1 FROM model_tariffs mt\n WHERE mt.deployed_model_id = cm.id AND mt.user_id = ak.user_id\n AND mt.valid_from <= NOW() AND (mt.valid_until IS NULL OR mt.valid_until > NOW())\n AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0)\n ))\n )\n )\n -- Inference data plane only: platform (management) keys must never\n -- enter onwards' key set. Mirrors is_inference_purpose in\n -- db::models::api_keys (SQL cannot call it). The system key is\n -- purpose 'platform' (it calls admin endpoints, see migration 071)\n -- but is used internally for onwards inference, so it is exempt.\n AND (\n ak.user_id = '00000000-0000-0000-0000-000000000000'\n OR ak.purpose IN ('realtime', 'batch', 'playground', 'continuation')\n )\n ) ak\n WHERE cm.is_composite = TRUE\n AND cm.deleted = FALSE\n ORDER BY cm.id, ak.id\n ", + "query": "\n SELECT\n cm.id as composite_model_id,\n ak.id as api_key_id,\n ak.secret as api_key_secret,\n ak.purpose as api_key_purpose,\n ak.user_zero_data_retention,\n ak.user_id,\n ak.user_username,\n ak.user_display_name as \"user_display_name?\"\n FROM deployed_models cm\n CROSS JOIN LATERAL (\n SELECT DISTINCT\n ak.id,\n ak.secret,\n ak.purpose,\n ak.user_id,\n u.zero_data_retention as user_zero_data_retention,\n u.username as user_username,\n u.display_name as user_display_name\n FROM api_keys ak\n JOIN users u ON u.id = ak.user_id\n WHERE (\n -- System user always has access\n ak.user_id = '00000000-0000-0000-0000-000000000000'\n -- OR user is in a group assigned to this composite model (via deployment_groups)\n OR EXISTS (\n SELECT 1 FROM user_groups ug\n INNER JOIN deployment_groups dg ON ug.group_id = dg.group_id\n WHERE dg.deployment_id = cm.id\n AND ug.user_id = ak.user_id\n )\n -- OR composite model is in public group (nil UUID)\n OR EXISTS (\n SELECT 1 FROM deployment_groups dg\n WHERE dg.deployment_id = cm.id\n AND dg.group_id = '00000000-0000-0000-0000-000000000000'\n )\n -- OR this is a batch API key and composite model is an escalation target\n OR (\n ak.purpose = 'batch'\n AND cm.alias = ANY($1::text[])\n )\n )\n -- General free-model access is preserved; a zero customer deal does not make a paid model free.\n AND (\n ak.user_id = '00000000-0000-0000-0000-000000000000'\n -- Positive balance read directly from the total\n -- user_balance_checkpoints read model (kept current by the\n -- writers folding synchronously with each charge). The\n -- is_deleted guard mirrors the old balance CTE, which only\n -- contained non-deleted users; key deletion is not implied by\n -- user deletion, so this check is load-bearing.\n -- Keep this lookup inline so PostgreSQL can build a hashed\n -- set of enabled accounts once instead of calling a function\n -- for every model/key pair. The outer guard excludes deleted users.\n OR (u.is_deleted = false AND (EXISTS (\n SELECT 1 FROM user_feature_flags f\n WHERE f.user_id = u.id\n AND f.feature_flag = 'ALLOW_NEGATIVE_BALANCE'\n AND f.enabled\n ) OR EXISTS (\n SELECT 1 FROM user_balance_checkpoints ub\n WHERE ub.user_id = ak.user_id AND ub.balance > 0\n )))\n OR (\n -- Keep the legacy general-tariff rule; only this account's\n -- positive deal can require credit on an otherwise free model.\n NOT EXISTS (\n SELECT 1 FROM model_tariffs mt\n WHERE mt.deployed_model_id = cm.id AND mt.user_id IS NULL\n AND (mt.serving_class IS NULL OR class_routes_active(cm.id))\n AND mt.valid_until IS NULL\n AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0)\n ) AND NOT EXISTS (\n SELECT 1 FROM model_tariffs mt\n WHERE mt.deployed_model_id = cm.id AND mt.user_id = ak.user_id\n AND mt.valid_from <= NOW() AND (mt.valid_until IS NULL OR mt.valid_until > NOW())\n AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0)\n )\n )\n )\n -- Account deletion revokes access independently of key deletion.\n AND (ak.user_id = '00000000-0000-0000-0000-000000000000' OR u.is_deleted = false)\n AND ak.is_deleted = false\n -- Spending-cap gate: exclude every key of a cap scope (the capped\n -- root and its hidden batch child alike) once the scope's\n -- CALENDAR-ALIGNED (UTC, non-rolling) window spend has reached the\n -- root's limit. Guarded so the uncapped majority short-circuits on\n -- the first branch; the subquery is two PK probes. Generally free\n -- models remain usable, but zero customer deals do not bypass caps\n -- on paid models. Un-capping needs no job or traffic: the\n -- window-membership check (shared function, migration 123) turns\n -- false at the calendar boundary and the periodic fallback sync\n -- readmits the keys.\n AND (\n (ak.spend_limit IS NULL AND ak.parent_api_key_id IS NULL)\n OR NOT EXISTS (\n SELECT 1\n FROM api_keys root\n LEFT JOIN api_key_spend_checkpoints ck ON ck.api_key_id = root.id\n WHERE root.id = COALESCE(ak.parent_api_key_id, ak.id)\n AND root.spend_limit IS NOT NULL\n AND api_key_cap_window_current(ck.window_started_at, root.spend_limit_interval)\n AND ck.window_spend >= root.spend_limit\n AND (EXISTS (\n SELECT 1 FROM model_tariffs mt\n WHERE mt.deployed_model_id = cm.id AND mt.user_id IS NULL\n AND (mt.serving_class IS NULL OR class_routes_active(cm.id))\n AND mt.valid_until IS NULL\n AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0)\n ) OR EXISTS (\n SELECT 1 FROM model_tariffs mt\n WHERE mt.deployed_model_id = cm.id AND mt.user_id = ak.user_id\n AND mt.valid_from <= NOW() AND (mt.valid_until IS NULL OR mt.valid_until > NOW())\n AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0)\n ))\n )\n )\n -- Inference data plane only: platform (management) keys must never\n -- enter onwards' key set. Mirrors is_inference_purpose in\n -- db::models::api_keys (SQL cannot call it). The system key is\n -- purpose 'platform' (it calls admin endpoints, see migration 071)\n -- but is used internally for onwards inference, so it is exempt.\n AND (\n ak.user_id = '00000000-0000-0000-0000-000000000000'\n OR ak.purpose IN ('realtime', 'batch', 'playground', 'continuation')\n )\n ) ak\n WHERE cm.is_composite = TRUE\n AND cm.deleted = FALSE\n ORDER BY cm.id, ak.id\n ", "describe": { "columns": [ { @@ -60,5 +60,5 @@ true ] }, - "hash": "c2c9cd3d6e383666f056cc686f46b971dcfc39d39f0a98ec42a3b7913d2421d1" + "hash": "056f692525815bb06d25a97b61384e22c1121ba776c803b558e4fa17e7a9a42c" } diff --git a/.sqlx/query-9c7c89bf8db504e0a82390db6f812dbe4111e69606e91d5bf15e683eef6b882f.json b/.sqlx/query-0e6afd09498a445b607a343bd16bcfcbdb4f1efc6fb9177b9962431f1e246f1b.json similarity index 90% rename from .sqlx/query-9c7c89bf8db504e0a82390db6f812dbe4111e69606e91d5bf15e683eef6b882f.json rename to .sqlx/query-0e6afd09498a445b607a343bd16bcfcbdb4f1efc6fb9177b9962431f1e246f1b.json index 00f3342fd..31f00dbc9 100644 --- a/.sqlx/query-9c7c89bf8db504e0a82390db6f812dbe4111e69606e91d5bf15e683eef6b882f.json +++ b/.sqlx/query-0e6afd09498a445b607a343bd16bcfcbdb4f1efc6fb9177b9962431f1e246f1b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n id as composite_model_id,\n alias,\n realtime_inflight_limit,\n batch_capacity,\n lb_strategy,\n fallback_enabled,\n fallback_on_rate_limit,\n fallback_on_status,\n fallback_realtime_on_status,\n fallback_with_replacement,\n fallback_max_attempts,\n backoff_enabled,\n backoff_initial_ms,\n backoff_max_ms,\n backoff_factor,\n backoff_jitter,\n backoff_max_total_ms,\n first_token_timeout_ms,\n aimd,\n affinity,\n sanitize_responses,\n trusted\n FROM deployed_models\n WHERE is_composite = TRUE\n AND deleted = FALSE\n ", + "query": "\n SELECT\n id as composite_model_id,\n alias,\n realtime_inflight_limit,\n batch_capacity,\n lb_strategy,\n fallback_enabled,\n fallback_on_rate_limit,\n fallback_on_status,\n fallback_realtime_on_status,\n fallback_with_replacement,\n fallback_max_attempts,\n backoff_enabled,\n backoff_initial_ms,\n backoff_max_ms,\n backoff_factor,\n backoff_jitter,\n backoff_max_total_ms,\n first_token_timeout_ms,\n aimd,\n affinity,\n sanitize_responses,\n trusted,\n routing_mode\n FROM deployed_models\n WHERE is_composite = TRUE\n AND deleted = FALSE\n ", "describe": { "columns": [ { @@ -112,6 +112,11 @@ "ordinal": 21, "name": "trusted", "type_info": "Bool" + }, + { + "ordinal": 22, + "name": "routing_mode", + "type_info": "Text" } ], "parameters": { @@ -139,8 +144,9 @@ true, true, false, + false, false ] }, - "hash": "9c7c89bf8db504e0a82390db6f812dbe4111e69606e91d5bf15e683eef6b882f" + "hash": "0e6afd09498a445b607a343bd16bcfcbdb4f1efc6fb9177b9962431f1e246f1b" } diff --git a/.sqlx/query-478ac8aed75ee5f70690dbfbdfaa7d449ac41421177dd420a7d06318a8296249.json b/.sqlx/query-478ac8aed75ee5f70690dbfbdfaa7d449ac41421177dd420a7d06318a8296249.json deleted file mode 100644 index d238fc652..000000000 --- a/.sqlx/query-478ac8aed75ee5f70690dbfbdfaa7d449ac41421177dd420a7d06318a8296249.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT p.user_id, dm.alias, p.toleration_preset AS \"toleration_preset!\"\n FROM account_model_policies p\n JOIN deployed_models dm ON dm.id = p.deployed_model_id\n WHERE p.toleration_preset IS NOT NULL\n AND dm.deleted = FALSE\n AND (p.serving_class IS NULL OR p.serving_class = 'standard')\n AND ($1::uuid IS NULL OR p.user_id = $1)\n ORDER BY p.serving_class NULLS FIRST\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "user_id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "alias", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "toleration_preset!", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "478ac8aed75ee5f70690dbfbdfaa7d449ac41421177dd420a7d06318a8296249" -} diff --git a/.sqlx/query-0da3a55ed1d2463606a487e650d1ea70e311b3e0bdff508a7af5843ca36085a7.json b/.sqlx/query-4a6574b3e88ce099609925502c5781a46ccb1b42ba5475b1d278ba85c3a7f123.json similarity index 90% rename from .sqlx/query-0da3a55ed1d2463606a487e650d1ea70e311b3e0bdff508a7af5843ca36085a7.json rename to .sqlx/query-4a6574b3e88ce099609925502c5781a46ccb1b42ba5475b1d278ba85c3a7f123.json index 1ec663731..b2198789e 100644 --- a/.sqlx/query-0da3a55ed1d2463606a487e650d1ea70e311b3e0bdff508a7af5843ca36085a7.json +++ b/.sqlx/query-4a6574b3e88ce099609925502c5781a46ccb1b42ba5475b1d278ba85c3a7f123.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n dm.alias,\n mct.write_multiplier_5m,\n mct.write_multiplier_1h,\n mct.write_multiplier_24h,\n mct.read_multiplier,\n mct.valid_from,\n mct.valid_until,\n mct.user_id as \"account?\",\n mct.serving_class\n FROM deployed_models dm\n JOIN model_cache_tariffs mct ON mct.deployed_model_id = dm.id\n WHERE dm.alias = ANY($1)\n AND (mct.user_id IS NULL OR mct.user_id = ANY($2))\n AND (mct.user_id IS NOT NULL OR mct.serving_class IS NULL)\n ORDER BY dm.alias, mct.valid_from DESC\n ", + "query": "\n SELECT\n dm.alias,\n mct.write_multiplier_5m,\n mct.write_multiplier_1h,\n mct.write_multiplier_24h,\n mct.read_multiplier,\n mct.valid_from,\n mct.valid_until,\n mct.user_id as \"account?\",\n mct.serving_class\n FROM deployed_models dm\n JOIN model_cache_tariffs mct ON mct.deployed_model_id = dm.id\n WHERE dm.alias = ANY($1)\n AND (mct.user_id IS NULL OR mct.user_id = ANY($2))\n AND (mct.user_id IS NOT NULL OR mct.serving_class IS NULL OR class_routes_active(dm.id))\n ORDER BY dm.alias, mct.valid_from DESC\n ", "describe": { "columns": [ { @@ -67,5 +67,5 @@ true ] }, - "hash": "0da3a55ed1d2463606a487e650d1ea70e311b3e0bdff508a7af5843ca36085a7" + "hash": "4a6574b3e88ce099609925502c5781a46ccb1b42ba5475b1d278ba85c3a7f123" } diff --git a/.sqlx/query-0a49c811f08a2861700d2dd05d70860d89733ae081274551d8a6cd83a9bf3033.json b/.sqlx/query-52097c3c9be9eb05dffc11e94ef84966d29bbf444c79b6a486fc293131441c8d.json similarity index 54% rename from .sqlx/query-0a49c811f08a2861700d2dd05d70860d89733ae081274551d8a6cd83a9bf3033.json rename to .sqlx/query-52097c3c9be9eb05dffc11e94ef84966d29bbf444c79b6a486fc293131441c8d.json index de0bbadc5..78c7e3fc0 100644 --- a/.sqlx/query-0a49c811f08a2861700d2dd05d70860d89733ae081274551d8a6cd83a9bf3033.json +++ b/.sqlx/query-52097c3c9be9eb05dffc11e94ef84966d29bbf444c79b6a486fc293131441c8d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT MIN(mct.min_prefix_tokens) AS min_prefix\n FROM deployed_models dm\n JOIN model_cache_tariffs mct\n ON mct.deployed_model_id = dm.id\n AND mct.user_id IS NULL AND mct.serving_class IS NULL\n AND mct.valid_from <= now()\n AND (mct.valid_until IS NULL OR mct.valid_until > now())\n WHERE dm.alias = $1 AND dm.deleted = false\n ", + "query": "\n SELECT MIN(mct.min_prefix_tokens) AS min_prefix\n FROM deployed_models dm\n JOIN model_cache_tariffs mct\n ON mct.deployed_model_id = dm.id\n AND mct.user_id IS NULL AND mct.serving_class IS NULL\n AND mct.valid_from <= now()\n AND (mct.valid_until IS NULL OR mct.valid_until > now())\n WHERE dm.deleted = false\n AND (dm.alias = $1\n -- A class route (`alias:`) inherits its model's\n -- enablement; its cache entries are scoped by that name.\n OR (class_routes_active(dm.id) AND EXISTS (\n SELECT 1 FROM model_serving_classes c\n WHERE c.deployed_model_id = dm.id AND dm.alias || ':' || c.class_key = $1)))\n ", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "0a49c811f08a2861700d2dd05d70860d89733ae081274551d8a6cd83a9bf3033" + "hash": "52097c3c9be9eb05dffc11e94ef84966d29bbf444c79b6a486fc293131441c8d" } diff --git a/.sqlx/query-60874278f2fe3452e770a0a097d5bf57d8b9c0e526c7ee63c9268c1be7e2ee5c.json b/.sqlx/query-60874278f2fe3452e770a0a097d5bf57d8b9c0e526c7ee63c9268c1be7e2ee5c.json new file mode 100644 index 000000000..c5b1cea84 --- /dev/null +++ b/.sqlx/query-60874278f2fe3452e770a0a097d5bf57d8b9c0e526c7ee63c9268c1be7e2ee5c.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT p.user_id, dm.alias, p.serving_class, p.toleration_preset AS \"toleration_preset!\"\n FROM account_model_policies p\n JOIN deployed_models dm ON dm.id = p.deployed_model_id\n WHERE p.toleration_preset IS NOT NULL\n AND dm.deleted = FALSE\n AND ($1::uuid IS NULL OR p.user_id = $1)\n ORDER BY p.serving_class NULLS FIRST\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "alias", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "serving_class", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "toleration_preset!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "60874278f2fe3452e770a0a097d5bf57d8b9c0e526c7ee63c9268c1be7e2ee5c" +} diff --git a/.sqlx/query-65d8c0ab1d433908c405401b2eb44c776bbe2674830208f257d3cdd630077295.json b/.sqlx/query-65d8c0ab1d433908c405401b2eb44c776bbe2674830208f257d3cdd630077295.json new file mode 100644 index 000000000..b99b2b916 --- /dev/null +++ b/.sqlx/query-65d8c0ab1d433908c405401b2eb44c776bbe2674830208f257d3cdd630077295.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT dm.alias, c.class_key\n FROM model_serving_classes c\n JOIN deployed_models dm ON dm.id = c.deployed_model_id\n -- Regardless of the model's current mode: a record queued before a\n -- rollback is still recorded under its model and class (and priced\n -- at the model's price, as class prices apply only while class\n -- routes serve). A model of that exact name wins.\n WHERE c.class_key <> 'standard'\n AND dm.alias || ':' || c.class_key = ANY($1)\n AND NOT EXISTS (\n SELECT 1 FROM deployed_models named\n WHERE named.alias = dm.alias || ':' || c.class_key AND named.deleted = false)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "alias", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "class_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "65d8c0ab1d433908c405401b2eb44c776bbe2674830208f257d3cdd630077295" +} diff --git a/.sqlx/query-95bac76d1001d7010e8921cf4d531dc0de7b833f86ee0d35929c784990c7c0eb.json b/.sqlx/query-a110fbc089f38fa11f396d99054effd753021c172cb04a493d4ceebe5fe1313a.json similarity index 83% rename from .sqlx/query-95bac76d1001d7010e8921cf4d531dc0de7b833f86ee0d35929c784990c7c0eb.json rename to .sqlx/query-a110fbc089f38fa11f396d99054effd753021c172cb04a493d4ceebe5fe1313a.json index fa2155e1c..d8660aa66 100644 --- a/.sqlx/query-95bac76d1001d7010e8921cf4d531dc0de7b833f86ee0d35929c784990c7c0eb.json +++ b/.sqlx/query-a110fbc089f38fa11f396d99054effd753021c172cb04a493d4ceebe5fe1313a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT p.user_id, dm.alias, p.serving_class, p.priority, p.ttft_ms, p.itl_ms\n FROM account_model_policies p\n INNER JOIN deployed_models dm ON dm.id = p.deployed_model_id\n INNER JOIN users u ON u.id = p.user_id\n WHERE dm.deleted = FALSE\n AND u.is_deleted = FALSE\n AND (p.serving_class IS NULL OR p.serving_class = 'standard')\n ORDER BY p.serving_class NULLS FIRST\n ", + "query": "\n SELECT p.user_id, dm.alias, p.serving_class, p.priority, p.ttft_ms, p.itl_ms\n FROM account_model_policies p\n INNER JOIN deployed_models dm ON dm.id = p.deployed_model_id\n INNER JOIN users u ON u.id = p.user_id\n WHERE dm.deleted = FALSE\n AND u.is_deleted = FALSE\n ORDER BY p.serving_class NULLS FIRST\n ", "describe": { "columns": [ { @@ -46,5 +46,5 @@ true ] }, - "hash": "95bac76d1001d7010e8921cf4d531dc0de7b833f86ee0d35929c784990c7c0eb" + "hash": "a110fbc089f38fa11f396d99054effd753021c172cb04a493d4ceebe5fe1313a" } diff --git a/.sqlx/query-a860e2b6d0ee9c557d216330b65212dea553954b42cc5ac434c2272e1edef0e3.json b/.sqlx/query-a860e2b6d0ee9c557d216330b65212dea553954b42cc5ac434c2272e1edef0e3.json new file mode 100644 index 000000000..6696fe61a --- /dev/null +++ b/.sqlx/query-a860e2b6d0ee9c557d216330b65212dea553954b42cc5ac434c2272e1edef0e3.json @@ -0,0 +1,92 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n c.deployed_model_id,\n c.class_key,\n c.upstream_model_name,\n c.realtime_inflight_limit,\n dm.alias,\n dm.reasoning_translation_overrides as model_reasoning_translation_overrides,\n ie.reasoning_translation as endpoint_reasoning_translation,\n ie.url as \"endpoint_url!\",\n ie.api_key as endpoint_api_key,\n ie.auth_header_name,\n ie.auth_header_prefix,\n ie.accepts_scheduling_priority as endpoint_accepts_scheduling_priority,\n ie.kind as endpoint_kind\n FROM model_serving_classes c\n INNER JOIN deployed_models dm ON dm.id = c.deployed_model_id\n INNER JOIN inference_endpoints ie ON ie.id = c.inference_endpoint_id\n WHERE dm.routing_mode = 'class_routes'\n AND dm.deleted = FALSE\n ORDER BY c.deployed_model_id, c.class_key\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "deployed_model_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "class_key", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "upstream_model_name", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "realtime_inflight_limit", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "alias", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "model_reasoning_translation_overrides", + "type_info": "Jsonb" + }, + { + "ordinal": 6, + "name": "endpoint_reasoning_translation", + "type_info": "Jsonb" + }, + { + "ordinal": 7, + "name": "endpoint_url!", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "endpoint_api_key", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "auth_header_name", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "auth_header_prefix", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "endpoint_accepts_scheduling_priority", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "endpoint_kind", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + true, + false, + true, + false, + false, + false, + false + ] + }, + "hash": "a860e2b6d0ee9c557d216330b65212dea553954b42cc5ac434c2272e1edef0e3" +} diff --git a/.sqlx/query-56272eaf5c3432783cc071eaf5789e8ff061cffb4684eea2ba55b3c5e8a91df3.json b/.sqlx/query-a9a1a3262fa9dd88315a81698039a070ba74f6b62407a047d916aa780cbc3a39.json similarity index 91% rename from .sqlx/query-56272eaf5c3432783cc071eaf5789e8ff061cffb4684eea2ba55b3c5e8a91df3.json rename to .sqlx/query-a9a1a3262fa9dd88315a81698039a070ba74f6b62407a047d916aa780cbc3a39.json index 921152ef0..5a4eb6045 100644 --- a/.sqlx/query-56272eaf5c3432783cc071eaf5789e8ff061cffb4684eea2ba55b3c5e8a91df3.json +++ b/.sqlx/query-a9a1a3262fa9dd88315a81698039a070ba74f6b62407a047d916aa780cbc3a39.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n dm.alias,\n ie.name as \"provider_name?\",\n mt.id as \"tariff_id?\",\n mt.api_key_purpose as \"tariff_purpose?\",\n mt.valid_from as \"tariff_valid_from?\",\n mt.valid_until as \"tariff_valid_until?\",\n mt.input_price_per_token as \"tariff_input_price?\",\n mt.output_price_per_token as \"tariff_output_price?\",\n mt.completion_window as \"tariff_completion_window?\",\n mt.user_id as \"tariff_account?\",\n mt.serving_class as \"tariff_class?\"\n FROM deployed_models dm\n LEFT JOIN inference_endpoints ie ON dm.hosted_on = ie.id\n LEFT JOIN model_tariffs mt ON mt.deployed_model_id = dm.id\n AND (mt.user_id IS NULL OR mt.user_id = ANY($2))\n AND (mt.user_id IS NOT NULL OR mt.serving_class IS NULL)\n WHERE dm.alias = ANY($1)\n ORDER BY dm.alias, mt.valid_from DESC\n ", + "query": "\n SELECT\n dm.alias,\n ie.name as \"provider_name?\",\n mt.id as \"tariff_id?\",\n mt.api_key_purpose as \"tariff_purpose?\",\n mt.valid_from as \"tariff_valid_from?\",\n mt.valid_until as \"tariff_valid_until?\",\n mt.input_price_per_token as \"tariff_input_price?\",\n mt.output_price_per_token as \"tariff_output_price?\",\n mt.completion_window as \"tariff_completion_window?\",\n mt.user_id as \"tariff_account?\",\n mt.serving_class as \"tariff_class?\"\n FROM deployed_models dm\n LEFT JOIN inference_endpoints ie ON dm.hosted_on = ie.id\n LEFT JOIN model_tariffs mt ON mt.deployed_model_id = dm.id\n AND (mt.user_id IS NULL OR mt.user_id = ANY($2))\n AND (mt.user_id IS NOT NULL OR mt.serving_class IS NULL OR class_routes_active(dm.id))\n WHERE dm.alias = ANY($1)\n ORDER BY dm.alias, mt.valid_from DESC\n ", "describe": { "columns": [ { @@ -79,5 +79,5 @@ true ] }, - "hash": "56272eaf5c3432783cc071eaf5789e8ff061cffb4684eea2ba55b3c5e8a91df3" + "hash": "a9a1a3262fa9dd88315a81698039a070ba74f6b62407a047d916aa780cbc3a39" } diff --git a/.sqlx/query-bab139227c9d2d18b2d2b27c8142ca04a75ae27446be09785e131b8ab26514a4.json b/.sqlx/query-bab139227c9d2d18b2d2b27c8142ca04a75ae27446be09785e131b8ab26514a4.json new file mode 100644 index 000000000..4d034cb85 --- /dev/null +++ b/.sqlx/query-bab139227c9d2d18b2d2b27c8142ca04a75ae27446be09785e131b8ab26514a4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO model_serving_classes (deployed_model_id, class_key, display_name, inference_endpoint_id, upstream_model_name)\n VALUES ($1, 'standard', 'Standard', $2, 'upstream')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "bab139227c9d2d18b2d2b27c8142ca04a75ae27446be09785e131b8ab26514a4" +} diff --git a/.sqlx/query-06816f2ab961c5dc85297a021c7321311bd03715cfd4489e1e3997c7ac167b48.json b/.sqlx/query-be7558679287411b4a7b7e3258924e05276a7f0a481b36c440221e97d85a48a9.json similarity index 62% rename from .sqlx/query-06816f2ab961c5dc85297a021c7321311bd03715cfd4489e1e3997c7ac167b48.json rename to .sqlx/query-be7558679287411b4a7b7e3258924e05276a7f0a481b36c440221e97d85a48a9.json index 5b6075832..14b30f238 100644 --- a/.sqlx/query-06816f2ab961c5dc85297a021c7321311bd03715cfd4489e1e3997c7ac167b48.json +++ b/.sqlx/query-be7558679287411b4a7b7e3258924e05276a7f0a481b36c440221e97d85a48a9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT DISTINCT ON (cm.alias)\n cm.alias,\n first_value(dmc.render_kwargs) OVER (\n PARTITION BY dmc.composite_model_id\n ORDER BY (dmc.render_kwargs IS NULL), dmc.sort_order ASC\n ) AS render_kwargs,\n dmc.strip_leading_bos\n FROM deployed_model_components dmc\n JOIN deployed_models cm ON cm.id = dmc.composite_model_id\n JOIN deployed_models dm ON dm.id = dmc.deployed_model_id\n WHERE dmc.pool = 'completions'\n AND dmc.enabled = true\n AND cm.deleted = false\n AND dm.deleted = false\n ORDER BY\n cm.alias,\n EXISTS (\n SELECT 1 FROM deployed_model_components shared\n WHERE shared.composite_model_id = dmc.composite_model_id\n AND shared.deployed_model_id = dmc.deployed_model_id\n AND shared.pool = 'default'\n ),\n dmc.sort_order ASC\n ", + "query": "\n SELECT DISTINCT ON (cm.alias)\n cm.alias,\n first_value(dmc.render_kwargs) OVER (\n PARTITION BY dmc.composite_model_id\n ORDER BY (dmc.render_kwargs IS NULL), dmc.sort_order ASC\n ) AS render_kwargs,\n dmc.strip_leading_bos\n FROM deployed_model_components dmc\n JOIN deployed_models cm ON cm.id = dmc.composite_model_id\n JOIN deployed_models dm ON dm.id = dmc.deployed_model_id\n WHERE dmc.pool = 'completions'\n AND dmc.enabled = true\n AND cm.deleted = false\n -- A class-routed model's targets have no completions pool.\n AND NOT class_routes_active(cm.id)\n AND dm.deleted = false\n ORDER BY\n cm.alias,\n EXISTS (\n SELECT 1 FROM deployed_model_components shared\n WHERE shared.composite_model_id = dmc.composite_model_id\n AND shared.deployed_model_id = dmc.deployed_model_id\n AND shared.pool = 'default'\n ),\n dmc.sort_order ASC\n ", "describe": { "columns": [ { @@ -28,5 +28,5 @@ false ] }, - "hash": "06816f2ab961c5dc85297a021c7321311bd03715cfd4489e1e3997c7ac167b48" + "hash": "be7558679287411b4a7b7e3258924e05276a7f0a481b36c440221e97d85a48a9" } diff --git a/.sqlx/query-cca601f4ce2ac39ab6a94ff8e812973fb19d139ab3136ed18fe524d6b3b5a27a.json b/.sqlx/query-cca601f4ce2ac39ab6a94ff8e812973fb19d139ab3136ed18fe524d6b3b5a27a.json new file mode 100644 index 000000000..bed2b91c3 --- /dev/null +++ b/.sqlx/query-cca601f4ce2ac39ab6a94ff8e812973fb19d139ab3136ed18fe524d6b3b5a27a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE deployed_models SET routing_mode = 'class_routes' WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "cca601f4ce2ac39ab6a94ff8e812973fb19d139ab3136ed18fe524d6b3b5a27a" +} diff --git a/.sqlx/query-feca7d99d5ccb731a488420af2c49ea4246acffc7e894fc715a8aa8062b1fe27.json b/.sqlx/query-feca7d99d5ccb731a488420af2c49ea4246acffc7e894fc715a8aa8062b1fe27.json new file mode 100644 index 000000000..69aee3bcc --- /dev/null +++ b/.sqlx/query-feca7d99d5ccb731a488420af2c49ea4246acffc7e894fc715a8aa8062b1fe27.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT dm.alias, o.class_key, o.user_id, o.inflight_limit\n FROM realtime_inflight_class_overrides o\n INNER JOIN deployed_models dm ON dm.id = o.deployed_model_id\n WHERE dm.deleted = FALSE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "alias", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "class_key", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "user_id", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "inflight_limit", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "feca7d99d5ccb731a488420af2c49ea4246acffc7e894fc715a8aa8062b1fe27" +} diff --git a/docs/src/conceptual-guides/how-billing-works.md b/docs/src/conceptual-guides/how-billing-works.md index 0987f101a..6af30c715 100644 --- a/docs/src/conceptual-guides/how-billing-works.md +++ b/docs/src/conceptual-guides/how-billing-works.md @@ -58,20 +58,30 @@ access to a class. An existing private alias keeps its own prices; there is no implicit pricing fallback between model aliases. First discard tariffs outside their validity interval or for another account -or class. Then choose the first eligible purpose/window entry below: +or class. The remaining scopes rank in this order: + +1. **Account + class**: the account's deal for the resolved class. +2. **Account all-class**: the account's class-less deal. It covers the `standard` + class and models without classes only; another class falls through to the + next scope. +3. **Public class**: the model's own price for the class, read only for models + with `routing_mode: class_routes` (see [model provisioning][classes]). +4. **General model**: the model-level price. + +Then choose the first eligible purpose/window entry: | Priority | Realtime | Playground | Batch/flex, completion window W | |---|---|---|---| -| 1 | Account + class, realtime | Account + class, playground | Account + standard, batch W | -| 2 | Account all-class, realtime | Account + class, realtime | Account all-class, batch W | -| 3 | General model, realtime | Account all-class, playground | General model, batch W | -| 4 | — | Account all-class, realtime | Account + standard, realtime | -| 5 | — | General model, playground | Account all-class, realtime | -| 6 | — | General model, realtime | General model, realtime | +| 1 | Each scope in order, realtime | Account + class: playground, then realtime | Each scope in order, batch W (class `standard`) | +| 2 | — | Account all-class: playground, then realtime | Each scope in order, realtime (class `standard`) | +| 3 | — | Public class: playground, then realtime | — | +| 4 | — | General model: playground, then realtime | — | + +[classes]: ../reference/model-provisioning.md#serving-classes-and-class-routing Playground's realtime fallback is evaluated **within each scope** before moving to the next scope. An organisation's realtime deal therefore beats a general -playground price. Batch/flex exhausts all three exact-window batch scopes before +playground price. Batch/flex exhausts every exact-window batch scope before trying realtime prices in the same scope order, using only the standard class. A 1h batch price never substitutes for 24h. Realtime/playground use no completion window. @@ -132,7 +142,8 @@ rolls back atomically. Historical org token/cache prices prevent hard account deletion; normal soft deletion retains them. Released migrations are immutable. Cache multipliers are selected independently by account + resolved class → -account all-class → general model, at the same timestamp. They have no +account all-class (the `standard` class, or a model without classes) → public +class → general model, at the same timestamp. They have no purpose/window dimension. A general cache tariff is required for Control Layer cache pricing to apply; an organisation override alone cannot enable it. Applicable cache-read/write multipliers adjust the selected input token rate. diff --git a/docs/src/how-to/organization-model-overlays.md b/docs/src/how-to/organization-model-overlays.md index a06935996..3f1eaec08 100644 --- a/docs/src/how-to/organization-model-overlays.md +++ b/docs/src/how-to/organization-model-overlays.md @@ -117,9 +117,11 @@ models: read_multiplier: "0.15" ``` -A request's serving class is where it is routed, and `class_pricing` prices one -of the model's classes (`standard` until models carry more). A class price wins -over the all-class `tariffs` for requests in that class. Batch and flex use +A request's serving class is where it is routed, and `class_pricing` prices the +model's `standard` class. A class price wins over the all-class `tariffs` for +requests in that class. The all-class `tariffs` cover `standard` only: other +classes of a class-routed model are billed at the model's price for that class, +or at a price set in the account file. Batch and flex use standard; flex uses the batch `1h` tariff. Prefer putting batch prices in the all-class `tariffs` list. @@ -131,8 +133,9 @@ for alerting; nothing routes on them. ## Billing, display and admission -Billing selects the key owner's class-specific price, then its all-class price, -then the general model price. Each scope must match the purpose and, for batch, +Billing selects the key owner's class-specific price, then its all-class price +(for the `standard` class only), then the model's price for the class, then the +general model price. Each scope must match the purpose and, for batch, the exact completion window. Playground may fall back to realtime within the same scope. Batch first exhausts exact-window prices across standard-class, all-class organization and general model scopes. Only if none matches does it diff --git a/docs/src/reference/model-provisioning.md b/docs/src/reference/model-provisioning.md index d6e84d133..72a7d0959 100644 --- a/docs/src/reference/model-provisioning.md +++ b/docs/src/reference/model-provisioning.md @@ -139,24 +139,24 @@ its pools; repeating the deployment declaration would be an alias collision. Every virtual model requires a non-empty `routing.pools.default`. The only pool names currently supported are `default` and `completions`. -## Staging class destinations and public prices +## Serving classes and class routing -`clay.class_routes` stages explicit destinations on the existing model UUID. This -is preparatory storage: this release continues using the legacy route, general -price, cache configuration and model discovery. Public class prices are deliberately -excluded from runtime tariff/cache histories and existing dashboard price panels; -customer-specific class deals retain their established billing and credit checks. -This release does not accept new inference names or enable class routing. Keep the existing `deployments`, `routing`, -`tariffs` and `cache_tariff` definitions while staging classes. +A model can offer more than one serving class. `model` routes to its `standard` +class and `model:` to any other class it declares, each with its own +upstream, realtime prices and default realtime limit. Classes are declared in +`clay.class_routes`; they take effect only once the model sets +`routing_mode: class_routes`. Until then the model keeps its legacy route +(`deployments`, `routing`, `tariffs`), and declared classes are stored but unused. ```yaml # Merge into the existing clay block, retaining its other fields: clay: + routing_mode: class_routes # default: legacy class_routes: standard: display_name: Standard endpoint: gateway - upstream_model_name: dynamo-example/model:throughput + upstream_model_name: example/model tariffs: - name: Standard realtime purpose: realtime @@ -165,8 +165,8 @@ clay: fast: display_name: Fast endpoint: gateway - upstream_model_name: dynamo-example/model:fast - aliases: [example/model-fast] + upstream_model_name: example/model-fast + realtime_inflight_limit: 20 tariffs: - name: Fast realtime purpose: realtime @@ -179,51 +179,90 @@ clay: read_multiplier: "0.1" ``` -The endpoint must exist. Upstream names are configurable, endpoint-scoped strings -that must be nonempty and contain no whitespace. Classes may share a destination. -Class keys retain their IDs when a display name, -destination or price changes. Nonempty class configuration requires `standard` -and `fast`, each with a realtime tariff. Additional keys may use lowercase letters, -digits, hyphens and underscores, starting with a letter. The retired numeric-target -`serving_classes` declaration is accepted only as an empty map; remove it. - -Batch tariffs are allowed only under `standard`: batch always resolves to that -class regardless of the submitted name. Other classes cannot declare batch prices -that would never be selected. Model-level batch tariffs remain supported. - -Optional aliases are exact synonyms, not discovery entries or extra model rows. -Do not list the primary bare name or generated `:fast` name as synonyms. Validation -reserves primary class names and rejects collisions with existing models and other -synonyms. Ordinary model creates/renames share the catalog lock and collision check. -Synonyms cannot be rebound to another model/class by a catalog edit; deliberate -private-name migration requires a later, separately validated procedure. - -For a model present in the catalog, class routes and synonyms are complete desired -state. Omitted synonyms are removed; omitted classes are removed and their public -prices retired, preserving tariff history. An unchanged declaration preserves IDs, -timestamps and price versions. A whole model omitted from the catalog follows the -existing ownership-release rule below, rather than deleting its classes. +Rules: + +- `standard` is required whenever `class_routes` is nonempty, and + `routing_mode: class_routes` requires a declared `standard` class. Other classes + are optional. Keys use lowercase letters, digits, hyphens and underscores, + starting with a letter. Every class needs a realtime tariff. +- The endpoint must exist. Upstream names are endpoint-scoped strings, nonempty + and without whitespace. Classes may share a destination. +- `realtime_inflight_limit` caps one account's concurrent realtime requests to + that class. It must be positive; omitted, the class uses the model's + `settings.realtime_inflight_limit`. Each class counts separately. +- Batch tariffs are allowed only under `standard`. Model-level batch tariffs + remain supported. +- `aliases` must be empty: class names are generated (`model`, `model:`), + and synonyms are not supported. The retired numeric-target `serving_classes` + declaration is accepted only as an empty map. +- An activated model cannot carry redirect traffic rules. Deny rules apply to + every class. + +### What activation changes + +With `routing_mode: class_routes`, the gateway serves one route per class: `model` +for `standard` and `model:` for the rest. Each route forwards to its class's +endpoint and upstream name, without the composite's members or stream +continuation. The model's fallback settings still apply as retries against that +one upstream. `self_hosted_only` does not apply: the class names its upstream. +Every route shares the model's deny rules, and only `standard` carries the +model's batch capacity. + +- **Realtime only.** Flex, background and batch requests use the `standard` + class. A flex or background request naming `model:` is refused with + 400 `serving_class_realtime_only`; a batch line naming it is refused at upload. +- **Discovery.** `/v1/models` lists `model:` beside the model for each + class other than `standard`, with the reasoning efforts its upstream supports. + Class entries carry no prices yet. +- **Analytics and billing.** A request to `model:` is recorded under the + model's name with `requested_serving_class` and `resolved_serving_class` set to + the class, and is priced as that class ([how billing works][billing]). +- **Prompt caching.** A class route inherits its model's cache enablement and + minimum prefix. `standard` keeps the model's cache scope; other classes use + their own, because they reach different engines. +- **Account settings.** Account files can set priority, scheduling hints, + toleration presets, realtime limits and prices for any class the model declares. + A setting without a class applies to the model as a whole; a class's own setting + wins for that class. + +An activation the gateway cannot serve (no `standard` class, or redirect rules +added outside the catalogue) falls back to the legacy route and is logged as +`invalid_class_activation`. + +To roll back on a release that supports class routing, set `routing_mode: legacy` +and redeploy. Keep the legacy `deployments`, `routing`, `tariffs` and +`cache_tariff` in place while a model is activated, so the rollback has a route +and prices to return to; class caching also inherits the model's `cache_tariff`. +The catalogue owns the column: it is written on every reconcile, and a model +omitted from the catalogue keeps its last value. Class prices apply only while +class routes serve, so a class request still waiting to be billed when the model +rolls back is recorded in its class but priced at the model's price. + +To roll back to an older release, first deploy the catalogue with +`routing_mode: legacy` on every model. Then remove the `routing_mode` and +`class_routes..realtime_inflight_limit` fields, and any `class_routes` +without a `fast` class, before deploying the older image: older releases reject +those at startup. + +### Ownership + +For a model present in the catalogue, `class_routes` is complete desired state. +Omitted classes are removed and their public prices retired, preserving tariff +history. An unchanged declaration preserves IDs, timestamps and price versions. +Class keys keep their IDs when a display name, destination, limit or price +changes. A whole model omitted from the catalogue follows the existing +ownership-release rule below rather than deleting its classes. Public class tariffs use the existing ledger with `user_id = NULL` and a class -key. The future resolver order is account/class, account/all-classes, public/class, -then general compatibility price. Batch still exhausts exact completion-window -prices before its standard realtime fallback; zero is a price. Class cache tariffs -contain multipliers, not token prices. They inherit model-level cache enablement -and minimum prefix length; omission inherits the model's multipliers. Declaring -class multipliers requires a model-level `cache_tariff`. - -The database's `routing_mode` defaults to `legacy` and is outside catalog ownership; -putting it in YAML is rejected. This release is **not activation-ready**: request -resolution, billing snapshots, reactive forwarding and writer support must land -before switching any model. The catalog refuses edits to models already set to -`class_routes`, instead of overwriting an active route with a composite definition. -The class-aware SQL resolver is not an activation mechanism: switching this flag -manually would bypass the supported rollout boundary. Class-aware admission, -durable billing/cache context and dashboard readers must be integrated together -before enabling it. Do not enable individual public-price readers in this release. -Deploy this support release and update the catalog validator before publishing -catalogs containing the new field. Do not run an older binary against staged public -class prices: its readers do not distinguish them from general prices. +key, and are read only for models with `routing_mode: class_routes`. Class cache +tariffs contain multipliers, not token prices; omission inherits the model's +multipliers, and declaring them requires a model-level `cache_tariff`. + +Deploy a release that supports these fields, and update the catalogue validator, +before publishing catalogues that use them: older releases reject the +`routing_mode` field and refuse to start. + +[billing]: ../conceptual-guides/how-billing-works.md#organisation-and-serving-class-price-precedence ## Authoritative behavior @@ -413,8 +452,15 @@ realtime_inflight: Rules: -- `classes` may name only `standard` for now. A class the file omits keeps the - model's defaults. +- `classes` may name `standard` and any class the model declares in its + catalogue [`class_routes`](#serving-classes-and-class-routing). A class the + file omits keeps the model's defaults. Settings for a class other than + `standard` take effect once the model is activated (`routing_mode: + class_routes`); its limit counts that class's requests only. +- Model-level `priority` and `toleration_preset` apply to every class; class-level + ones override them for that class. Prices do not carry across classes: a class + without its own `tariffs` in the file is billed at the model's price for that + class. - Limits, realtime and playground prices, and the cache tariff are declared per class; model-level `tariffs` take batch prices only. - Priority inherits class, then model, then account, then the default. It is @@ -461,7 +507,7 @@ dwctl-model-provisioning validate-account-limits ./account-limits.d \ dwctl-model-provisioning account-schema ``` -It rejects unknown fields and classes, limits and latency targets below 1, an +It rejects unknown fields, classes the model does not declare, limits and latency targets below 1, an account declared in two files, entries that configure nothing, prices at the wrong level, models absent from the model catalog and, with `--org-overlays`, pairs configured in both catalogs. Whether each account exists can only be diff --git a/dwctl/migrations/20261022120000_class_routing.sql b/dwctl/migrations/20261022120000_class_routing.sql new file mode 100644 index 000000000..2288d220a --- /dev/null +++ b/dwctl/migrations/20261022120000_class_routing.sql @@ -0,0 +1,78 @@ +-- Class routing: a class's own realtime limit, per-account limits on classes +-- other than standard, and config notifications when classes change. All +-- additive; nothing routes on these until a model is set to class_routes. +SET LOCAL lock_timeout = '5s'; + +ALTER TABLE model_serving_classes ADD COLUMN realtime_inflight_limit INTEGER + CONSTRAINT chk_model_serving_classes_realtime_inflight_limit CHECK (realtime_inflight_limit > 0); + +COMMENT ON COLUMN model_serving_classes.realtime_inflight_limit IS + 'Concurrent realtime requests one account may have in flight on this class. NULL uses the model''s realtime_inflight_limit.'; + +COMMENT ON COLUMN deployed_models.routing_mode IS + 'legacy routes through the model''s composite; class_routes routes each serving class to its own upstream. Set by the model catalog.'; + +CREATE TRIGGER model_serving_classes_notify + AFTER INSERT OR UPDATE OR DELETE ON model_serving_classes + EXECUTE FUNCTION notify_config_change(); + +-- Per-account realtime limits on a class other than standard. The standard +-- class keeps realtime_inflight_limit_overrides, so its limiter key and rows +-- are unchanged when a model is activated. +CREATE TABLE realtime_inflight_class_overrides ( + deployed_model_id UUID NOT NULL REFERENCES deployed_models(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + class_key TEXT NOT NULL CONSTRAINT chk_realtime_inflight_class_overrides_class CHECK (class_key ~ '^[a-z][a-z0-9_-]*$' AND class_key <> 'standard'), + inflight_limit INTEGER NOT NULL CONSTRAINT chk_realtime_inflight_class_overrides_limit CHECK (inflight_limit > 0), + PRIMARY KEY (deployed_model_id, user_id, class_key) +); + +COMMENT ON TABLE realtime_inflight_class_overrides IS + 'Per-account realtime in-flight limits on a non-standard serving class, from the account catalog. Standard-class limits live in realtime_inflight_limit_overrides.'; + +CREATE INDEX idx_realtime_inflight_class_overrides_user_id ON realtime_inflight_class_overrides (user_id); + +CREATE TRIGGER realtime_inflight_class_overrides_notify + AFTER INSERT OR UPDATE OR DELETE ON realtime_inflight_class_overrides + EXECUTE FUNCTION notify_config_change(); + +-- Whether a model serves its class routes: set to class_routes, with a +-- standard class and no redirect rule. Otherwise the gateway serves its +-- legacy route, and every reader of class data must agree. +CREATE FUNCTION class_routes_active(model_id UUID) RETURNS BOOLEAN LANGUAGE SQL STABLE AS $$ + SELECT EXISTS (SELECT 1 FROM deployed_models dm WHERE dm.id = model_id AND dm.routing_mode = 'class_routes') + AND EXISTS (SELECT 1 FROM model_serving_classes c WHERE c.deployed_model_id = model_id AND c.class_key = 'standard') + AND NOT EXISTS (SELECT 1 FROM model_traffic_rules r WHERE r.deployed_model_id = model_id AND r.action = 'redirect') +$$; + +COMMENT ON FUNCTION class_routes_active(UUID) IS + 'True when the gateway serves the model''s class routes; false when it serves the legacy route.'; + +-- An account's class-less deal covers the standard class only: any other +-- class is priced by the account's deal for that class, else the model's. +CREATE OR REPLACE FUNCTION effective_model_tariff( + model_id UUID, account_id UUID, purpose TEXT, completion_window TEXT, + resolved_class TEXT, at_time TIMESTAMPTZ +) RETURNS SETOF model_tariffs LANGUAGE SQL STABLE AS $$ + SELECT mt.id, mt.deployed_model_id, mt.name, mt.input_price_per_token, mt.output_price_per_token, mt.valid_from, mt.valid_until, mt.api_key_purpose, mt.completion_window, mt.user_id, mt.serving_class, mt.provisioning_source + FROM model_tariffs mt + WHERE COALESCE(purpose, 'realtime') IN ('realtime','batch','playground') + AND mt.deployed_model_id = model_id + AND (mt.user_id = account_id OR mt.user_id IS NULL) + AND (mt.user_id IS NOT NULL OR mt.serving_class IS NULL OR class_routes_active(model_id)) + AND (mt.serving_class = CASE WHEN purpose = 'batch' THEN 'standard' ELSE resolved_class END OR mt.serving_class IS NULL) + AND NOT (mt.user_id IS NOT NULL AND mt.serving_class IS NULL + AND COALESCE(CASE WHEN purpose = 'batch' THEN 'standard' ELSE resolved_class END, 'standard') <> 'standard') + AND mt.valid_from <= at_time AND (mt.valid_until IS NULL OR mt.valid_until > at_time) + AND (mt.api_key_purpose = COALESCE(purpose, 'realtime') + OR (purpose IN ('playground','batch') AND mt.api_key_purpose = 'realtime')) + AND mt.completion_window IS NOT DISTINCT FROM + CASE WHEN mt.api_key_purpose = 'batch' THEN effective_model_tariff.completion_window ELSE NULL END + ORDER BY CASE WHEN purpose = 'batch' AND mt.api_key_purpose = 'realtime' THEN 1 ELSE 0 END, + CASE WHEN mt.user_id = account_id AND mt.serving_class = CASE WHEN purpose = 'batch' THEN 'standard' ELSE resolved_class END THEN 0 + WHEN mt.user_id = account_id THEN 1 + WHEN mt.serving_class IS NOT NULL THEN 2 ELSE 3 END, + CASE WHEN mt.api_key_purpose = COALESCE(purpose, 'realtime') THEN 0 ELSE 1 END, + mt.valid_from DESC, mt.id + LIMIT 1 +$$; diff --git a/dwctl/src/account_limits.rs b/dwctl/src/account_limits.rs index 86668f2b0..1b1ed0cd2 100644 --- a/dwctl/src/account_limits.rs +++ b/dwctl/src/account_limits.rs @@ -35,9 +35,13 @@ const ACCOUNT_LIMITS_LOCK: i64 = 0x4457_4143_4354_4c4d; /// Ownership marker on policy rows written by this catalog. const POLICY_SOURCE: &str = "account-catalog"; -/// Classes an account file may configure. Only `standard` exists until -/// models gain further serving classes. -const CONFIGURABLE_CLASSES: [&str; 1] = ["standard"]; +/// A class key: lowercase, as the model catalog's `class_routes` keys. +fn is_class_key(key: &str) -> bool { + key.bytes().next().is_some_and(|c| c.is_ascii_lowercase()) + && key + .bytes() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'_' || c == b'-') +} const STANDARD_CLASS: &str = "standard"; #[derive(Debug, Clone)] @@ -148,7 +152,22 @@ impl AccountDocument { self.realtime_inflight.keys().chain(self.models.keys()) } - /// The effective limits, merging both shapes (equal duplicates are one limit). + /// Realtime limits on classes other than standard, by (alias, class). + fn class_limits(&self) -> BTreeMap<(&str, &str), i32> { + self.models + .iter() + .flat_map(|(alias, model)| { + model + .classes + .iter() + .filter(|(class, _)| class.as_str() != STANDARD_CLASS) + .filter_map(move |(class, policy)| policy.realtime_inflight.map(|limit| ((alias.as_str(), class.as_str()), limit))) + }) + .collect() + } + + /// The effective standard-class limits, merging both shapes (equal + /// duplicates are one limit). fn limits(&self) -> BTreeMap<&str, i32> { let mut limits: BTreeMap<&str, i32> = self .realtime_inflight @@ -201,14 +220,23 @@ impl AccountLimitsCatalog { } pub fn validate_models(&self, models: &Catalog) -> Result<()> { - let aliases: HashSet<_> = models.models.iter().map(|model| model.clay.alias.as_str()).collect(); + let catalog: HashMap<_, _> = models + .models + .iter() + .map(|model| (model.clay.alias.as_str(), &model.clay.class_routes)) + .collect(); for file in &self.files { for alias in file.document.aliases() { - ensure!( - aliases.contains(alias.as_str()), - "{}: model {alias:?} is absent from the model catalog", - file.source - ); + let Some(classes) = catalog.get(alias.as_str()) else { + bail!("{}: model {alias:?} is absent from the model catalog", file.source); + }; + for class in file.document.models.get(alias).into_iter().flat_map(|model| model.classes.keys()) { + ensure!( + class == STANDARD_CLASS || classes.contains_key(class), + "{}: model {alias:?} has no class {class:?} in the model catalog", + file.source + ); + } } } Ok(()) @@ -289,11 +317,7 @@ impl AccountLimitsCatalog { ); for (class, policy) in &model.classes { let context = format!("{context}, class {class:?}"); - ensure!( - CONFIGURABLE_CLASSES.contains(&class.as_str()), - "{context}: unknown class; configurable classes are {}", - CONFIGURABLE_CLASSES.join(", ") - ); + ensure!(is_class_key(class), "{context}: invalid class key"); ensure!(!policy.is_empty(), "{context} configures nothing"); if let Some(limit) = policy.realtime_inflight { ensure!(limit > 0, "{context}: realtime_inflight must be positive"); @@ -391,7 +415,9 @@ pub async fn apply(pool: &PgPool, catalog: &AccountLimitsCatalog) -> Result<()> .await .context("read account catalog effective timestamp")?; + verify_classes(&mut transaction, catalog, &models).await?; apply_limits(&mut transaction, catalog, &accounts, &models).await?; + apply_class_limits(&mut transaction, catalog, &accounts, &models).await?; apply_priorities(&mut transaction, catalog, &accounts).await?; apply_policies(&mut transaction, catalog, &accounts, &models).await?; apply_prices(&mut transaction, catalog, &accounts, &models, effective_at).await?; @@ -445,6 +471,79 @@ async fn apply_limits( Ok(()) } +/// A class other than standard must exist on the model (the model catalog, +/// applied just before, declares its classes). +async fn verify_classes(db: &mut PgConnection, catalog: &AccountLimitsCatalog, models: &HashMap) -> Result<()> { + let declared: HashSet<(Uuid, String)> = sqlx::query_as("SELECT deployed_model_id, class_key FROM model_serving_classes") + .fetch_all(&mut *db) + .await + .context("read model classes")? + .into_iter() + .collect(); + for file in &catalog.files { + for (alias, model) in &file.document.models { + for class in model.classes.keys().filter(|class| class.as_str() != STANDARD_CLASS) { + ensure!( + declared.contains(&(models[alias], class.clone())), + "{}: model {alias:?} has no class {class:?}", + file.source + ); + } + } + } + Ok(()) +} + +/// Per-account limits on non-standard classes, owned table-wide like the +/// standard-class limits. +async fn apply_class_limits( + db: &mut PgConnection, + catalog: &AccountLimitsCatalog, + accounts: &HashMap, + models: &HashMap, +) -> Result<()> { + let mut model_ids = Vec::new(); + let mut account_ids = Vec::new(); + let mut classes = Vec::new(); + let mut limits = Vec::new(); + for file in &catalog.files { + for ((alias, class), limit) in file.document.class_limits() { + model_ids.push(models[alias]); + account_ids.push(accounts[&file.document.account]); + classes.push(class.to_owned()); + limits.push(limit); + } + } + sqlx::query( + "DELETE FROM realtime_inflight_class_overrides o + WHERE NOT EXISTS ( + SELECT 1 FROM UNNEST($1::uuid[], $2::uuid[], $3::text[]) AS d(deployed_model_id, user_id, class_key) + WHERE d.deployed_model_id = o.deployed_model_id AND d.user_id = o.user_id AND d.class_key = o.class_key + )", + ) + .bind(&model_ids) + .bind(&account_ids) + .bind(&classes) + .execute(&mut *db) + .await + .context("remove undeclared account class limits")?; + sqlx::query( + "INSERT INTO realtime_inflight_class_overrides (deployed_model_id, user_id, class_key, inflight_limit) + SELECT d.deployed_model_id, d.user_id, d.class_key, d.inflight_limit + FROM UNNEST($1::uuid[], $2::uuid[], $3::text[], $4::int[]) AS d(deployed_model_id, user_id, class_key, inflight_limit) + ON CONFLICT (deployed_model_id, user_id, class_key) DO UPDATE SET inflight_limit = EXCLUDED.inflight_limit + WHERE realtime_inflight_class_overrides.inflight_limit IS DISTINCT FROM EXCLUDED.inflight_limit", + ) + .bind(&model_ids) + .bind(&account_ids) + .bind(&classes) + .bind(&limits) + .execute(&mut *db) + .await + .context("write account class limits")?; + Ok(()) +} + /// The account-wide priority column is written only by this catalog: declared /// accounts get their value (or none), every other account is cleared. async fn apply_priorities(db: &mut PgConnection, catalog: &AccountLimitsCatalog, accounts: &HashMap) -> Result<()> { @@ -627,34 +726,43 @@ async fn apply_prices( .reconcile_cache_tariff(model_id, Some(account), None, None, effective_at) .await .with_context(|| context(alias))?; - let standard = model.classes.get(STANDARD_CLASS); - provisioning - .reconcile_tariffs( - model_id, - Some(account), - Some(STANDARD_CLASS), - standard.map_or(&[], |class| class.tariffs.as_slice()), - effective_at, - ) - .await - .with_context(|| context(alias))?; - let cache = standard.and_then(|class| class.cache_tariff.as_ref()).map(CachePrices::as_tariff); - provisioning - .reconcile_cache_tariff(model_id, Some(account), Some(STANDARD_CLASS), cache.as_ref(), effective_at) - .await - .with_context(|| context(alias))?; - // Rows adopted under any other class scope are not declared: close them. + // Standard is always reconciled (an omitted standard retires its + // prices); every other declared class too. + let declared_classes: Vec<&str> = std::iter::once(STANDARD_CLASS) + .chain(model.classes.keys().map(String::as_str).filter(|class| *class != STANDARD_CLASS)) + .collect(); + for class in &declared_classes { + let prices = model.classes.get(*class); + provisioning + .reconcile_tariffs( + model_id, + Some(account), + Some(class), + prices.map_or(&[], |p| p.tariffs.as_slice()), + effective_at, + ) + .await + .with_context(|| context(alias))?; + let cache = prices.and_then(|p| p.cache_tariff.as_ref()).map(CachePrices::as_tariff); + provisioning + .reconcile_cache_tariff(model_id, Some(account), Some(class), cache.as_ref(), effective_at) + .await + .with_context(|| context(alias))?; + } + // Rows this catalog owns under a class the file no longer declares: close them. + let declared_classes: Vec = declared_classes.into_iter().map(str::to_owned).collect(); for table in ["model_tariffs", "model_cache_tariffs"] { sqlx::query(&format!( "UPDATE {table} SET valid_until = $4 WHERE user_id = $1 AND deployed_model_id = $2 AND provisioning_source = $3 - AND serving_class IS NOT NULL AND serving_class <> 'standard' + AND serving_class IS NOT NULL AND NOT (serving_class = ANY($5)) AND valid_from <= $4 AND (valid_until IS NULL OR valid_until > $4)" )) .bind(account) .bind(model_id) .bind(ACCOUNT_CATALOG_TARIFF_SOURCE) .bind(effective_at) + .bind(&declared_classes) .execute(&mut *db) .await .with_context(|| context(alias))?; @@ -899,8 +1007,8 @@ mod tests { "batch prices belong in the model's", ), ( - "account: acme\nmodels:\n org/model:\n classes:\n fast:\n realtime_inflight: 5\n".to_owned(), - "unknown class", + "account: acme\nmodels:\n org/model:\n classes:\n Fast:\n realtime_inflight: 5\n".to_owned(), + "invalid class key", ), ( "account: acme\nrealtime_inflight:\n org/model: 10\nmodels:\n org/model:\n classes:\n standard:\n realtime_inflight: 20\n".to_owned(), @@ -1265,4 +1373,104 @@ mod tests { apply(&pool, &AccountLimitsCatalog::load(directory.path()).unwrap()).await.unwrap(); assert!(presets().await.is_empty()); } + + async fn insert_class(pool: &PgPool, model: Uuid, class: &str) { + let endpoint: Uuid = sqlx::query_scalar( + "INSERT INTO inference_endpoints (name, url, created_by) VALUES ($1, 'http://upstream.test', '00000000-0000-0000-0000-000000000000') RETURNING id", + ) + .bind(format!("endpoint-{class}")) + .fetch_one(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO model_serving_classes (deployed_model_id, class_key, display_name, inference_endpoint_id, upstream_model_name) VALUES ($1, $2, $2, $3, $2)", + ) + .bind(model) + .bind(class) + .bind(endpoint) + .execute(pool) + .await + .unwrap(); + } + + #[dwctl_test_macros::test] + async fn a_declared_class_carries_its_own_limits_and_prices(pool: PgPool) { + let org = insert_account(&pool, "acme", "organization").await; + let model = insert_virtual_model(&pool, "org/model").await; + let directory = tempdir().unwrap(); + write( + directory.path(), + "acme.yaml", + &format!( + "account: acme\nmodels:\n org/model:\n classes:\n standard:\n realtime_inflight: 40\n fast:\n realtime_inflight: 5\n tariffs: [{REALTIME}]\n" + ), + ); + let catalog = AccountLimitsCatalog::load(directory.path()).unwrap(); + + // A class the model does not have is refused before anything is written. + let error = format!("{:#}", apply(&pool, &catalog).await.unwrap_err()); + assert!(error.contains("has no class \"fast\""), "{error}"); + assert!(stored(&pool).await.is_empty()); + + insert_class(&pool, model, "standard").await; + insert_class(&pool, model, "fast").await; + apply(&pool, &catalog).await.unwrap(); + assert_eq!(stored(&pool).await, vec![(model, org, 40)], "standard keeps its table and key"); + let class_limits: Vec<(Uuid, Uuid, String, i32)> = + sqlx::query_as("SELECT deployed_model_id, user_id, class_key, inflight_limit FROM realtime_inflight_class_overrides") + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(class_limits, vec![(model, org, "fast".to_owned(), 5)]); + assert_eq!( + active_prices(&pool, org).await, + vec![(Some("fast".into()), "realtime".into(), None, ACCOUNT_CATALOG_TARIFF_SOURCE.into())] + ); + + // Dropping the class from the file clears its limit and retires its price. + write( + directory.path(), + "acme.yaml", + "account: acme\nmodels:\n org/model:\n classes:\n standard:\n realtime_inflight: 40\n", + ); + apply(&pool, &AccountLimitsCatalog::load(directory.path()).unwrap()).await.unwrap(); + let remaining: i64 = sqlx::query_scalar("SELECT count(*) FROM realtime_inflight_class_overrides") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(remaining, 0); + assert!(active_prices(&pool, org).await.is_empty()); + } + + #[test] + fn offline_validation_checks_classes_against_the_model_catalog() { + let models = tempdir().unwrap(); + fs::write( + models.path().join("model.yaml"), + "model: org/model\nclay:\n alias: org/model\n deployments:\n - {alias: provider, model_name: org/model, endpoint: onwards}\n routing:\n pools:\n default:\n - deployment: provider\n", + ) + .unwrap(); + let models = Catalog::load(models.path()).unwrap(); + let directory = tempdir().unwrap(); + write( + directory.path(), + "a.yaml", + "account: acme\nmodels:\n org/model:\n classes:\n fast:\n priority: 1\n", + ); + let error = AccountLimitsCatalog::load(directory.path()) + .unwrap() + .validate_models(&models) + .unwrap_err() + .to_string(); + assert!(error.contains("has no class \"fast\""), "{error}"); + write( + directory.path(), + "a.yaml", + "account: acme\nmodels:\n org/model:\n classes:\n standard:\n priority: 1\n", + ); + AccountLimitsCatalog::load(directory.path()) + .unwrap() + .validate_models(&models) + .unwrap(); + } } diff --git a/dwctl/src/api/handlers/ai_models.rs b/dwctl/src/api/handlers/ai_models.rs index f30b4e54a..64601c414 100644 --- a/dwctl/src/api/handlers/ai_models.rs +++ b/dwctl/src/api/handlers/ai_models.rs @@ -278,8 +278,28 @@ pub async fn list_ai_models( .await .map_err(|e| database_error("list_accessible_models", e))?; + // A class-routed model also answers to `alias:` for each class + // other than standard; list those beside the model. + let class_routes: Vec<(String, String)> = { + let model_ids = rows.iter().map(|row| row.get::("id")).collect::>(); + sqlx::query_as( + "SELECT dm.alias, c.class_key FROM model_serving_classes c + JOIN deployed_models dm ON dm.id = c.deployed_model_id + WHERE class_routes_active(dm.id) AND c.class_key <> 'standard' AND dm.id = ANY($1) + ORDER BY dm.alias, c.class_key", + ) + .bind(&model_ids) + .fetch_all(&mut *conn) + .await + .map_err(|e| database_error("list_class_routes", e))? + }; + let reasoning_policies = if include_reasoning_capabilities { - let aliases = rows.iter().map(|row| row.get("alias")).collect::>(); + let aliases = rows + .iter() + .map(|row| row.get("alias")) + .chain(class_routes.iter().map(|(alias, class)| format!("{alias}:{class}"))) + .collect::>(); Deployments::new(&mut conn) .get_reasoning_policies(&aliases) .await @@ -335,29 +355,54 @@ pub async fn list_ai_models( Default::default() }; + let mut data: Vec = rows + .into_iter() + .map(|row| { + let id: String = row.get("alias"); + let supported_reasoning_efforts = include_reasoning_capabilities + .then(|| reasoning_policies.get(&id).and_then(|policy| policy.supported_efforts())) + .flatten(); + let (tariffs, cache_pricing) = pricing_models + .remove(&id) + .map(|model| (model.tariffs, model.cache_pricing)) + .unwrap_or_default(); + ModelObject { + id, + object: "model".to_string(), + created: row.get::, _>("created").unwrap_or_default(), + owned_by: "None".to_string(), + supported_reasoning_efforts, + tariffs, + cache_pricing, + } + }) + .collect(); + // Class entries share the model's metadata, with reasoning efforts from + // their own upstream; their prices are not listed here yet. + let classes: Vec = class_routes + .into_iter() + .filter_map(|(alias, class)| { + let model = data.iter().find(|model| model.id == alias)?; + let id = format!("{alias}:{class}"); + let supported_reasoning_efforts = include_reasoning_capabilities + .then(|| reasoning_policies.get(&id).and_then(|policy| policy.supported_efforts())) + .flatten(); + Some(ModelObject { + id, + object: "model".to_string(), + created: model.created, + owned_by: model.owned_by.clone(), + supported_reasoning_efforts, + tariffs: None, + cache_pricing: None, + }) + }) + .collect(); + data.extend(classes); + data.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(Json(ModelsListResponse { object: "list".to_string(), - data: rows - .into_iter() - .map(|row| { - let id: String = row.get("alias"); - let supported_reasoning_efforts = include_reasoning_capabilities - .then(|| reasoning_policies.get(&id).and_then(|policy| policy.supported_efforts())) - .flatten(); - let (tariffs, cache_pricing) = pricing_models - .remove(&id) - .map(|model| (model.tariffs, model.cache_pricing)) - .unwrap_or_default(); - ModelObject { - id, - object: "model".to_string(), - created: row.get::, _>("created").unwrap_or_default(), - owned_by: "None".to_string(), - supported_reasoning_efforts, - tariffs, - cache_pricing, - } - }) - .collect(), + data, })) } diff --git a/dwctl/src/api/handlers/files.rs b/dwctl/src/api/handlers/files.rs index 2d49d10b6..aa4ee4757 100644 --- a/dwctl/src/api/handlers/files.rs +++ b/dwctl/src/api/handlers/files.rs @@ -45,7 +45,7 @@ use futures::StreamExt; use futures::stream::Stream; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::pin::Pin; use std::sync::{Arc, Mutex}; use tokio::sync::mpsc; @@ -180,6 +180,7 @@ impl OpenAIBatchRequest { accessible_models: &HashMap, allowed_url_paths: &[String], tolerations: &AccountTolerations, + class_routes: &HashSet, ) -> Result { // Validate custom_id is safe for HTTP headers validate_custom_id(&self.custom_id)?; @@ -200,6 +201,15 @@ impl OpenAIBatchRequest { })? .to_string(); + // A class other than standard serves realtime only; batch work runs + // on the model's standard class. + if class_routes.contains(&model) { + let alias = model.split_once(':').map_or(model.as_str(), |(alias, _)| alias); + return Err(Error::BadRequest { + message: format!("Model '{model}' is available for realtime requests only; use '{alias}' in batch files."), + }); + } + // Batches use deadline scheduling, so a model suffix is ignored. // Resolve the base model before validating access. let (model, _) = crate::inference::model_selector::split_model_suffix(&model).map_err(|error| Error::BadRequest { @@ -811,6 +821,9 @@ struct FileRequestContext { api_key: String, accessible_models: HashMap, allowed_url_paths: Vec, + /// Class routes (`alias:`) of class-routed models, refused in + /// batch lines. + class_routes: HashSet, /// The toleration presets of the account that owns this upload, resolved /// per line by its model. A pinned line's list is written verbatim to /// `nvext.routing_constraints.tolerations` on its template body. @@ -855,6 +868,7 @@ fn create_file_stream( accessible_models, allowed_url_paths, tolerations, + class_routes, } = req_ctx; let normalizer = config.normalizer.clone(); let normalizer_mode = config.normalizer_mode; @@ -1065,6 +1079,7 @@ fn create_file_stream( &accessible_models, &allowed_url_paths, &tolerations, + &class_routes, ) { Ok(mut template) => { // Normalise image URLs in the per-template body @@ -1168,6 +1183,7 @@ fn create_file_stream( &accessible_models, &allowed_url_paths, &tolerations, + &class_routes, ) { Ok(mut template) => { // Normalise image URLs in the trailing line as well. @@ -1459,6 +1475,18 @@ pub async fn upload_file( accessible_models, allowed_url_paths: config.batches.allowed_url_paths.clone(), tolerations, + class_routes: state + .onwards_targets + .as_ref() + .map(|targets| { + targets + .targets + .iter() + .filter(|entry| entry.value().default_pool().class_route_of().is_some()) + .map(|entry| entry.key().clone()) + .collect() + }) + .unwrap_or_default(), }, Some(api_key_id), ); diff --git a/dwctl/src/continuation/mod.rs b/dwctl/src/continuation/mod.rs index 17883a103..e5f5deabe 100644 --- a/dwctl/src/continuation/mod.rs +++ b/dwctl/src/continuation/mod.rs @@ -281,6 +281,8 @@ impl ContinuationRoutes { WHERE dmc.pool = 'completions' AND dmc.enabled = true AND cm.deleted = false + -- A class-routed model's targets have no completions pool. + AND NOT class_routes_active(cm.id) AND dm.deleted = false ORDER BY cm.alias, @@ -558,6 +560,42 @@ mod tests { assert!(routes.is_empty()); } + #[dwctl_test_macros::test] + async fn class_routed_models_are_not_resumable(pool: PgPool) { + let user = create_test_user(&pool, Role::PlatformManager).await; + let endpoint = create_test_endpoint(&pool, "ep", user.id).await; + let resumable = create_composite(&pool, "alias-resumable", user.id).await; + let dynamo = create_test_model(&pool, "m-dynamo", "dynamo", endpoint, user.id).await; + add_component(&pool, resumable, dynamo, "default").await; + add_component_at(&pool, resumable, dynamo, "completions", 0).await; + + let routes = ContinuationRoutes::new(); + routes.refresh(&pool).await.unwrap(); + assert!(routes.is_enabled("alias-resumable")); + + sqlx::query!("UPDATE deployed_models SET routing_mode = 'class_routes' WHERE id = $1", resumable) + .execute(&pool) + .await + .unwrap(); + routes.refresh(&pool).await.unwrap(); + assert!( + routes.is_enabled("alias-resumable"), + "an activation without a standard class serves the legacy route, continuation included" + ); + + sqlx::query!( + "INSERT INTO model_serving_classes (deployed_model_id, class_key, display_name, inference_endpoint_id, upstream_model_name) + VALUES ($1, 'standard', 'Standard', $2, 'upstream')", + resumable, + endpoint + ) + .execute(&pool) + .await + .unwrap(); + routes.refresh(&pool).await.unwrap(); + assert!(!routes.is_enabled("alias-resumable"), "class routes have no completions pool"); + } + #[dwctl_test_macros::test] async fn per_route_config_reaches_the_cache(pool: PgPool) { let user = create_test_user(&pool, Role::PlatformManager).await; diff --git a/dwctl/src/db/handlers/deployments.rs b/dwctl/src/db/handlers/deployments.rs index 368e0753b..f9fa52f40 100644 --- a/dwctl/src/db/handlers/deployments.rs +++ b/dwctl/src/db/handlers/deployments.rs @@ -1542,6 +1542,7 @@ impl<'c> Deployments<'c> { requested.hosted_on, requested.reasoning_translation_overrides WHERE requested.is_composite = FALSE + AND NOT class_routes_active(requested.id) UNION ALL @@ -1551,13 +1552,40 @@ impl<'c> Deployments<'c> { FROM deployed_model_components link INNER JOIN deployed_models component ON component.id = link.deployed_model_id WHERE requested.is_composite = TRUE + AND NOT class_routes_active(requested.id) AND link.composite_model_id = requested.id AND component.deleted = FALSE + + UNION ALL + + -- A class-routed model's name is served by its standard class. + SELECT + class.inference_endpoint_id, + requested.reasoning_translation_overrides + FROM model_serving_classes class + WHERE class_routes_active(requested.id) + AND class.deployed_model_id = requested.id + AND class.class_key = 'standard' ) provider ON TRUE INNER JOIN inference_endpoints endpoint ON endpoint.id = provider.hosted_on WHERE requested.alias = ANY($1) AND requested.deleted = FALSE - ORDER BY requested.alias + + UNION ALL + + -- A class route (`alias:`) is served by its class's endpoint. + SELECT + model.alias || ':' || class.class_key AS routing_alias, + endpoint.reasoning_translation AS endpoint_reasoning_translation, + model.reasoning_translation_overrides AS model_reasoning_translation_overrides + FROM model_serving_classes class + INNER JOIN deployed_models model ON model.id = class.deployed_model_id + INNER JOIN inference_endpoints endpoint ON endpoint.id = class.inference_endpoint_id + WHERE class.class_key <> 'standard' + AND model.alias || ':' || class.class_key = ANY($1) + AND model.deleted = FALSE + AND class_routes_active(model.id) + ORDER BY routing_alias "#, ) .bind(aliases) @@ -4839,6 +4867,66 @@ mod tests { ); } + #[dwctl_test_macros::test] + async fn a_class_routed_model_takes_reasoning_efforts_from_its_class_endpoints(pool: PgPool) { + let user = create_test_user(&pool).await; + let (composite_id, _) = create_composite_and_components(&pool, user.id, 1).await; + let alias: String = sqlx::query_scalar("SELECT alias FROM deployed_models WHERE id = $1") + .bind(composite_id) + .fetch_one(&pool) + .await + .unwrap(); + let efforts = |supported: &[&str]| { + let all = ["none", "minimal", "low", "medium", "high", "xhigh", "max"]; + serde_json::json!({ + "chat_completions": { + "unsupported_efforts": all.iter().filter(|e| !supported.contains(e)).collect::>(), + "writes": [{"target_path": "/reasoning_effort", + "values": supported.iter().map(|e| (e.to_string(), serde_json::json!(e))).collect::>()}] + } + }) + }; + for (class, supported) in [("standard", vec!["low", "medium", "high"]), ("fast", vec!["high"])] { + let endpoint = crate::test::utils::create_test_endpoint(&pool, &format!("class-{class}"), user.id).await; + sqlx::query("UPDATE inference_endpoints SET reasoning_translation = $1 WHERE id = $2") + .bind(efforts(&supported)) + .bind(endpoint) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO model_serving_classes (deployed_model_id, class_key, display_name, inference_endpoint_id, upstream_model_name) + VALUES ($1, $2, $2, $3, 'upstream')", + ) + .bind(composite_id) + .bind(class) + .bind(endpoint) + .execute(&pool) + .await + .unwrap(); + } + sqlx::query("UPDATE deployed_models SET routing_mode = 'class_routes' WHERE id = $1") + .bind(composite_id) + .execute(&pool) + .await + .unwrap(); + + let fast = format!("{alias}:fast"); + let mut conn = pool.acquire().await.unwrap(); + let policies = Deployments::new(&mut conn) + .get_reasoning_policies(&[alias.clone(), fast.clone()]) + .await + .unwrap(); + assert_eq!( + policies[&alias].supported_efforts().unwrap().chat_completions, + Some(vec![ReasoningEffort::Low, ReasoningEffort::Medium, ReasoningEffort::High]) + ); + assert_eq!( + policies[&fast].supported_efforts().unwrap().chat_completions, + Some(vec![ReasoningEffort::High]) + ); + } + /// (deployed_model_id, sort_order) pairs in priority order. async fn ordered_components(pool: &PgPool, composite_id: DeploymentId) -> Vec<(DeploymentId, i32)> { let mut conn = pool.acquire().await.unwrap(); diff --git a/dwctl/src/db/handlers/model_provisioning/classes.rs b/dwctl/src/db/handlers/model_provisioning/classes.rs index 893ddb316..b788a3de8 100644 --- a/dwctl/src/db/handlers/model_provisioning/classes.rs +++ b/dwctl/src/db/handlers/model_provisioning/classes.rs @@ -61,14 +61,6 @@ impl ModelProvisioning<'_> { .fetch_all(&mut *self.db) .await?; let managed: HashSet<&str> = catalog.models.iter().map(|m| m.clay.alias.as_str()).collect(); - // Activation is intentionally unsupported until every runtime reader/writer - // understands class routing. In particular, never overwrite an active route. - for (_, alias, mode) in &models { - ensure!( - !managed.contains(alias.as_str()) || mode == "legacy", - "model {alias:?} uses class_routes; this catalog writer cannot edit activated models yet" - ); - } let managed_ids: HashSet = models .iter() .filter(|(_, a, _)| managed.contains(a.as_str())) @@ -108,38 +100,6 @@ impl ModelProvisioning<'_> { ); } } - let stored: Vec<(String, Uuid, String)> = sqlx::query_as( - "SELECT a.alias,a.deployed_model_id,c.class_key FROM model_aliases a JOIN model_serving_classes c ON c.id=a.serving_class_id", - ) - .fetch_all(&mut *self.db) - .await?; - let mut synonyms = HashSet::new(); - for (alias, model, _) in &stored { - if !managed_ids.contains(model) { - ensure!( - !primary.contains(&alias.to_lowercase()), - "primary name conflicts with existing synonym {alias:?}" - ); - synonyms.insert(alias.to_lowercase()); - } - } - for m in &catalog.models { - let id = models.iter().find(|(_, a, _)| a == &m.clay.alias).map(|(id, _, _)| *id); - for (key, class) in &m.clay.class_routes { - for alias in &class.aliases { - ensure!( - !primary.contains(&alias.to_lowercase()) && synonyms.insert(alias.to_lowercase()), - "synonym {alias:?} conflicts with an existing model/name" - ); - if let Some((_, owner, old_class)) = stored.iter().find(|(a, _, _)| a.eq_ignore_ascii_case(alias)) { - ensure!( - Some(*owner) == id && old_class == key, - "synonym {alias:?} cannot be rebound to another model/class; retire it explicitly first" - ); - } - } - } - } Ok(()) } @@ -150,15 +110,10 @@ impl ModelProvisioning<'_> { endpoints: &HashMap, at: DateTime, ) -> Result<()> { - let aliases: Vec<&str> = model - .clay - .class_routes - .values() - .flat_map(|c| c.aliases.iter().map(String::as_str)) - .collect(); - sqlx::query("DELETE FROM model_aliases WHERE deployed_model_id=$1 AND NOT (alias=ANY($2))") + // Class synonyms are retired: a class is reached only as `alias` or + // `alias:`. + sqlx::query("DELETE FROM model_aliases WHERE deployed_model_id=$1") .bind(model_id) - .bind(&aliases) .execute(&mut *self.db) .await?; let old: Vec = sqlx::query_scalar("SELECT class_key FROM model_serving_classes WHERE deployed_model_id=$1") @@ -176,28 +131,15 @@ impl ModelProvisioning<'_> { } for (key, class) in &model.clay.class_routes { // The explicit no-op condition avoids notification/timestamp churn on restart. - sqlx::query("INSERT INTO model_serving_classes (deployed_model_id,class_key,display_name,inference_endpoint_id,upstream_model_name) - VALUES ($1,$2,$3,$4,$5) ON CONFLICT (deployed_model_id,class_key) DO UPDATE SET - display_name=EXCLUDED.display_name,inference_endpoint_id=EXCLUDED.inference_endpoint_id,upstream_model_name=EXCLUDED.upstream_model_name - WHERE ROW(model_serving_classes.display_name,model_serving_classes.inference_endpoint_id,model_serving_classes.upstream_model_name) - IS DISTINCT FROM ROW(EXCLUDED.display_name,EXCLUDED.inference_endpoint_id,EXCLUDED.upstream_model_name)") + sqlx::query("INSERT INTO model_serving_classes (deployed_model_id,class_key,display_name,inference_endpoint_id,upstream_model_name,realtime_inflight_limit) + VALUES ($1,$2,$3,$4,$5,$6) ON CONFLICT (deployed_model_id,class_key) DO UPDATE SET + display_name=EXCLUDED.display_name,inference_endpoint_id=EXCLUDED.inference_endpoint_id,upstream_model_name=EXCLUDED.upstream_model_name, + realtime_inflight_limit=EXCLUDED.realtime_inflight_limit + WHERE ROW(model_serving_classes.display_name,model_serving_classes.inference_endpoint_id,model_serving_classes.upstream_model_name,model_serving_classes.realtime_inflight_limit) + IS DISTINCT FROM ROW(EXCLUDED.display_name,EXCLUDED.inference_endpoint_id,EXCLUDED.upstream_model_name,EXCLUDED.realtime_inflight_limit)") .bind(model_id).bind(key).bind(&class.display_name).bind(endpoints[&class.endpoint]).bind(&class.upstream_model_name) + .bind(class.realtime_inflight_limit) .execute(&mut *self.db).await?; - let id: Uuid = sqlx::query_scalar("SELECT id FROM model_serving_classes WHERE deployed_model_id=$1 AND class_key=$2") - .bind(model_id) - .bind(key) - .fetch_one(&mut *self.db) - .await?; - for alias in &class.aliases { - sqlx::query( - "INSERT INTO model_aliases (alias,deployed_model_id,serving_class_id) VALUES ($1,$2,$3) ON CONFLICT (alias) DO NOTHING", - ) - .bind(alias) - .bind(model_id) - .bind(id) - .execute(&mut *self.db) - .await?; - } self.reconcile_tariffs(model_id, None, Some(key), &class.tariffs, at).await?; self.reconcile_cache_tariff( model_id, @@ -208,6 +150,12 @@ impl ModelProvisioning<'_> { ) .await?; } + // Activation follows the classes it routes to, in the same transaction. + sqlx::query("UPDATE deployed_models SET routing_mode = $2 WHERE id = $1 AND routing_mode IS DISTINCT FROM $2") + .bind(model_id) + .bind(model.clay.routing_mode.as_str()) + .execute(&mut *self.db) + .await?; Ok(()) } } diff --git a/dwctl/src/inference/middleware.rs b/dwctl/src/inference/middleware.rs index 7c66483e2..1f031aba9 100644 --- a/dwctl/src/inference/middleware.rs +++ b/dwctl/src/inference/middleware.rs @@ -182,6 +182,23 @@ pub async fn inference_middleware` on a class-routed model) + // serves realtime only: asynchronous work is scheduled on the standard + // class by its deadline. Checked on the requested tier, so an endpoint that + // serves flex as realtime still refuses it. + if matches!(requested_tier, ServiceTier::Flex | ServiceTier::Background) + && let Some(model) = request_value["model"].as_str() + && let Some(standard) = parts + .extensions + .get::() + .and_then(|recorder| recorder.class_route_of(model)) + { + let model = model.to_owned(); + let message = + format!("Model '{model}' is available for realtime requests only; send flex and background requests to '{standard}'."); + return invalid_request(&parts, Some(&model), &message, "serving_class_realtime_only", "model"); + } + let scrubbed_suffix = match strip_async_model_suffix(&mut request_value, is_daemon_processed) { Ok(changed) => changed, Err(err) => { diff --git a/dwctl/src/inference/rejections.rs b/dwctl/src/inference/rejections.rs index 19133c45f..476868905 100644 --- a/dwctl/src/inference/rejections.rs +++ b/dwctl/src/inference/rejections.rs @@ -45,6 +45,12 @@ impl Recorder { pub fn configured_alias<'m>(&self, model: &'m str) -> Option<&'m str> { self.targets.configured_alias(model) } + + /// The model `model` is a class route of (`alias:`, which serves + /// realtime requests only), as marked by the config writer. + pub fn class_route_of(&self, model: &str) -> Option { + self.targets.class_route_of(model) + } } /// The code a dwctl layer refused a response's request with, and the span it @@ -379,3 +385,32 @@ mod tests { assert_eq!(model_of(b"{not json"), None); } } + +#[cfg(test)] +mod class_route_tests { + use onwards::target::Targets; + + use super::Recorder; + + #[test] + fn only_targets_marked_by_the_config_writer_are_class_routes() { + let targets = Targets::from_config( + serde_json::from_value(serde_json::json!({ + "targets": { + "vendor:legacy": {"url": "https://upstream.example.com/v1/"}, + "model": {"providers": [{"url": "https://upstream.example.com/v1/"}]}, + "model:fast": {"providers": [{"url": "https://upstream.example.com/v1/"}], "class_route_of": "model"} + }, + "auth": {"global_keys": [], "key_definitions": {}} + })) + .unwrap(), + ) + .unwrap(); + let recorder = Recorder::new(targets); + assert_eq!(recorder.class_route_of("model:fast").as_deref(), Some("model")); + // A colon in a legacy alias does not make it a class route. + assert_eq!(recorder.class_route_of("vendor:legacy"), None); + assert_eq!(recorder.class_route_of("model"), None); + assert_eq!(recorder.class_route_of("unknown:fast"), None); + } +} diff --git a/dwctl/src/model_provisioning.rs b/dwctl/src/model_provisioning.rs index 90d13d9c9..f0117b333 100644 --- a/dwctl/src/model_provisioning.rs +++ b/dwctl/src/model_provisioning.rs @@ -49,6 +49,24 @@ pub struct ModelDocument { #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] +// Mirrors validation: activation needs class routes with `standard` and no +// redirect rule, and classes need `standard`. +#[schemars(extend("allOf" = [ + { + "if": {"required": ["routing_mode"], "properties": {"routing_mode": {"const": "class_routes"}}}, + "then": { + "required": ["class_routes"], + "properties": { + "class_routes": {"required": ["standard"]}, + "traffic_rules": {"items": {"not": {"required": ["action"], "properties": {"action": {"const": "redirect"}}}}} + } + } + }, + { + "if": {"required": ["class_routes"], "properties": {"class_routes": {"minProperties": 1}}}, + "then": {"properties": {"class_routes": {"required": ["standard"]}}} + } +]))] pub struct ClayModel { pub alias: String, /// Defaults to the document's top-level `model` value. @@ -76,18 +94,43 @@ pub struct ClayModel { pub access_groups: Vec, #[serde(default)] pub traffic_rules: Vec, + /// How the model routes: `legacy` (its composite routing) or + /// `class_routes` (each class in `class_routes` to its own upstream: + /// `alias` to `standard`, `alias:` to that class). + #[serde(default)] + pub routing_mode: RoutingMode, /// Retired. Accepted only when empty, so existing catalog files still /// load; remove it. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] #[schemars(extend("maxProperties" = 0))] pub serving_classes: BTreeMap, - /// Explicit class destinations and prices, staged independently of legacy routing. - /// Declaring these does not activate routes or advertise additional model names. + /// Class destinations and prices. Declaring them alone changes nothing; + /// `routing_mode: class_routes` activates them. `standard` is required. #[serde(default)] #[schemars(extend("propertyNames" = {"pattern": "^[a-z][a-z0-9_-]*$"}))] pub class_routes: BTreeMap, } +/// How a virtual model routes its requests. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RoutingMode { + /// The model's composite routing (`routing`). + #[default] + Legacy, + /// Each class in `class_routes` routes to its own upstream. + ClassRoutes, +} + +impl RoutingMode { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Legacy => "legacy", + Self::ClassRoutes => "class_routes", + } + } +} + /// Complete catalog declaration for a stable class of the existing public model. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -95,9 +138,16 @@ pub struct ClassRoute { pub display_name: String, pub endpoint: String, pub upstream_model_name: String, - /// Optional exact ingress spellings, never additional discovery entries. - #[serde(default)] + /// Retired. Accepted only when empty: a class is reached only by the + /// model's alias (`alias` or `alias:`). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[schemars(extend("maxItems" = 0))] pub aliases: Vec, + /// Concurrent realtime requests one account may have in flight on this + /// class. Omit for the model's `settings.realtime_inflight_limit`. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub realtime_inflight_limit: Option, pub tariffs: Vec, /// Omit to inherit the model's all-class multipliers; does not enable caching. #[serde(default)] @@ -528,10 +578,25 @@ impl Catalog { } for model in &self.models { + ensure!( + model.clay.routing_mode == RoutingMode::Legacy || model.clay.class_routes.contains_key("standard"), + "{}: routing_mode class_routes requires a standard class in class_routes", + model.source + ); + ensure!( + model.clay.routing_mode == RoutingMode::Legacy + || !model + .clay + .traffic_rules + .iter() + .any(|rule| matches!(rule, TrafficRule::Redirect { .. })), + "{}: a class-routed model cannot redirect traffic; use deny rules", + model.source + ); if !model.clay.class_routes.is_empty() { ensure!( - model.clay.class_routes.contains_key("standard") && model.clay.class_routes.contains_key("fast"), - "{}: class_routes requires standard and fast", + model.clay.class_routes.contains_key("standard"), + "{}: class_routes requires a standard class", model.source ); for (key, class) in &model.clay.class_routes { @@ -561,6 +626,16 @@ impl Catalog { "{}: class {key:?} requires a realtime tariff", model.source ); + ensure!( + class.aliases.is_empty(), + "{}: class {key:?}: class aliases are retired; a class is reached as the model's alias or alias:{key}", + model.source + ); + ensure!( + class.realtime_inflight_limit.is_none_or(|limit| limit > 0), + "{}: class {key:?}: realtime_inflight_limit must be positive", + model.source + ); if let Some(cache) = &class.cache_tariff { ensure!( model.clay.cache_tariff.is_some(), @@ -581,16 +656,6 @@ impl Catalog { } } for model in &self.models { - for class in model.clay.class_routes.values() { - for alias in &class.aliases { - ensure!( - !alias.is_empty() && !alias.chars().any(char::is_whitespace), - "{}: class alias must be nonempty without whitespace", - model.source - ); - validate_alias(&mut aliases, alias, &model.source, "class synonym")?; - } - } let mut component_keys = HashSet::new(); for (pool, components) in &model.clay.routing.pools { ensure!( diff --git a/dwctl/src/model_provisioning/class_routes_tests.rs b/dwctl/src/model_provisioning/class_routes_tests.rs index fcedee812..7120ba1af 100644 --- a/dwctl/src/model_provisioning/class_routes_tests.rs +++ b/dwctl/src/model_provisioning/class_routes_tests.rs @@ -43,7 +43,7 @@ clay: display_name: Fast endpoint: gateway upstream_model_name: dynamo-example/fast - aliases: [example/model-fast] + realtime_inflight_limit: 7 tariffs: - name: fast purpose: realtime @@ -66,24 +66,53 @@ async fn setup(pool: &PgPool) { } #[test] -fn class_catalog_rejects_missing_fast() { +fn class_catalog_requires_standard_and_nothing_else() { let mut v = fixture(); v["clay"]["class_routes"].as_mapping_mut().unwrap().remove("fast"); - assert!(catalog(&v).unwrap_err().to_string().contains("standard and fast")); + catalog(&v).unwrap(); + let mut v = fixture(); + v["clay"]["class_routes"].as_mapping_mut().unwrap().remove("standard"); + assert!(catalog(&v).unwrap_err().to_string().contains("requires a standard class")); } #[test] -fn class_catalog_rejects_synonym_shadowing_primary_fast_name() { +fn class_catalog_rejects_class_aliases_and_bad_limits() { + let mut v = fixture(); + v["clay"]["class_routes"]["fast"]["aliases"] = serde_yaml::to_value(["example/model-fast"]).unwrap(); + assert!(catalog(&v).unwrap_err().to_string().contains("class aliases are retired")); let mut v = fixture(); - v["clay"]["class_routes"]["fast"]["aliases"] = serde_yaml::to_value(["example/model:fast"]).unwrap(); - assert!(catalog(&v).is_err()); + v["clay"]["class_routes"]["fast"]["realtime_inflight_limit"] = serde_yaml::to_value(0).unwrap(); + assert!( + catalog(&v) + .unwrap_err() + .to_string() + .contains("realtime_inflight_limit must be positive") + ); } #[test] -fn class_catalog_rejects_activation_in_yaml() { +fn class_catalog_activation_needs_standard_and_no_redirects() { let mut v = fixture(); v["clay"]["routing_mode"] = serde_yaml::Value::String("class_routes".into()); - assert!(catalog(&v).is_err()); + catalog(&v).unwrap(); + + let mut without_classes = v.clone(); + without_classes["clay"].as_mapping_mut().unwrap().remove("class_routes"); + assert!( + catalog(&without_classes) + .unwrap_err() + .to_string() + .contains("requires a standard class") + ); + + let mut redirecting = v.clone(); + redirecting["clay"]["traffic_rules"] = + serde_yaml::from_str("- {action: redirect, purpose: batch, target: legacy-example/model}").unwrap(); + assert!(catalog(&redirecting).unwrap_err().to_string().contains("cannot redirect")); + + let mut denying = v; + denying["clay"]["traffic_rules"] = serde_yaml::from_str("- {action: deny, purpose: playground}").unwrap(); + catalog(&denying).unwrap(); } fn add_batch_price(value: &mut serde_yaml::Value, class: &str) { @@ -103,7 +132,6 @@ fn class_catalog_rejects_batch_prices_on_nonstandard_classes() { let mut value = fixture(); if key != "fast" { value["clay"]["class_routes"][key] = value["clay"]["class_routes"]["fast"].clone(); - value["clay"]["class_routes"][key]["aliases"] = serde_yaml::to_value(Vec::::new()).unwrap(); } add_batch_price(&mut value, key); let error = catalog(&value).unwrap_err().to_string(); @@ -137,11 +165,12 @@ async fn staging_classes_preserves_legacy_route_and_effective_price(pool: PgPool .await .unwrap(); assert_eq!(members, 1); - let names: Vec = sqlx::query_scalar("SELECT alias FROM model_aliases ORDER BY alias") - .fetch_all(&pool) - .await - .unwrap(); - assert_eq!(names, ["example/model-fast"]); + let limits: Vec<(String, Option)> = + sqlx::query_as("SELECT class_key, realtime_inflight_limit FROM model_serving_classes ORDER BY class_key") + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(limits, [("fast".into(), Some(7)), ("standard".into(), None)]); for class in [None, Some("standard"), Some("fast")] { let price: Decimal = sqlx::query_scalar("SELECT input_price_per_token FROM effective_model_tariff($1,NULL,'realtime',NULL,$2,NOW())") @@ -232,9 +261,9 @@ async fn omitted_class_configuration_retires_prices_without_erasing_history(pool } #[sqlx::test] -async fn synonym_collision_with_existing_private_model_rolls_back_catalog(pool: PgPool) { +async fn class_name_collision_with_existing_model_rolls_back_catalog(pool: PgPool) { setup(&pool).await; - sqlx::query("INSERT INTO deployed_models (alias,model_name,is_composite,created_by) VALUES ('example/model-fast','private',true,'00000000-0000-0000-0000-000000000000')").execute(&pool).await.unwrap(); + sqlx::query("INSERT INTO deployed_models (alias,model_name,is_composite,created_by) VALUES ('example/model:fast','private',true,'00000000-0000-0000-0000-000000000000')").execute(&pool).await.unwrap(); assert!(apply(&pool, &catalog(&fixture()).unwrap()).await.is_err()); let count: i64 = sqlx::query_scalar("SELECT count(*) FROM model_serving_classes") .fetch_one(&pool) @@ -279,31 +308,35 @@ fn class_catalog_schema_restricts_class_keys() { } #[sqlx::test] -async fn catalog_refuses_activated_models_without_resetting_operator_mode(pool: PgPool) { +async fn the_catalog_owns_activation_and_rolls_it_back(pool: PgPool) { setup(&pool).await; - let c = catalog(&fixture()).unwrap(); - apply(&pool, &c).await.unwrap(); - sqlx::query("UPDATE deployed_models SET routing_mode='class_routes' WHERE alias='example/model'") - .execute(&pool) - .await - .unwrap(); - let error = apply(&pool, &c).await.unwrap_err(); - assert!(error.to_string().contains("activated")); - let mode: String = sqlx::query_scalar("SELECT routing_mode FROM deployed_models WHERE alias='example/model'") - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(mode, "class_routes"); + let mode = || async { + sqlx::query_scalar::<_, String>("SELECT routing_mode FROM deployed_models WHERE alias='example/model'") + .fetch_one(&pool) + .await + .unwrap() + }; + let mut v = fixture(); + v["clay"]["routing_mode"] = serde_yaml::Value::String("class_routes".into()); + apply(&pool, &catalog(&v).unwrap()).await.unwrap(); + assert_eq!(mode().await, "class_routes"); + // Editing an activated model is ordinary catalog work now. + v["clay"]["class_routes"]["fast"]["upstream_model_name"] = serde_yaml::Value::String("moved".into()); + apply(&pool, &catalog(&v).unwrap()).await.unwrap(); + assert_eq!(mode().await, "class_routes"); + // Rolling back is removing the field. + apply(&pool, &catalog(&fixture()).unwrap()).await.unwrap(); + assert_eq!(mode().await, "legacy"); } #[sqlx::test] -async fn ordinary_model_creation_cannot_claim_a_class_synonym(pool: PgPool) { +async fn ordinary_model_creation_cannot_claim_a_class_name(pool: PgPool) { use crate::db::handlers::{Deployments, repository::Repository}; use crate::db::models::deployments::DeploymentCreateDBRequest; setup(&pool).await; apply(&pool, &catalog(&fixture()).unwrap()).await.unwrap(); let mut conn = pool.acquire().await.unwrap(); - for name in ["example/model-fast", "example/model:fast"] { + for name in ["example/model:fast", "EXAMPLE/model:FAST"] { let request = DeploymentCreateDBRequest::builder() .created_by(Uuid::nil()) .alias(name.into()) @@ -315,13 +348,15 @@ async fn ordinary_model_creation_cannot_claim_a_class_synonym(pool: PgPool) { } #[sqlx::test] -async fn renaming_a_model_cannot_shadow_an_existing_synonym_with_its_class_name(pool: PgPool) { +async fn renaming_a_model_cannot_shadow_an_existing_model_with_its_class_name(pool: PgPool) { use crate::db::handlers::{Deployments, repository::Repository}; use crate::db::models::deployments::DeploymentUpdateDBRequest; setup(&pool).await; - let mut v = fixture(); - v["clay"]["class_routes"]["fast"]["aliases"] = serde_yaml::to_value(["renamed/model:fast"]).unwrap(); - apply(&pool, &catalog(&v).unwrap()).await.unwrap(); + sqlx::query("INSERT INTO deployed_models (alias,model_name,is_composite,created_by) VALUES ('renamed/model:fast','other',true,'00000000-0000-0000-0000-000000000000')") + .execute(&pool) + .await + .unwrap(); + apply(&pool, &catalog(&fixture()).unwrap()).await.unwrap(); let id: Uuid = sqlx::query_scalar("SELECT id FROM deployed_models WHERE alias='example/model'") .fetch_one(&pool) .await @@ -352,7 +387,7 @@ async fn concurrent_model_writer_waits_for_catalog_and_then_rejects_collision(po let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()").fetch_one(&mut *conn).await.unwrap(); let request = DeploymentCreateDBRequest::builder() .created_by(Uuid::nil()) - .alias("example/model-fast".into()) + .alias("example/model:fast".into()) .model_name("competing".into()) .is_composite(true) .build(); diff --git a/dwctl/src/pricing/class_tests.rs b/dwctl/src/pricing/class_tests.rs index 3980ab948..79b507175 100644 --- a/dwctl/src/pricing/class_tests.rs +++ b/dwctl/src/pricing/class_tests.rs @@ -157,10 +157,16 @@ fn cache_class_deals_require_a_general_enablement_row_and_respect_history() { .read, Decimal::ZERO ); + // The account's class-less deal covers standard only; another class + // without its own deal falls to the general row. assert_eq!( resolve_cache_multipliers(&rows, now, Some(account), Some("throughput")) .unwrap() .read, + Decimal::ONE + ); + assert_eq!( + resolve_cache_multipliers(&rows, now, Some(account), Some("standard")).unwrap().read, Decimal::new(5, 1) ); assert_eq!( @@ -451,6 +457,8 @@ async fn playground_exhausts_each_customer_scope_before_general_prices(pool: PgP mask & (1 << index) != 0 && (scope.is_none() || *scope == owner) && (c.is_none() || *c == class) + // An account's class-less deal covers the standard class only. + && !(scope.is_some() && c.is_none() && class.is_some_and(|k| k != "standard")) && (purpose == ApiKeyPurpose::Playground || *p == ApiKeyPurpose::Realtime) }) .map(|(_, (_, _, _, price))| (Decimal::from(*price), Decimal::from(*price))); @@ -796,6 +804,9 @@ async fn public_class_prices_follow_account_deals_and_preserve_exact_batch_windo .await .unwrap(); let model:Uuid=sqlx::query_scalar("INSERT INTO deployed_models (model_name,alias,is_composite,created_by,routing_mode) VALUES ('public-class-test','public-class-test',true,$1,'class_routes') RETURNING id").bind(account).fetch_one(&pool).await.unwrap(); + // An active activation has a standard class. + sqlx::query("WITH ep AS (INSERT INTO inference_endpoints (name,url,created_by) VALUES ('public-class-ep','http://localhost:8080',$2) RETURNING id) INSERT INTO model_serving_classes (deployed_model_id,class_key,display_name,inference_endpoint_id,upstream_model_name) SELECT $1,'standard','Standard',id,'upstream' FROM ep") + .bind(model).bind(account).execute(&pool).await.unwrap(); let now = Utc::now(); let mut rows = Vec::new(); for (owner, class, purpose, window, price) in [ @@ -872,9 +883,15 @@ fn public_class_cache_multipliers_inherit_enablement_and_follow_account_deals() resolve_cache_multipliers(&rows, now, None, Some("fast")).unwrap().read, Decimal::from(2) ); + // A class-less account deal does not reach the fast class... rows.push(row(Some(account), None, 1)); assert_eq!( resolve_cache_multipliers(&rows, now, Some(account), Some("fast")).unwrap().read, + Decimal::from(2) + ); + // ...it covers standard. + assert_eq!( + resolve_cache_multipliers(&rows, now, Some(account), Some("standard")).unwrap().read, Decimal::ONE ); rows.push(row(Some(account), Some("fast"), 0)); diff --git a/dwctl/src/pricing/mod.rs b/dwctl/src/pricing/mod.rs index 270f8205b..d8c7d027e 100644 --- a/dwctl/src/pricing/mod.rs +++ b/dwctl/src/pricing/mod.rs @@ -180,7 +180,7 @@ where JOIN model_cache_tariffs mct ON mct.deployed_model_id = dm.id WHERE dm.alias = ANY($1) AND (mct.user_id IS NULL OR mct.user_id = ANY($2)) - AND (mct.user_id IS NOT NULL OR mct.serving_class IS NULL) + AND (mct.user_id IS NOT NULL OR mct.serving_class IS NULL OR class_routes_active(dm.id)) ORDER BY dm.alias, mct.valid_from DESC "#, aliases, @@ -230,12 +230,15 @@ pub(crate) fn resolve_cache_multipliers( .max_by_key(|r| r.valid_from) }; let general = valid(None, None)?; + // An account's class-less deal covers the standard class only: any other + // class is priced by the account's deal for that class, else the model's. + let standard = serving_class.is_none_or(|class| class == "standard"); Some( account .and_then(|a| { serving_class .and_then(|class| valid(Some(a), Some(class))) - .or_else(|| valid(Some(a), None)) + .or_else(|| standard.then(|| valid(Some(a), None)).flatten()) }) .or_else(|| serving_class.and_then(|class| valid(None, Some(class)))) .unwrap_or(general), @@ -292,9 +295,12 @@ pub(crate) fn find_best_tariff( if t.completion_window.as_deref() != if t.purpose == ApiKeyPurpose::Batch { window } else { None } { return None; } + // An account's class-less deal covers the standard class only: + // any other class is priced by the account's deal for that class, + // else the model's. let scope_rank = match (t.account, t.serving_class.as_deref()) { (Some(owner), Some(c)) if Some(owner) == account && Some(c) == class => 0, - (Some(owner), None) if Some(owner) == account => 1, + (Some(owner), None) if Some(owner) == account && class.is_none_or(|c| c == "standard") => 1, (None, Some(c)) if Some(c) == class => 2, (None, None) => 3, _ => return None, diff --git a/dwctl/src/prompt_cache/model_config.rs b/dwctl/src/prompt_cache/model_config.rs index c13dcae2f..35d8ec04c 100644 --- a/dwctl/src/prompt_cache/model_config.rs +++ b/dwctl/src/prompt_cache/model_config.rs @@ -74,7 +74,13 @@ impl ModelConfigResolver { AND mct.user_id IS NULL AND mct.serving_class IS NULL AND mct.valid_from <= now() AND (mct.valid_until IS NULL OR mct.valid_until > now()) - WHERE dm.alias = $1 AND dm.deleted = false + WHERE dm.deleted = false + AND (dm.alias = $1 + -- A class route (`alias:`) inherits its model's + -- enablement; its cache entries are scoped by that name. + OR (class_routes_active(dm.id) AND EXISTS ( + SELECT 1 FROM model_serving_classes c + WHERE c.deployed_model_id = dm.id AND dm.alias || ':' || c.class_key = $1))) "#, virtual_model, ) diff --git a/dwctl/src/request_logging/batcher.rs b/dwctl/src/request_logging/batcher.rs index 73f146bb5..467c6ddf5 100644 --- a/dwctl/src/request_logging/batcher.rs +++ b/dwctl/src/request_logging/batcher.rs @@ -541,6 +541,23 @@ where .into_iter() .collect(); + // A request to a class route names `alias:`; it is recorded and + // priced as the model's alias in that class. + let class_routes = self.batch_lookup_class_routes(tx, buffer).await?; + let buffer: Vec = buffer + .iter() + .cloned() + .map(|mut raw| { + if let Some((alias, class)) = raw.request_model.as_deref().and_then(|model| class_routes.get(model)) { + raw.request_model = Some(alias.clone()); + raw.requested_serving_class = Some(class.clone()); + raw.resolved_serving_class = Some(class.clone()); + } + raw + }) + .collect(); + let buffer = buffer.as_slice(); + // Collect unique model aliases let models: Vec<&str> = buffer .iter() @@ -786,7 +803,7 @@ where LEFT JOIN inference_endpoints ie ON dm.hosted_on = ie.id LEFT JOIN model_tariffs mt ON mt.deployed_model_id = dm.id AND (mt.user_id IS NULL OR mt.user_id = ANY($2)) - AND (mt.user_id IS NOT NULL OR mt.serving_class IS NULL) + AND (mt.user_id IS NOT NULL OR mt.serving_class IS NULL OR class_routes_active(dm.id)) WHERE dm.alias = ANY($1) ORDER BY dm.alias, mt.valid_from DESC "#, @@ -830,6 +847,48 @@ where Ok(map) } + /// The class routes (`alias:` of a model with that class) among the + /// buffer's request models, mapped to (alias, class). + async fn batch_lookup_class_routes( + &self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + buffer: &[RawAnalyticsRecord], + ) -> Result, sqlx::Error> { + let names: Vec = buffer + .iter() + .filter_map(|r| r.request_model.clone()) + .filter(|model| model.contains(':')) + .collect::>() + .into_iter() + .collect(); + if names.is_empty() { + return Ok(HashMap::new()); + } + let rows = sqlx::query!( + r#" + SELECT dm.alias, c.class_key + FROM model_serving_classes c + JOIN deployed_models dm ON dm.id = c.deployed_model_id + -- Regardless of the model's current mode: a record queued before a + -- rollback is still recorded under its model and class (and priced + -- at the model's price, as class prices apply only while class + -- routes serve). A model of that exact name wins. + WHERE c.class_key <> 'standard' + AND dm.alias || ':' || c.class_key = ANY($1) + AND NOT EXISTS ( + SELECT 1 FROM deployed_models named + WHERE named.alias = dm.alias || ':' || c.class_key AND named.deleted = false) + "#, + &names + ) + .fetch_all(&mut **tx) + .await?; + Ok(rows + .into_iter() + .map(|row| (format!("{}:{}", row.alias, row.class_key), (row.alias, row.class_key))) + .collect()) + } + /// Batch lookup cache tariffs (per model, per tier) for the given aliases. /// /// Fetches general and billed-account rows (including expired) so requests price as @@ -2260,6 +2319,38 @@ mod integration_tests { while batcher.project_outbox_batch().await.unwrap() > 0 {} } + #[dwctl_test_macros::test] + async fn class_route_records_resolve_after_a_rollback(pool: sqlx::PgPool) { + // Records queued while a model was class-routed still name `alias:` + // when they are enriched after the model went back to legacy. A model + // whose own alias is `alias:` keeps its records. + let routed = create_test_model(&pool, "routed").await; + let named = create_test_model(&pool, "named").await; + create_test_model(&pool, "named:fast").await; + for model in [routed, named] { + sqlx::query( + "INSERT INTO model_serving_classes (deployed_model_id, class_key, display_name, inference_endpoint_id, upstream_model_name) + SELECT id, 'fast', 'Fast', hosted_on, 'upstream-fast' FROM deployed_models WHERE id = $1", + ) + .bind(model) + .execute(&pool) + .await + .unwrap(); + } + let config = crate::test::utils::create_test_config(); + let (batcher, _writer) = AnalyticsBatcher::::new(pool.clone(), config, None); + let buffer = vec![ + create_raw_record("routed:fast", None, 1, 1), + create_raw_record("named:fast", None, 1, 1), + ]; + let mut tx = pool.begin().await.unwrap(); + let routes = batcher.batch_lookup_class_routes(&mut tx, &buffer).await.unwrap(); + assert_eq!( + routes, + HashMap::from([("routed:fast".to_owned(), ("routed".to_owned(), "fast".to_owned()))]) + ); + } + #[dwctl_test_macros::test] async fn tariff_lookups_scope_accounts_without_losing_history_or_unpriced_models(pool: sqlx::PgPool) { let model = create_test_model(&pool, "scoped-history").await; @@ -3184,7 +3275,9 @@ mod integration_tests { } let key = create_api_key_for_user(&pool, account, ApiKeyPurpose::Playground).await; let mut record = create_raw_record(alias, Some(key), 1000, 500); - record.resolved_serving_class = Some("interactive".to_string()); + // An account's class-less deal covers the standard class. + let served_class = class.unwrap_or("standard"); + record.resolved_serving_class = Some(served_class.to_string()); run_batcher_with_records(&pool, vec![record]).await; let expected = input * Decimal::from(1000) + output * Decimal::from(500); let mut conn = pool.acquire().await.unwrap(); @@ -3201,7 +3294,7 @@ mod integration_tests { assert_eq!(cost, Some(expected), "zero is a price, not unresolved: {alias}"); let mut repo = Tariffs::new(&mut conn); assert_eq!( - repo.get_effective_pricing_at_timestamp(model, Some(account), "playground", None, Some("interactive"), Utc::now()) + repo.get_effective_pricing_at_timestamp(model, Some(account), "playground", None, Some(served_class), Utc::now()) .await .unwrap(), Some((input, output)) diff --git a/dwctl/src/sync/key_policy.rs b/dwctl/src/sync/key_policy.rs index 6ec572742..dd563d726 100644 --- a/dwctl/src/sync/key_policy.rs +++ b/dwctl/src/sync/key_policy.rs @@ -47,19 +47,25 @@ pub struct KeyPolicy { /// allocating a fresh `String`. pub toleration_preset: Option>, /// Per-model preset ids from the account catalog (`account_model_policies`), - /// keyed by model alias, each overriding `toleration_preset` for requests to - /// that model. A class policy has already been folded over its model's. - /// Shared by every key of the account. + /// keyed by the model's gateway target (its alias for the standard class, + /// `alias:` for any other), each overriding `toleration_preset` for + /// requests to it. A model-wide policy is folded under the alias and also + /// applies to the model's other classes. Shared by every key of the account. pub model_toleration_presets: Option>>>, } impl KeyPolicy { - /// The preset id that applies to a request for `model`: the model's (or - /// its class's) from the account catalog, else the account's own. + /// The preset id that applies to a request for `model` (a model alias, + /// or `alias:`): that class's from the account catalog, else the + /// model's, else the account's own. pub fn toleration_preset_for(&self, model: &str) -> Option<&str> { - self.model_toleration_presets - .as_ref() + let presets = self.model_toleration_presets.as_ref(); + presets .and_then(|presets| presets.get(model)) + .or_else(|| { + let (alias, _class) = model.rsplit_once(':')?; + presets.and_then(|presets| presets.get(&model_wide_key(alias))) + }) .or(self.toleration_preset.as_ref()) .map(|preset| &**preset) } @@ -246,6 +252,13 @@ async fn load(pool: &PgPool) -> Result, sqlx::Error> .collect()) } +/// The map key of a model-wide preset, kept apart from the standard target's +/// so a standard-class preset never leaks to the model's other classes. A +/// model alias cannot contain a NUL. +fn model_wide_key(alias: &str) -> String { + format!("{alias}\0") +} + /// Per-model preset ids from the account catalog, for every account or for /// one, each account's map shared by `Arc` across its keys. A class row /// (`standard`, the only class a request resolves to today) overrides its @@ -256,12 +269,11 @@ async fn load_model_presets<'c>( ) -> Result>>>, sqlx::Error> { let rows = sqlx::query!( r#" - SELECT p.user_id, dm.alias, p.toleration_preset AS "toleration_preset!" + SELECT p.user_id, dm.alias, p.serving_class, p.toleration_preset AS "toleration_preset!" FROM account_model_policies p JOIN deployed_models dm ON dm.id = p.deployed_model_id WHERE p.toleration_preset IS NOT NULL AND dm.deleted = FALSE - AND (p.serving_class IS NULL OR p.serving_class = 'standard') AND ($1::uuid IS NULL OR p.user_id = $1) ORDER BY p.serving_class NULLS FIRST "#, @@ -272,10 +284,21 @@ async fn load_model_presets<'c>( let mut presets: HashMap>> = HashMap::new(); for row in rows { - presets - .entry(row.user_id) - .or_default() - .insert(row.alias, Arc::from(row.toleration_preset)); + let preset: Arc = Arc::from(row.toleration_preset); + let account = presets.entry(row.user_id).or_default(); + match row.serving_class.as_deref() { + // Model-wide: the standard target, and the fallback for other classes. + None => { + account.insert(model_wide_key(&row.alias), preset.clone()); + account.insert(row.alias, preset); + } + Some("standard") => { + account.insert(row.alias, preset); + } + Some(class) => { + account.insert(format!("{}:{class}", row.alias), preset); + } + } } Ok(presets.into_iter().map(|(account, presets)| (account, Arc::new(presets))).collect()) } @@ -613,4 +636,29 @@ mod tests { Some(&serde_json::json!([])) ); } + + #[test] + fn class_presets_fall_back_to_the_model_wide_preset_not_the_standard_one() { + let policy = |entries: &[(&str, &str)]| KeyPolicy { + toleration_preset: Some("account".into()), + model_toleration_presets: Some(Arc::new( + entries.iter().map(|(k, v)| (k.to_string(), Arc::::from(*v))).collect(), + )), + ..Default::default() + }; + let wide = model_wide_key("m"); + let p = policy(&[(wide.as_str(), "wide"), ("m", "wide"), ("m:fast", "fast-only")]); + assert_eq!(p.toleration_preset_for("m"), Some("wide")); + assert_eq!(p.toleration_preset_for("m:fast"), Some("fast-only")); + assert_eq!( + p.toleration_preset_for("m:other"), + Some("wide"), + "a model-wide preset covers every class" + ); + assert_eq!(p.toleration_preset_for("n:fast"), Some("account")); + // A standard-class preset stays on standard. + let p = policy(&[("m", "standard-only")]); + assert_eq!(p.toleration_preset_for("m"), Some("standard-only")); + assert_eq!(p.toleration_preset_for("m:fast"), Some("account")); + } } diff --git a/dwctl/src/sync/onwards_config/mod.rs b/dwctl/src/sync/onwards_config/mod.rs index 9c6ec3566..3d2a57872 100644 --- a/dwctl/src/sync/onwards_config/mod.rs +++ b/dwctl/src/sync/onwards_config/mod.rs @@ -165,8 +165,42 @@ fn parse_stored_targets(value: Option) -> Option Result { +/// Per-account overlays: model-level (organisation overlays and account +/// policies without a class) and per gateway target (the model's `alias` for +/// the standard class, `alias:` for any other), each a model-level +/// overlay with that class's own policy folded over it. +#[derive(Debug, Default)] +struct Overlays { + model_level: OverlaysByAlias, + by_target: OverlaysByAlias, +} + +impl Overlays { + /// The overlays a class target uses: its own, else the model's. + fn for_class(&self, alias: &str, class: &str) -> HashMap { + let key = class_target_key(alias, class); + self.by_target + .get(&key) + .or_else(|| self.model_level.get(alias)) + .cloned() + .unwrap_or_default() + } +} + +/// The gateway target a class is reached on: the model's alias for standard, +/// `alias:` otherwise. +fn class_target_key(alias: &str, class: &str) -> String { + if class == STANDARD_CLASS { + alias.to_owned() + } else { + format!("{alias}:{class}") + } +} + +const STANDARD_CLASS: &str = "standard"; + +/// Loads every alias's overlays, model-level and per class target. +async fn load_overlays_from_db(db: &PgPool) -> Result { let rows = sqlx::query!( r#" SELECT mo.user_id, dm.alias, mo.targets, mo.self_hosted_only @@ -180,10 +214,10 @@ async fn load_overlays_from_db(db: &PgPool) -> Result Result Result { + let overlay = model_level + .entry(row.alias) + .or_default() + .entry(row.user_id.to_string()) + .or_default(); + if row.priority.is_some() { + overlay.priority = row.priority; + } + } + Some(class) => class_rows.push((row.alias, class, row.user_id.to_string(), row.priority, row.ttft_ms, row.itl_ms)), + } + } + + // Every model's standard target starts as its model-level overlays. + let mut by_target = model_level.clone(); + for (alias, class, account, priority, ttft_ms, itl_ms) in class_rows { + let base = model_level.get(&alias).and_then(|overlays| overlays.get(&account)).cloned(); + let overlay = by_target + .entry(class_target_key(&alias, &class)) + .or_insert_with(|| model_level.get(&alias).cloned().unwrap_or_default()) + .entry(account) + .or_insert_with(|| base.unwrap_or_default()); + if priority.is_some() { + overlay.priority = priority; } // Only set values override, so a priority-only policy keeps any // targets already on the overlay. - if row.serving_class.is_some() { - if let Some(ttft) = row.ttft_ms.and_then(|v| u32::try_from(v).ok()) { - overlay.ttft_ms = Some(ttft); - } - if let Some(itl) = row.itl_ms.and_then(|v| u32::try_from(v).ok()) { - overlay.itl_ms = Some(itl); - } + if let Some(ttft) = ttft_ms.and_then(|v| u32::try_from(v).ok()) { + overlay.ttft_ms = Some(ttft); + } + if let Some(itl) = itl_ms.and_then(|v| u32::try_from(v).ok()) { + overlay.itl_ms = Some(itl); } } - Ok(overlays) + Ok(Overlays { model_level, by_target }) } async fn load_inflight_overrides_from_db(db: &PgPool) -> Result { @@ -249,6 +303,24 @@ async fn load_inflight_overrides_from_db(db: &PgPool) -> Result`). + let class_rows = sqlx::query!( + r#" + SELECT dm.alias, o.class_key, o.user_id, o.inflight_limit + FROM realtime_inflight_class_overrides o + INNER JOIN deployed_models dm ON dm.id = o.deployed_model_id + WHERE dm.deleted = FALSE + "# + ) + .fetch_all(db) + .await?; + for row in class_rows { + overrides + .entry(class_target_key(&row.alias, &row.class_key)) + .or_default() + .insert(row.user_id.to_string(), row.inflight_limit as u32); + } Ok(overrides) } @@ -773,6 +845,20 @@ struct OnwardsCompositeModel { components: Vec, // API keys that have access to this composite model api_keys: Vec, + /// `deployed_models.routing_mode`: `class_routes` serves `classes` + /// instead of `components`. + routing_mode: String, + /// The model's serving classes (loaded only for class-routed models). + classes: Vec, +} + +/// One serving class of a class-routed model: its own upstream and limit. +#[derive(Debug, Clone)] +struct OnwardsClassRoute { + class_key: String, + realtime_inflight_limit: Option, + /// The class's upstream, as a single-provider target. + target: OnwardsTarget, } /// Loads composite models with their components and API keys from the database @@ -908,7 +994,8 @@ async fn load_composite_models_from_db(db: &PgPool, escalation_models: &[String] -- positive deal can require credit on an otherwise free model. NOT EXISTS ( SELECT 1 FROM model_tariffs mt - WHERE mt.deployed_model_id = cm.id AND mt.user_id IS NULL AND mt.serving_class IS NULL + WHERE mt.deployed_model_id = cm.id AND mt.user_id IS NULL + AND (mt.serving_class IS NULL OR class_routes_active(cm.id)) AND mt.valid_until IS NULL AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0) ) AND NOT EXISTS ( @@ -944,7 +1031,8 @@ async fn load_composite_models_from_db(db: &PgPool, escalation_models: &[String] AND ck.window_spend >= root.spend_limit AND (EXISTS ( SELECT 1 FROM model_tariffs mt - WHERE mt.deployed_model_id = cm.id AND mt.user_id IS NULL AND mt.serving_class IS NULL + WHERE mt.deployed_model_id = cm.id AND mt.user_id IS NULL + AND (mt.serving_class IS NULL OR class_routes_active(cm.id)) AND mt.valid_until IS NULL AND (mt.input_price_per_token > 0 OR mt.output_price_per_token > 0) ) OR EXISTS ( @@ -1001,7 +1089,8 @@ async fn load_composite_models_from_db(db: &PgPool, escalation_models: &[String] aimd, affinity, sanitize_responses, - trusted + trusted, + routing_mode FROM deployed_models WHERE is_composite = TRUE AND deleted = FALSE @@ -1047,10 +1136,92 @@ async fn load_composite_models_from_db(db: &PgPool, escalation_models: &[String] routing_rules: Vec::new(), // Populated from separate query below components: Vec::new(), api_keys: Vec::new(), + routing_mode: row.routing_mode, + classes: Vec::new(), }, ); } + // Serving classes of class-routed models, each with its own upstream. + let class_rows = sqlx::query!( + r#" + SELECT + c.deployed_model_id, + c.class_key, + c.upstream_model_name, + c.realtime_inflight_limit, + dm.alias, + dm.reasoning_translation_overrides as model_reasoning_translation_overrides, + ie.reasoning_translation as endpoint_reasoning_translation, + ie.url as "endpoint_url!", + ie.api_key as endpoint_api_key, + ie.auth_header_name, + ie.auth_header_prefix, + ie.accepts_scheduling_priority as endpoint_accepts_scheduling_priority, + ie.kind as endpoint_kind + FROM model_serving_classes c + INNER JOIN deployed_models dm ON dm.id = c.deployed_model_id + INNER JOIN inference_endpoints ie ON ie.id = c.inference_endpoint_id + WHERE dm.routing_mode = 'class_routes' + AND dm.deleted = FALSE + ORDER BY c.deployed_model_id, c.class_key + "# + ) + .fetch_all(db) + .await?; + for row in class_rows { + let Some(composite) = composite_map.get_mut(&row.deployed_model_id) else { + continue; + }; + let endpoint_url = match url::Url::parse(&row.endpoint_url) { + Ok(url) => url, + Err(e) => { + warn!(alias = %row.alias, class = %row.class_key, "Skipping class with an invalid endpoint URL: {e}"); + continue; + } + }; + let target_key = class_target_key(&row.alias, &row.class_key); + composite.classes.push(OnwardsClassRoute { + class_key: row.class_key, + realtime_inflight_limit: row.realtime_inflight_limit, + target: OnwardsTarget { + model_name: row.upstream_model_name, + reasoning_translation: resolve_reasoning_translation( + row.endpoint_reasoning_translation, + row.model_reasoning_translation_overrides, + &target_key, + ), + alias: target_key, + sanitize_responses: composite.sanitize_responses, + // A class's upstream is trusted like a self-hosted member + // when the endpoint is self-hosted. + trusted: parse_provider_kind(&row.endpoint_kind) != ProviderKind::External, + routing_rules: Vec::new(), + fallback_enabled: false, + fallback_on_rate_limit: false, + fallback_on_status: Vec::new(), + fallback_realtime_on_status: Vec::new(), + fallback_with_replacement: false, + fallback_max_attempts: None, + backoff_enabled: false, + backoff_initial_ms: 100, + backoff_max_ms: 5_000, + backoff_factor: 2.0, + backoff_jitter: "full".to_string(), + backoff_max_total_ms: None, + first_token_timeout_ms: None, + aimd: None, + endpoint_url, + endpoint_api_key: row.endpoint_api_key, + auth_header_name: row.auth_header_name, + auth_header_prefix: row.auth_header_prefix, + endpoint_accepts_scheduling_priority: row.endpoint_accepts_scheduling_priority, + endpoint_kind: parse_provider_kind(&row.endpoint_kind), + api_keys: Vec::new(), + }, + }); + } + // Add enabled components to their composite models for row in component_rows { // Parse the endpoint URL, skipping this component if invalid @@ -1167,6 +1338,142 @@ fn effective_batch_inflight_limit(batch_capacity: Option, default_batch_inf } } +/// The pool fallback configuration a composite's settings describe. +fn composite_fallback(composite: &OnwardsCompositeModel) -> Option { + if composite.fallback_enabled { + let backoff = composite.backoff_enabled.then_some(OnwardsBackoffConfig { + // The DB CHECK constraints guarantee these are positive and + // ordered, but we use `max(1)` defensively to avoid panicking + // the proxy if a row ever escaped validation. + initial_ms: composite.backoff_initial_ms.max(1) as u64, + max_ms: composite.backoff_max_ms.max(composite.backoff_initial_ms.max(1)) as u64, + factor: composite.backoff_factor.max(1.0), + jitter: match composite.backoff_jitter.as_str() { + "none" => OnwardsJitterStrategy::None, + _ => OnwardsJitterStrategy::Full, + }, + }); + let max_total_backoff_ms = composite.backoff_max_total_ms.and_then(|n| u64::try_from(n).ok()); + Some(OnwardsFallbackConfig { + enabled: true, + on_rate_limit: composite.fallback_on_rate_limit, + // Convert i32 status codes to u16 for onwards + on_status: composite.fallback_on_status.iter().map(|&s| s as u16).collect(), + realtime_on_status: composite.fallback_realtime_on_status.iter().map(|&s| s as u16).collect(), + with_replacement: composite.fallback_with_replacement, + max_attempts: composite + .fallback_max_attempts + .and_then(|n| usize::try_from(n).ok().filter(|&v| v >= 1)), + backoff, + max_total_backoff_ms, + first_token_timeout_ms: composite.first_token_timeout_ms.map(|ms| ms as u64), + aimd: composite.aimd.clone(), + affinity: composite.affinity.clone(), + }) + } else { + None + } +} + +/// The gateway targets of a class-routed model: one per class (`alias` for +/// standard, `alias:` otherwise), each with the class's single +/// upstream. `None` when the activation is unusable (no standard class, or a +/// redirect rule): the model then keeps its legacy route. +fn convert_class_routes( + composite: &OnwardsCompositeModel, + key_definitions: &mut HashMap, + overlays: &Overlays, + inflight_overrides: &InflightOverridesByAlias, + default_batch_inflight_capacity: Option, +) -> Option> { + let has_standard = composite.classes.iter().any(|class| class.class_key == STANDARD_CLASS); + let redirects = composite + .routing_rules + .iter() + .any(|rule| matches!(rule.action, RoutingAction::Redirect { .. })); + if !has_standard || redirects { + crate::background_error!( + ONWARDS_SYNC, + "invalid_class_activation", + Error, + alias = %composite.alias, + has_standard, + redirects, + "Class-routed model has no standard class or redirects traffic; serving its legacy route" + ); + return None; + } + for api_key in &composite.api_keys { + key_definitions.insert( + api_key.id.to_string(), + KeyDefinition { + key: api_key.secret.clone(), + rate_limit: None, + concurrency_limit: None, + labels: key_labels(api_key), + }, + ); + } + let keys: Option> = + (!composite.api_keys.is_empty()).then(|| composite.api_keys.iter().map(|k| k.secret.clone().into()).collect()); + let fallback = composite_fallback(composite).map(|mut config| { + // One upstream: there is nothing to keep a conversation on. + config.affinity = None; + config + }); + let targets = composite + .classes + .iter() + .map(|class| { + let target = &class.target; + let standard = class.class_key == STANDARD_CLASS; + let provider = ProviderSpec { + url: target.endpoint_url.clone(), + onwards_key: target.endpoint_api_key.clone(), + onwards_model: Some(target.model_name.clone()), + rate_limit: None, + concurrency_limit: None, + upstream_auth_header_name: (target.auth_header_name != "Authorization").then(|| target.auth_header_name.clone()), + upstream_auth_header_prefix: (target.auth_header_prefix != "Bearer ").then(|| target.auth_header_prefix.clone()), + response_headers: None, + weight: 1, + sanitize_response: composite.sanitize_responses, + request_timeout_secs: None, + trusted: Some(target.trusted), + propagate_trace_context: None, + reasoning_translation: target.reasoning_translation.clone().map(Into::into), + accepts_scheduling_priority: target.endpoint_accepts_scheduling_priority, + kind: target.endpoint_kind, + }; + let pool = PoolSpec { + keys: keys.clone(), + rate_limit: None, + concurrency_limit: None, + fallback: fallback.clone(), + strategy: OnwardsLoadBalanceStrategy::Priority, + providers: vec![provider], + response_headers: None, + sanitize_response: composite.sanitize_responses, + trusted: false, + // Deny rules are model-wide: every class carries them. + routing_rules: composite.routing_rules.clone(), + overlays: overlays.for_class(&composite.alias, &class.class_key), + inflight_limit: Some(class.realtime_inflight_limit.unwrap_or(composite.realtime_inflight_limit) as u32), + // Batch is standard-only and counted on the model's alias. + batch_inflight_limit: standard + .then(|| effective_batch_inflight_limit(composite.batch_capacity, default_batch_inflight_capacity, &composite.alias)) + .flatten(), + // `alias` itself is the model; only `alias:` is a class route. + class_route_of: (!standard).then(|| composite.alias.clone()), + account_inflight_limits: inflight_overrides.get(&target.alias).cloned().unwrap_or_default(), + }; + debug!(alias = %target.alias, upstream = %target.model_name, "Class route configured"); + (target.alias.clone(), TargetSpecOrList::Pool(pool)) + }) + .collect(); + Some(targets) +} + /// Converts a composite model to a TargetSpecOrList with weighted providers /// /// Uses onwards 0.10.0 weighted provider types for load balancing across @@ -1204,40 +1511,7 @@ fn convert_composite_to_target_spec( LoadBalancingStrategy::Priority => OnwardsLoadBalanceStrategy::Priority, }; - // Build fallback configuration - let fallback = if composite.fallback_enabled { - let backoff = composite.backoff_enabled.then_some(OnwardsBackoffConfig { - // The DB CHECK constraints guarantee these are positive and - // ordered, but we use `max(1)` defensively to avoid panicking - // the proxy if a row ever escaped validation. - initial_ms: composite.backoff_initial_ms.max(1) as u64, - max_ms: composite.backoff_max_ms.max(composite.backoff_initial_ms.max(1)) as u64, - factor: composite.backoff_factor.max(1.0), - jitter: match composite.backoff_jitter.as_str() { - "none" => OnwardsJitterStrategy::None, - _ => OnwardsJitterStrategy::Full, - }, - }); - let max_total_backoff_ms = composite.backoff_max_total_ms.and_then(|n| u64::try_from(n).ok()); - Some(OnwardsFallbackConfig { - enabled: true, - on_rate_limit: composite.fallback_on_rate_limit, - // Convert i32 status codes to u16 for onwards - on_status: composite.fallback_on_status.iter().map(|&s| s as u16).collect(), - realtime_on_status: composite.fallback_realtime_on_status.iter().map(|&s| s as u16).collect(), - with_replacement: composite.fallback_with_replacement, - max_attempts: composite - .fallback_max_attempts - .and_then(|n| usize::try_from(n).ok().filter(|&v| v >= 1)), - backoff, - max_total_backoff_ms, - first_token_timeout_ms: composite.first_token_timeout_ms.map(|ms| ms as u64), - aimd: composite.aimd.clone(), - affinity: composite.affinity.clone(), - }) - } else { - None - }; + let fallback = composite_fallback(composite); // Build provider specs from components, grouped by the pool each membership // belongs to. The query orders by (pool, sort_order), so each pool's members @@ -1369,6 +1643,7 @@ fn convert_composite_to_target_spec( } else { None }, + class_route_of: None, account_inflight_limits: if pool_name == DEFAULT_COMPONENT_POOL { inflight_overrides.get(&composite.alias).cloned().unwrap_or_default() } else { @@ -1401,7 +1676,7 @@ fn convert_to_config_file( targets: Vec, composites: Vec, strict_mode: bool, - overlays: &OverlaysByAlias, + overlays: &Overlays, inflight_overrides: &InflightOverridesByAlias, accounts: AccountsById, default_batch_inflight_capacity: Option, @@ -1513,9 +1788,10 @@ fn convert_to_config_file( sanitize_response: target.sanitize_responses, trusted: false, routing_rules: target.routing_rules, - overlays: overlays.get(&target.alias).cloned().unwrap_or_default(), + overlays: overlays.by_target.get(&target.alias).cloned().unwrap_or_default(), inflight_limit: None, batch_inflight_limit: None, + class_route_of: None, account_inflight_limits: HashMap::new(), }; @@ -1525,6 +1801,18 @@ fn convert_to_config_file( // Convert composite models (including those with no components - they'll return 503) for composite in composites { + if composite.routing_mode == "class_routes" + && let Some(class_targets) = convert_class_routes( + &composite, + &mut key_definitions, + overlays, + inflight_overrides, + default_batch_inflight_capacity, + ) + { + target_specs.extend(class_targets); + continue; + } if composite.components.is_empty() { crate::background_error!( ONWARDS_SYNC, @@ -1538,7 +1826,7 @@ fn convert_to_config_file( let (alias, spec) = convert_composite_to_target_spec( &composite, &mut key_definitions, - overlays, + &overlays.by_target, inflight_overrides, default_batch_inflight_capacity, ); diff --git a/dwctl/src/sync/onwards_config/tests.rs b/dwctl/src/sync/onwards_config/tests.rs index 1163f9d94..7411325b3 100644 --- a/dwctl/src/sync/onwards_config/tests.rs +++ b/dwctl/src/sync/onwards_config/tests.rs @@ -2722,9 +2722,127 @@ async fn a_priority_only_policy_keeps_overlay_targets(pool: sqlx::PgPool) { .await .unwrap(); let overlays = super::load_overlays_from_db(&pool).await.unwrap(); - let overlay = &overlays["hint-model"][&user.to_string()]; + let overlay = &overlays.by_target["hint-model"][&user.to_string()]; assert_eq!( (overlay.priority, overlay.ttft_ms, overlay.itl_ms), (Some(400), Some(800), Some(30)) ); } + +/// Make the fixture's priority composite class-routed: standard on the default +/// endpoint, fast on the custom one with its own limit. +async fn activate_composite_classes(pool: &sqlx::PgPool, with_standard: bool) { + let composite = uuid::Uuid::parse_str("50000000-0000-0000-0000-000000000001").unwrap(); + sqlx::query("UPDATE deployed_models SET routing_mode = 'class_routes' WHERE id = $1") + .bind(composite) + .execute(pool) + .await + .unwrap(); + let classes: &[(&str, &str, &str, Option)] = if with_standard { + &[ + ("standard", "30000000-0000-0000-0000-000000000001", "pool-standard", None), + ("fast", "30000000-0000-0000-0000-000000000002", "pool-fast", Some(3)), + ] + } else { + &[("fast", "30000000-0000-0000-0000-000000000002", "pool-fast", Some(3))] + }; + for (class, endpoint, upstream, limit) in classes { + sqlx::query( + "INSERT INTO model_serving_classes (deployed_model_id, class_key, display_name, inference_endpoint_id, upstream_model_name, realtime_inflight_limit) + VALUES ($1, $2, $2, $3::uuid, $4, $5)", + ) + .bind(composite) + .bind(class) + .bind(endpoint) + .bind(upstream) + .bind(limit) + .execute(pool) + .await + .unwrap(); + } +} + +#[dwctl_test_macros::test(fixtures(path = "fixtures", scripts("cache_base")))] +async fn test_class_routed_model_has_one_target_per_class(pool: sqlx::PgPool) { + let composite = uuid::Uuid::parse_str("50000000-0000-0000-0000-000000000001").unwrap(); + let user_a = "00000000-0000-0000-0000-0000000000a1"; + activate_composite_classes(&pool, true).await; + sqlx::query("INSERT INTO realtime_inflight_limit_overrides (deployed_model_id, user_id, inflight_limit) VALUES ($1, $2::uuid, 50)") + .bind(composite) + .bind(user_a) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO realtime_inflight_class_overrides (deployed_model_id, user_id, class_key, inflight_limit) VALUES ($1, $2::uuid, 'fast', 9)") + .bind(composite) + .bind(user_a) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO account_model_policies (user_id, deployed_model_id, serving_class, priority, ttft_ms) + VALUES ($1::uuid, $2, NULL, 100, NULL), ($1::uuid, $2, 'fast', 500, 900)", + ) + .bind(user_a) + .bind(composite) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO model_traffic_rules (deployed_model_id, api_key_purpose, action) VALUES ($1, 'playground', 'deny')") + .bind(composite) + .execute(&pool) + .await + .unwrap(); + let model_limit: i32 = sqlx::query_scalar("SELECT realtime_inflight_limit FROM deployed_models WHERE id = $1") + .bind(composite) + .fetch_one(&pool) + .await + .unwrap(); + + let targets = super::load_targets_from_db(&pool, &[], false).await.unwrap(); + + let standard = targets.targets.get("composite-priority").unwrap().default_pool().clone(); + let fast = targets.targets.get("composite-priority:fast").unwrap().default_pool().clone(); + // Each class has a single provider: its own upstream. + assert_eq!(standard.providers().len(), 1); + assert_eq!(standard.providers()[0].target.onwards_model.as_deref(), Some("pool-standard")); + assert_eq!(fast.providers().len(), 1); + assert_eq!(fast.providers()[0].target.onwards_model.as_deref(), Some("pool-fast")); + assert!( + fast.providers()[0] + .target + .url + .as_str() + .starts_with("https://api.custom.example.com/") + ); + + // Limits: the class default and its account overrides, each from its own table. + let standard_limits = standard.inflight_limits().unwrap(); + assert_eq!(standard_limits.default, model_limit as u32); + assert_eq!(standard_limits.for_account(user_a), 50); + let fast_limits = fast.inflight_limits().unwrap(); + assert_eq!(fast_limits.default, 3); + assert_eq!(fast_limits.for_account(user_a), 9); + // Batch runs on standard only. + assert!(standard.batch_inflight_limit().is_some()); + assert_eq!(fast.batch_inflight_limit(), None); + + // The model-wide policy applies to every class; the class row narrows it. + assert_eq!(standard.overlays()[user_a].priority, Some(100)); + assert_eq!(fast.overlays()[user_a].priority, Some(500)); + assert_eq!(fast.overlays()[user_a].ttft_ms, Some(900)); + + // Deny rules are model-wide, and both classes admit the model's keys. + assert_eq!(standard.routing_rules().len(), 1); + assert_eq!(fast.routing_rules().len(), 1); + assert!(standard.keys().is_some() && fast.keys().is_some()); +} + +#[dwctl_test_macros::test(fixtures(path = "fixtures", scripts("cache_base")))] +async fn test_an_activation_without_standard_keeps_the_legacy_route(pool: sqlx::PgPool) { + activate_composite_classes(&pool, false).await; + let targets = super::load_targets_from_db(&pool, &[], false).await.unwrap(); + let legacy = targets.targets.get("composite-priority").unwrap(); + assert!(legacy.default_pool().providers().len() > 1, "the composite's members still serve"); + assert!(targets.targets.get("composite-priority:fast").is_none()); +} diff --git a/dwctl/src/test/serving_classes.rs b/dwctl/src/test/serving_classes.rs index befea30a8..a77c83101 100644 --- a/dwctl/src/test/serving_classes.rs +++ b/dwctl/src/test/serving_classes.rs @@ -416,3 +416,176 @@ async fn json_key_order_survives_batch_upload(pool: PgPool) { content.assert_status_ok(); assert!(content.text().contains(ORDERED_SCHEMA), "{}", content.text()); } + +/// Activate `policy` as a class-routed model: standard and fast, each its own +/// upstream on the fixture's endpoint, with public class prices. +async fn activate_policy_classes(f: &Fixture, pool: &PgPool) { + sqlx::query( + "INSERT INTO model_serving_classes (deployed_model_id, class_key, display_name, inference_endpoint_id, upstream_model_name, realtime_inflight_limit) + SELECT dm.id, c.key, c.key, ie.id, c.upstream, c.lim + FROM deployed_models dm, inference_endpoints ie, + (VALUES ('standard', 'pool-standard', NULL::int), ('fast', 'pool-fast', 3)) c(key, upstream, lim) + WHERE dm.alias = 'policy' AND ie.name = 'serving-test'", + ) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO model_tariffs (deployed_model_id, serving_class, name, input_price_per_token, output_price_per_token, api_key_purpose) + SELECT id, c.key, c.key, c.price, c.price, 'realtime' FROM deployed_models, + (VALUES ('standard', 0.000002::numeric), ('fast', 0.000004::numeric)) c(key, price) + WHERE alias = 'policy'", + ) + .execute(pool) + .await + .unwrap(); + sqlx::query("UPDATE deployed_models SET routing_mode = 'class_routes' WHERE alias = 'policy'") + .execute(pool) + .await + .unwrap(); + f.services.sync_onwards_config(pool).await.unwrap(); +} + +#[dwctl_test_macros::test] +async fn class_routes_serve_realtime_by_class_and_bill_the_class(pool: PgPool) { + let f = Fixture::new(&pool).await; + activate_policy_classes(&f, &pool).await; + + // Discovery lists the class beside the model. + let models: Value = f + .server + .get("/ai/v1/models") + .add_header("Authorization", format!("Bearer {}", f.key)) + .await + .json(); + let ids: Vec<&str> = models["data"].as_array().unwrap().iter().filter_map(|m| m["id"].as_str()).collect(); + assert!(ids.contains(&"policy") && ids.contains(&"policy:fast"), "{ids:?}"); + assert!(!ids.contains(&"policy:standard")); + + // Realtime: the alias reaches standard's upstream, alias:fast reaches fast's. + f.post("chat/completions", "policy", false).await.assert_status_ok(); + assert_eq!(f.last_body().await["model"], "pool-standard"); + f.post("chat/completions", "policy:fast", false).await.assert_status_ok(); + assert_eq!(f.last_body().await["model"], "pool-fast"); + + // Flex never runs on a non-standard class. + let response = f + .server + .post("/ai/v1/chat/completions") + .add_header("Authorization", format!("Bearer {}", f.key)) + .json(&json!({"model": "policy:fast", "messages": [{"role": "user", "content": "hi"}], "service_tier": "flex"})) + .await; + response.assert_status(StatusCode::BAD_REQUEST); + assert!(response.text().contains("serving_class_realtime_only"), "{}", response.text()); + assert!( + response.text().contains("'policy'"), + "the refusal names the standard route: {}", + response.text() + ); + // Not even where the endpoint would serve flex as realtime. + let response = f + .server + .post("/ai/v1/embeddings") + .add_header("Authorization", format!("Bearer {}", f.key)) + .json(&json!({"model": "policy:fast", "input": "hi", "service_tier": "flex"})) + .await; + response.assert_status(StatusCode::BAD_REQUEST); + assert!(response.text().contains("serving_class_realtime_only"), "{}", response.text()); + + // Both realtime requests are recorded as the model, in their class, at + // their class's price. + let rows = tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let rows: Vec<(String, Option, Option, Decimal)> = sqlx::query_as( + "SELECT model, requested_serving_class, resolved_serving_class, total_cost FROM http_analytics + WHERE user_id = $1 AND status_code = 200 ORDER BY total_cost", + ) + .bind(f.user) + .fetch_all(&pool) + .await + .unwrap(); + if rows.len() == 2 { + break rows; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + }) + .await + .expect("two priced analytics rows must be persisted"); + assert_eq!( + rows, + vec![ + ("policy".to_owned(), None, Some("standard".to_owned()), Decimal::new(30, 6)), + ( + "policy".to_owned(), + Some("fast".to_owned()), + Some("fast".to_owned()), + Decimal::new(60, 6) + ), + ] + ); + f.services.shutdown().await; +} + +#[dwctl_test_macros::test] +async fn an_activation_without_standard_serves_and_prices_the_legacy_route(pool: PgPool) { + let f = Fixture::new(&pool).await; + activate_policy_classes(&f, &pool).await; + sqlx::query("DELETE FROM model_serving_classes WHERE class_key = 'standard'") + .execute(&pool) + .await + .unwrap(); + f.services.sync_onwards_config(&pool).await.unwrap(); + + // No class route is listed or served, and the model takes its legacy route. + let models: Value = f + .server + .get("/ai/v1/models") + .add_header("Authorization", format!("Bearer {}", f.key)) + .await + .json(); + let ids: Vec<&str> = models["data"].as_array().unwrap().iter().filter_map(|m| m["id"].as_str()).collect(); + assert!(ids.contains(&"policy") && !ids.contains(&"policy:fast"), "{ids:?}"); + f.post("chat/completions", "policy", false).await.assert_status_ok(); + assert_ne!(f.last_body().await["model"], "pool-standard"); + + // Billing reads no class price for the model while its classes don't serve. + let class_prices: i64 = sqlx::query_scalar( + "SELECT count(*) FROM deployed_models dm, + effective_model_tariff(dm.id, NULL, 'realtime', NULL, 'fast', now()) t + WHERE dm.alias = 'policy' AND t.serving_class IS NOT NULL", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(class_prices, 0); + f.services.shutdown().await; +} + +#[dwctl_test_macros::test] +async fn batch_lines_naming_a_class_route_are_refused(pool: PgPool) { + let f = Fixture::new(&pool).await; + activate_policy_classes(&f, &pool).await; + let upload = |model: &str| { + let line = json!({"custom_id": "r0", "method": "POST", "url": "/v1/chat/completions", + "body": {"model": model, "messages": [{"role": "user", "content": "hi"}]}}); + MultipartForm::new() + .add_part("purpose", Part::text("batch")) + .add_part("file", Part::bytes(format!("{line}\n").into_bytes()).file_name("batch.jsonl")) + }; + let response = f + .server + .post("/ai/v1/files") + .add_header("Authorization", format!("Bearer {}", f.key)) + .multipart(upload("policy:fast")) + .await; + response.assert_status(StatusCode::BAD_REQUEST); + assert!(response.text().contains("realtime requests only"), "{}", response.text()); + f.server + .post("/ai/v1/files") + .add_header("Authorization", format!("Bearer {}", f.key)) + .multipart(upload("policy")) + .await + .assert_status(StatusCode::CREATED); + f.services.shutdown().await; +} diff --git a/model-provisioning.schema.json b/model-provisioning.schema.json index 767e151c4..6538827c0 100644 --- a/model-provisioning.schema.json +++ b/model-provisioning.schema.json @@ -156,6 +156,11 @@ }, "default": [] }, + "routing_mode": { + "description": "How the model routes: `legacy` (its composite routing) or\n`class_routes` (each class in `class_routes` to its own upstream:\n`alias` to `standard`, `alias:` to that class).", + "$ref": "#/$defs/RoutingMode", + "default": "legacy" + }, "serving_classes": { "description": "Retired. Accepted only when empty, so existing catalog files still\nload; remove it.", "type": "object", @@ -163,7 +168,7 @@ "maxProperties": 0 }, "class_routes": { - "description": "Explicit class destinations and prices, staged independently of legacy routing.\nDeclaring these does not activate routes or advertise additional model names.", + "description": "Class destinations and prices. Declaring them alone changes nothing;\n`routing_mode: class_routes` activates them. `standard` is required.", "type": "object", "additionalProperties": { "$ref": "#/$defs/ClassRoute" @@ -177,6 +182,67 @@ "additionalProperties": false, "required": [ "alias" + ], + "allOf": [ + { + "if": { + "properties": { + "routing_mode": { + "const": "class_routes" + } + }, + "required": [ + "routing_mode" + ] + }, + "then": { + "properties": { + "class_routes": { + "required": [ + "standard" + ] + }, + "traffic_rules": { + "items": { + "not": { + "properties": { + "action": { + "const": "redirect" + } + }, + "required": [ + "action" + ] + } + } + } + }, + "required": [ + "class_routes" + ] + } + }, + { + "if": { + "properties": { + "class_routes": { + "minProperties": 1 + } + }, + "required": [ + "class_routes" + ] + }, + "then": { + "properties": { + "class_routes": { + "required": [ + "standard" + ] + } + } + } + } ] }, "ModelKind": { @@ -770,6 +836,21 @@ "continuation" ] }, + "RoutingMode": { + "description": "How a virtual model routes its requests.", + "oneOf": [ + { + "description": "The model's composite routing (`routing`).", + "type": "string", + "const": "legacy" + }, + { + "description": "Each class in `class_routes` routes to its own upstream.", + "type": "string", + "const": "class_routes" + } + ] + }, "ClassRoute": { "description": "Complete catalog declaration for a stable class of the existing public model.", "type": "object", @@ -784,12 +865,21 @@ "type": "string" }, "aliases": { - "description": "Optional exact ingress spellings, never additional discovery entries.", + "description": "Retired. Accepted only when empty: a class is reached only by the\nmodel's alias (`alias` or `alias:`).", "type": "array", "items": { "type": "string" }, - "default": [] + "maxItems": 0 + }, + "realtime_inflight_limit": { + "description": "Concurrent realtime requests one account may have in flight on this\nclass. Omit for the model's `settings.realtime_inflight_limit`.", + "type": [ + "integer", + "null" + ], + "format": "int32", + "minimum": 1 }, "tariffs": { "type": "array", diff --git a/onwards/src/load_balancer.rs b/onwards/src/load_balancer.rs index 8abf2378e..944cf52fd 100644 --- a/onwards/src/load_balancer.rs +++ b/onwards/src/load_balancer.rs @@ -106,6 +106,8 @@ pub struct ProviderPool { /// The alias's global batch in-flight cap, shared across every daemon pod /// and control-layer replica. `None` means batch is uncapped on this alias. batch_inflight: Option, + /// The model this alias is a serving-class route of, when it is one. + class_route_of: Option, } /// The serving policy declared on an alias: its per-account overlays. Behind @@ -183,6 +185,7 @@ impl ProviderPool { serving: AliasServing::default(), inflight: None, batch_inflight: None, + class_route_of: None, } } @@ -219,6 +222,7 @@ impl ProviderPool { serving: AliasServing::default(), inflight: None, batch_inflight: None, + class_route_of: None, } } @@ -246,6 +250,16 @@ impl ProviderPool { self.batch_inflight } + pub fn with_class_route_of(mut self, model: Option) -> Self { + self.class_route_of = model; + self + } + + /// The model this alias is a serving-class route of, when it is one. + pub fn class_route_of(&self) -> Option<&str> { + self.class_route_of.as_deref() + } + /// Per-account overrides on this alias, keyed by account id. pub fn overlays(&self) -> &HashMap { self.serving.overlays() diff --git a/onwards/src/target.rs b/onwards/src/target.rs index 2f2070b82..3afc4288c 100644 --- a/onwards/src/target.rs +++ b/onwards/src/target.rs @@ -443,6 +443,12 @@ pub struct PoolSpec { #[serde(default)] pub batch_inflight_limit: Option, + /// The model this alias is a serving-class route of (`model:` + /// names the `` route of `model`), when it is one. Set by the + /// config writer; onwards routes the alias like any other. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub class_route_of: Option, + #[serde(default)] #[builder(default)] pub account_inflight_limits: HashMap, @@ -570,6 +576,7 @@ pub struct PoolConfig { pub overlays: HashMap, pub inflight_limit: Option, pub batch_inflight_limit: Option, + pub class_route_of: Option, pub account_inflight_limits: HashMap, pub providers: Vec, } @@ -589,6 +596,7 @@ impl From for PoolConfig { overlays: pool.overlays, inflight_limit: pool.inflight_limit, batch_inflight_limit: pool.batch_inflight_limit, + class_route_of: pool.class_route_of, account_inflight_limits: pool.account_inflight_limits, providers: pool.providers, } @@ -737,6 +745,7 @@ impl TargetSpecOrList { overlays: HashMap::new(), inflight_limit: None, batch_inflight_limit: None, + class_route_of: None, account_inflight_limits: HashMap::new(), providers, }) @@ -780,6 +789,7 @@ impl TargetSpecOrList { overlays: HashMap::new(), inflight_limit: None, batch_inflight_limit: None, + class_route_of: None, account_inflight_limits: HashMap::new(), providers: vec![provider], }) @@ -1497,7 +1507,8 @@ fn build_pool( default, accounts: pool_config.account_inflight_limits, })) - .with_batch_inflight_limit(pool_config.batch_inflight_limit)) + .with_batch_inflight_limit(pool_config.batch_inflight_limit) + .with_class_route_of(pool_config.class_route_of)) } impl Targets { @@ -1508,6 +1519,14 @@ impl Targets { Some(model).filter(|alias| self.targets.contains_key(*alias)) } + /// The model `alias` is a serving-class route of, when it is a configured + /// class route (see [`PoolSpec::class_route_of`]). + pub fn class_route_of(&self, alias: &str) -> Option { + self.targets + .get(alias) + .and_then(|pools| pools.default_pool().class_route_of().map(str::to_owned)) + } + pub async fn from_config_file(config_path: &PathBuf) -> Result { let contents = tokio::fs::read_to_string(config_path).await.map_err(|e| { anyhow!( @@ -2902,6 +2921,7 @@ mod tests { overlays: Default::default(), inflight_limit: None, batch_inflight_limit: None, + class_route_of: None, account_inflight_limits: Default::default(), };