diff --git a/docs-site/src/content/docs/guides/routing-profile-editor.md b/docs-site/src/content/docs/guides/routing-profile-editor.md index 2d48bcd7b18..5cf5fc6d71e 100644 --- a/docs-site/src/content/docs/guides/routing-profile-editor.md +++ b/docs-site/src/content/docs/guides/routing-profile-editor.md @@ -11,7 +11,7 @@ The **Models → Routing** tab in the OpenCodex dashboard can manage `config.rou 2. Select **Create profile**. 3. Enter an `id`. The canonical model id is `policy/`. 4. Add one or more explicit provider/model candidates. -5. Configure optional requirements, scoring weights, cost limits, and unknown-evidence behavior. +5. Configure optional requirements, scoring weights, cost limits (`maxEstimatedCostUsd`, optional `onUnknownCost`), and unknown-evidence behavior. 6. Save the profile. Profile ids are immutable after creation. To use a different id, create a new profile and remove the old one after updating callers. @@ -28,6 +28,14 @@ The dashboard sends the same profile object used by `config.routingProfiles` to A successful save persists the profile through the normal config writer, reconciles live state, and refreshes the model catalog. Validation failures leave the previous configuration unchanged and are shown in the editor. +When `limits.maxEstimatedCostUsd` is configured, `limits.onUnknownCost` defaults to `"allow"`: an unknown cost estimate does not get a +cap-specific exclusion, and dry-run / live route-decision traces stamp +`cost.capOutcome: "unknown-allowed"` so operators can tell the cap was not proven. Set `"exclude"` +when the ceiling must fail closed (`cost-limit-unknown`, with +`cost.capOutcome: "unknown-excluded"`). Configuring `onUnknownCost` alone is inert and does not emit a cap outcome. This is separate from +`unknownEvidence.cost`, which can still exclude or penalize unknown prices independently of the +cap outcome. + ## Dry-run a saved profile Select a saved profile and use **Dry-run evaluation** to add request evidence such as context-window size, tool use, image input, or structured output. Dry-run evaluates eligibility and scoring but never sends an upstream model request. @@ -57,7 +65,7 @@ Example save payload: ], "require": { "tools": true, "minContextWindow": 128000 }, "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.1, "quota": 0.1 }, - "limits": { "maxEstimatedCostUsd": 0.5 }, + "limits": { "maxEstimatedCostUsd": 0.5, "onUnknownCost": "allow" }, "unknownEvidence": { "capability": "exclude", "health": "penalize", diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 18ca8a36905..8a2b039c26e 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -114,7 +114,7 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c | `alias?` | `string` | — | Optional public model id in place of `policy/`. | | `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). | | `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights, normalized deterministically. `health`, `quota`, and `cost` have score dimensions; the configured-priority share is `1 - health - quota - cost` (default 0.55), and `latency` folds into that priority share rather than scoring independently. | -| `limits?` | object | — | Hard limits, e.g. `maxEstimatedCostUsd` (enforced by the dry-run evaluator when candidate cost evidence is known). | +| `limits?` | object | — | Hard limits. `maxEstimatedCostUsd` excludes a candidate when its estimated cost is known and above the cap. When that cap is set, `onUnknownCost` (`"allow"` default, or `"exclude"`) controls unknown estimates: allow prevents a cap-specific exclusion and records `cost.capOutcome: "unknown-allowed"`; exclude emits `cost-limit-unknown` and `capOutcome: "unknown-excluded"`. `onUnknownCost` alone (no cap) is inert. Separate from `unknownEvidence.cost`, which can still exclude or penalize unknown prices via `unknown-price` / scoring. | | `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | `require` supports: `minContextWindow` (positive integer), `minQuotaHeadroom` (0..1 fraction), @@ -145,7 +145,7 @@ candidate evidence is provided through the API (`POST /api/routing-profiles/dry- ], "require": { "tools": true, "minContextWindow": 128000 }, "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.10, "quota": 0.10 }, - "limits": { "maxEstimatedCostUsd": 0.50 }, + "limits": { "maxEstimatedCostUsd": 0.50, "onUnknownCost": "allow" }, "unknownEvidence": { "capability": "exclude", "health": "penalize", @@ -185,8 +185,12 @@ Both are virtual namespaces with aliases and collision validation; they differ i is chosen. Profile scoring combines the configured-priority component with the health (RI-06), quota (RI-07), and cost (RI-08) score dimensions where evidence is present; the `latency` weight folds into the priority share rather than scoring independently. Cost is also enforced through the -`limits.maxEstimatedCostUsd` cap (a candidate whose estimated cost exceeds the cap is excluded). -Per-request route-decision traces are recorded when a policy profile executes. +`limits.maxEstimatedCostUsd` cap: a candidate whose estimated cost is known and exceeds the cap is +excluded (`cost-limit`). When a cap is configured and the estimate is unknown, the default `limits.onUnknownCost: "allow"` +records `cost.capOutcome: "unknown-allowed"` on the route-decision trace without a cap exclusion; +set `onUnknownCost: "exclude"` for a fail-closed ceiling (`cost-limit-unknown`). Cap outcome is not +overall eligibility — `unknownEvidence.cost: "exclude"` can still add `unknown-price` and mark the +candidate ineligible. Per-request route-decision traces are recorded when a policy profile executes. ### Catalog eligibility diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 917a92f563a..31e85574669 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -46,6 +46,20 @@ export const de: Record = { "routing.candidate": "Kandidat", "routing.eligible": "Geeignet", "routing.exclusions": "Ausschlüsse", + "routing.costCap": "Kostenobergrenze", + "routing.capOutcome.satisfied": "innerhalb des Limits", + "routing.capOutcome.exceeded": "über dem Limit", + "routing.capOutcome.unknown-allowed": "unbekannt (erlaubt)", + "routing.capOutcome.unknown-excluded": "unbekannt (ausgeschlossen)", + "routing.exclusion.capability-unsatisfied": "Anforderung nicht erfüllt", + "routing.exclusion.unknown-capability": "unbekannte Fähigkeit", + "routing.exclusion.cost-limit": "über Kostenobergrenze", + "routing.exclusion.cost-limit-unknown": "unbekannte Kosten unter Obergrenze", + "routing.exclusion.cooldown": "Abklingzeit", + "routing.exclusion.unknown-health": "unbekannter Zustand", + "routing.exclusion.unknown-quota": "unbekanntes Kontingent", + "routing.exclusion.unknown-price": "unbekannter Preis", + "routing.exclusion.other": "Ausschluss: {code}", "routing.score": "Punktzahl", "routing.selected": "ausgewählt", "routing.yes": "ja", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 7bdebfeb07c..2dc6a0793a0 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -89,6 +89,20 @@ export const en = { "routing.candidate": "Candidate", "routing.eligible": "Eligible", "routing.exclusions": "Exclusions", + "routing.costCap": "Cost cap", + "routing.capOutcome.satisfied": "within limit", + "routing.capOutcome.exceeded": "over limit", + "routing.capOutcome.unknown-allowed": "unknown (allowed)", + "routing.capOutcome.unknown-excluded": "unknown (excluded)", + "routing.exclusion.capability-unsatisfied": "capability not met", + "routing.exclusion.unknown-capability": "unknown capability", + "routing.exclusion.cost-limit": "over cost cap", + "routing.exclusion.cost-limit-unknown": "unknown cost under cap", + "routing.exclusion.cooldown": "cooldown", + "routing.exclusion.unknown-health": "unknown health", + "routing.exclusion.unknown-quota": "unknown quota", + "routing.exclusion.unknown-price": "unknown price", + "routing.exclusion.other": "exclusion: {code}", "routing.score": "Score", "routing.selected": "selected", "routing.yes": "yes", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 4d57ca67013..c097ec8b43f 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -46,6 +46,20 @@ export const ja: Record = { "routing.candidate": "候補", "routing.eligible": "対象", "routing.exclusions": "除外", + "routing.costCap": "コスト上限", + "routing.capOutcome.satisfied": "上限内", + "routing.capOutcome.exceeded": "上限超過", + "routing.capOutcome.unknown-allowed": "不明（許可）", + "routing.capOutcome.unknown-excluded": "不明（除外）", + "routing.exclusion.capability-unsatisfied": "能力要件未達", + "routing.exclusion.unknown-capability": "能力不明", + "routing.exclusion.cost-limit": "コスト上限超過", + "routing.exclusion.cost-limit-unknown": "上限下でコスト不明", + "routing.exclusion.cooldown": "クールダウン", + "routing.exclusion.unknown-health": "健全性不明", + "routing.exclusion.unknown-quota": "割当不明", + "routing.exclusion.unknown-price": "価格不明", + "routing.exclusion.other": "除外: {code}", "routing.score": "スコア", "routing.selected": "選択済み", "routing.yes": "はい", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index bf842b922f7..d81b1543135 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -46,6 +46,20 @@ export const ko: Record = { "routing.candidate": "후보", "routing.eligible": "적격", "routing.exclusions": "제외", + "routing.costCap": "비용 상한", + "routing.capOutcome.satisfied": "한도 이내", + "routing.capOutcome.exceeded": "한도 초과", + "routing.capOutcome.unknown-allowed": "알 수 없음(허용)", + "routing.capOutcome.unknown-excluded": "알 수 없음(제외)", + "routing.exclusion.capability-unsatisfied": "기능 미충족", + "routing.exclusion.unknown-capability": "알 수 없는 기능", + "routing.exclusion.cost-limit": "비용 상한 초과", + "routing.exclusion.cost-limit-unknown": "상한 이하 비용 불명", + "routing.exclusion.cooldown": "쿨다운", + "routing.exclusion.unknown-health": "상태 불명", + "routing.exclusion.unknown-quota": "할당량 불명", + "routing.exclusion.unknown-price": "가격 불명", + "routing.exclusion.other": "제외: {code}", "routing.score": "점수", "routing.selected": "선택됨", "routing.yes": "예", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 48a584e5e23..07089f6e4de 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -46,6 +46,20 @@ export const ru: Record = { "routing.candidate": "Кандидат", "routing.eligible": "Допустим", "routing.exclusions": "Исключения", + "routing.costCap": "Лимит стоимости", + "routing.capOutcome.satisfied": "в пределах лимита", + "routing.capOutcome.exceeded": "сверх лимита", + "routing.capOutcome.unknown-allowed": "неизвестно (разрешено)", + "routing.capOutcome.unknown-excluded": "неизвестно (исключено)", + "routing.exclusion.capability-unsatisfied": "требование не выполнено", + "routing.exclusion.unknown-capability": "неизвестная возможность", + "routing.exclusion.cost-limit": "сверх лимита стоимости", + "routing.exclusion.cost-limit-unknown": "неизвестная стоимость при лимите", + "routing.exclusion.cooldown": "пауза", + "routing.exclusion.unknown-health": "неизвестное состояние", + "routing.exclusion.unknown-quota": "неизвестная квота", + "routing.exclusion.unknown-price": "неизвестная цена", + "routing.exclusion.other": "исключение: {code}", "routing.score": "Оценка", "routing.selected": "выбран", "routing.yes": "да", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 494fa072117..546aebead9e 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -88,6 +88,20 @@ export const tr: Record = { "routing.candidate": "Aday", "routing.eligible": "Uygun", "routing.exclusions": "Hariç Tutulanlar", + "routing.costCap": "Maliyet tavanı", + "routing.capOutcome.satisfied": "limit içinde", + "routing.capOutcome.exceeded": "limit aşıldı", + "routing.capOutcome.unknown-allowed": "bilinmiyor (izinli)", + "routing.capOutcome.unknown-excluded": "bilinmiyor (hariç tutuldu)", + "routing.exclusion.capability-unsatisfied": "yetenek karşılanmadı", + "routing.exclusion.unknown-capability": "bilinmeyen yetenek", + "routing.exclusion.cost-limit": "maliyet tavanı aşıldı", + "routing.exclusion.cost-limit-unknown": "tavan altında bilinmeyen maliyet", + "routing.exclusion.cooldown": "soğuma süresi", + "routing.exclusion.unknown-health": "bilinmeyen sağlık", + "routing.exclusion.unknown-quota": "bilinmeyen kota", + "routing.exclusion.unknown-price": "bilinmeyen fiyat", + "routing.exclusion.other": "hariç tutma: {code}", "routing.score": "Puan", "routing.selected": "seçildi", "routing.yes": "evet", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index f1f0eb62b3b..71c41897a0c 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -46,6 +46,20 @@ export const zh: Record = { "routing.candidate": "候选", "routing.eligible": "合格", "routing.exclusions": "排除项", + "routing.costCap": "成本上限", + "routing.capOutcome.satisfied": "未超限", + "routing.capOutcome.exceeded": "已超限", + "routing.capOutcome.unknown-allowed": "未知（允许）", + "routing.capOutcome.unknown-excluded": "未知（排除）", + "routing.exclusion.capability-unsatisfied": "能力未满足", + "routing.exclusion.unknown-capability": "能力未知", + "routing.exclusion.cost-limit": "超出成本上限", + "routing.exclusion.cost-limit-unknown": "上限下成本未知", + "routing.exclusion.cooldown": "冷却中", + "routing.exclusion.unknown-health": "健康状态未知", + "routing.exclusion.unknown-quota": "配额未知", + "routing.exclusion.unknown-price": "价格未知", + "routing.exclusion.other": "排除项：{code}", "routing.score": "分数", "routing.selected": "已选择", "routing.yes": "是", diff --git a/gui/src/pages/RoutingProfiles.tsx b/gui/src/pages/RoutingProfiles.tsx index 56543c89196..b6a19ac3cec 100644 --- a/gui/src/pages/RoutingProfiles.tsx +++ b/gui/src/pages/RoutingProfiles.tsx @@ -11,6 +11,7 @@ import { type OptionalBoolean, type RoutingProfileDraft, type RoutingProfileDto, + type UnknownCostCapMode, type UnknownEvidenceMode, } from "../routing-profile-editor-data"; import { readJsonIfOk } from "../fetch-json"; @@ -23,6 +24,7 @@ type DryRunCandidate = { eligible: boolean; exclusions: Array<{ code: string; detail?: string }>; score?: { total: number; components: Record }; + cost?: { capOutcome?: string; estimatedUsd?: number; incomplete?: boolean; limitUsd?: number }; }; type Analytics = { @@ -69,6 +71,7 @@ const NUMERIC_REQUIREMENTS = Object.keys(NUMERIC_REQUIREMENT_SPEC) as Array, + unavailable: string, +): string { + switch (value) { + case "satisfied": + return t("routing.capOutcome.satisfied"); + case "exceeded": + return t("routing.capOutcome.exceeded"); + case "unknown-allowed": + return t("routing.capOutcome.unknown-allowed"); + case "unknown-excluded": + return t("routing.capOutcome.unknown-excluded"); + default: + return unavailable; + } +} + +function fmtExclusion(code: string, t: ReturnType): string { + switch (code) { + case "capability-unsatisfied": + return t("routing.exclusion.capability-unsatisfied"); + case "unknown-capability": + return t("routing.exclusion.unknown-capability"); + case "cost-limit": + return t("routing.exclusion.cost-limit"); + case "cost-limit-unknown": + return t("routing.exclusion.cost-limit-unknown"); + case "cooldown": + return t("routing.exclusion.cooldown"); + case "unknown-health": + return t("routing.exclusion.unknown-health"); + case "unknown-quota": + return t("routing.exclusion.unknown-quota"); + case "unknown-price": + return t("routing.exclusion.unknown-price"); + default: + return t("routing.exclusion.other", { code }); + } +} + function isPlainObject(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -709,10 +754,28 @@ export default function RoutingProfiles({ value={draft.limits.maxEstimatedCostUsd} onChange={event => setDraft(current => current ? { ...current, - limits: { maxEstimatedCostUsd: event.target.value }, + limits: { ...current.limits, maxEstimatedCostUsd: event.target.value }, } : current)} /> +
@@ -817,6 +880,7 @@ export default function RoutingProfiles({ {t("routing.candidate")} {t("routing.eligible")} {t("routing.exclusions")} + {t("routing.costCap")} {t("routing.score")} @@ -828,7 +892,8 @@ export default function RoutingProfiles({ {index === dryRunResult.selectedIndex ? ` ✓ (${t("routing.selected")})` : ""} {candidate.eligible ? t("routing.yes") : t("routing.no")} - {candidate.exclusions.map(exclusion => exclusion.code).join(", ") || t("routing.none")} + {candidate.exclusions.map(exclusion => fmtExclusion(exclusion.code, t)).join(", ") || t("routing.none")} + {fmtCapOutcome(candidate.cost?.capOutcome, t, unavailable)} {candidate.score ? candidate.score.total.toFixed(3) : unavailable} ))} diff --git a/gui/src/routing-profile-editor-data.ts b/gui/src/routing-profile-editor-data.ts index 030825bf1b9..c178019ec06 100644 --- a/gui/src/routing-profile-editor-data.ts +++ b/gui/src/routing-profile-editor-data.ts @@ -1,4 +1,5 @@ export type UnknownEvidenceMode = "allow" | "penalize" | "exclude"; +export type UnknownCostCapMode = "allow" | "exclude"; export type OptionalBoolean = "" | "true" | "false"; export type RoutingProfileCandidate = { @@ -52,6 +53,7 @@ export type RoutingProfileDto = { }; limits: { maxEstimatedCostUsd?: number; + onUnknownCost?: UnknownCostCapMode; }; unknownEvidence: Record<"capability" | "health" | "quota" | "cost", UnknownEvidenceMode>; }; @@ -80,6 +82,7 @@ export type RoutingProfileDraft = { }; limits: { maxEstimatedCostUsd: string; + onUnknownCost: UnknownCostCapMode; }; unknownEvidence: Record<"capability" | "health" | "quota" | "cost", UnknownEvidenceMode>; }; @@ -134,7 +137,7 @@ export function newRoutingProfileDraft( encryptedCodexTasks: "", }, optimize: { ...DEFAULT_OPTIMIZE }, - limits: { maxEstimatedCostUsd: "" }, + limits: { maxEstimatedCostUsd: "", onUnknownCost: "allow" }, unknownEvidence: { ...DEFAULT_UNKNOWN_EVIDENCE }, }; } @@ -164,6 +167,7 @@ export function routingProfileDraftFromDto(profile: RoutingProfileDto): RoutingP }, limits: { maxEstimatedCostUsd: numberInput(profile.limits.maxEstimatedCostUsd), + onUnknownCost: profile.limits.onUnknownCost === "exclude" ? "exclude" : "allow", }, unknownEvidence: { ...profile.unknownEvidence }, }; @@ -209,6 +213,11 @@ export function routingProfilePutBody( encryptedCodexTasks: draftBoolean(draft.require.encryptedCodexTasks), }); const maxEstimatedCostUsd = optionalNumber(draft.limits.maxEstimatedCostUsd); + const onUnknownCost = draft.limits.onUnknownCost === "exclude" ? "exclude" as const : undefined; + const limits = compactRecord({ + maxEstimatedCostUsd, + onUnknownCost, + }); return { mode, @@ -227,9 +236,7 @@ export function routingProfilePutBody( cost: Number(draft.optimize.cost), quota: Number(draft.optimize.quota), }, - ...(maxEstimatedCostUsd !== undefined - ? { limits: { maxEstimatedCostUsd } } - : {}), + ...(Object.keys(limits).length > 0 ? { limits } : {}), unknownEvidence: { ...draft.unknownEvidence }, }, }; diff --git a/gui/tests/routing-profiles.test.tsx b/gui/tests/routing-profiles.test.tsx index 9262c937f5b..7250402b137 100644 --- a/gui/tests/routing-profiles.test.tsx +++ b/gui/tests/routing-profiles.test.tsx @@ -196,7 +196,7 @@ test("routing page loads profiles, analytics, and marks the dry-run selection", expect(dryRunBodies).toEqual([{ profile: "balanced", evidence: {} }]); expect(container.textContent).toContain("selected"); - expect(container.textContent).toContain("unknown-capability"); + expect(container.textContent).toContain("unknown capability"); expect(container.textContent).toContain("0.910"); } finally { await act(async () => { root.unmount(); }); diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index c1b3796d9c1..6554434f95b 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -304,14 +304,58 @@ export function evaluatePolicyProfile( const excludedByUnknown = unknown && profile.unknownEvidence.capability === "exclude"; const costLimit = profile.limits.maxEstimatedCostUsd; const estimatedCost = evidence.cost?.estimatedUsd; + const costEstimateKnown = typeof estimatedCost === "number" && Number.isFinite(estimatedCost); const overCostLimit = costLimit !== undefined - && typeof estimatedCost === "number" - && Number.isFinite(estimatedCost) - && estimatedCost > costLimit; + && costEstimateKnown + && estimatedCost! > costLimit; if (overCostLimit) { exclusions.push({ code: "cost-limit", detail: "maxEstimatedCostUsd" }); } - let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit; + // A cap can only be *proven* satisfied when the estimate is known. The live + // routing path often has no usage evidence yet, so the default stays + // "allow" to preserve the documented dry-run contract; operators who need a + // genuine hard ceiling opt into "exclude". Exclusions cover only the + // fail-closed path; the allow path stamps `cost.capOutcome` so operators + // can still distinguish "known under the cap" from "unknown cost allowed". + const unknownCostUnderCap = costLimit !== undefined && !costEstimateKnown; + const unknownCostBlocked = unknownCostUnderCap + && profile.limits.onUnknownCost === "exclude"; + if (unknownCostBlocked) { + exclusions.push({ code: "cost-limit-unknown", detail: "maxEstimatedCostUsd" }); + } + let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit && !unknownCostBlocked; + + // Trace/dry-run copy only: report the profile cap that was applied and the + // operator-visible outcome. Do not feed this copy into costScore() — that + // would silently change ranking when a caller supplied a different + // limitUsd (costScore uses limitUsd as its reference denominator). + let costForCandidate = evidence.cost; + if (costLimit !== undefined) { + const capOutcome = overCostLimit + ? "exceeded" as const + : unknownCostBlocked + ? "unknown-excluded" as const + : unknownCostUnderCap + ? "unknown-allowed" as const + : "satisfied" as const; + if (!costEstimateKnown) { + // Missing or non-finite estimates are the same unknown: never stamp + // Infinity/NaN into the trace, and always mark incomplete. + const { estimatedUsd: _nonFiniteOrMissing, ...rest } = evidence.cost ?? {}; + costForCandidate = { + ...rest, + incomplete: true, + limitUsd: costLimit, + capOutcome, + }; + } else { + costForCandidate = { + ...evidence.cost!, + limitUsd: costLimit, + capOutcome, + }; + } + } // Health scoring (RI-06): live hard cooldown is authoritative and // excludes; unknown health follows the profile's unknownEvidence policy; @@ -347,6 +391,8 @@ export function evaluatePolicyProfile( // Cost scoring (RI-08): the hard per-request ceiling was already checked // above; unknown cost follows the profile's unknownEvidence policy. + // Score against the caller's original evidence so trace stamping cannot + // move the costScore reference / ranking. const cost = evidence.cost; let costValue = cost ? costScore(cost) : null; if (costValue === null && profile.unknownEvidence.cost === "exclude") { @@ -394,7 +440,7 @@ export function evaluatePolicyProfile( ...(evidence.capability ? { capability: evidence.capability } : {}), ...(evidence.health ? { health: evidence.health } : {}), ...(evidence.quota ? { quota: evidence.quota } : {}), - ...(evidence.cost ? { cost: evidence.cost } : {}), + ...(costForCandidate ? { cost: costForCandidate } : {}), score, }; candidates.push(evaluated); diff --git a/src/routing/profile.ts b/src/routing/profile.ts index b35705954e6..80ca01ef9c1 100644 --- a/src/routing/profile.ts +++ b/src/routing/profile.ts @@ -9,6 +9,7 @@ import type { OcxConfig, OcxRoutingProfileConfig, OcxRoutingUnknownEvidenceMode, + OcxRoutingUnknownCostCapMode, } from "../types"; import { codexAccountNamespaceEntries } from "../codex/account-namespaces"; import { listComboIds, resolveComboId } from "../combos"; @@ -60,7 +61,7 @@ export interface NormalizedRoutingProfile { candidates: Array<{ provider: string; model: string }>; require: NormalizedRoutingProfileRequirements; optimize: { latency: number; health: number; cost: number; quota: number }; - limits: { maxEstimatedCostUsd?: number }; + limits: { maxEstimatedCostUsd?: number; onUnknownCost?: OcxRoutingUnknownCostCapMode }; unknownEvidence: Record<"capability" | "health" | "quota" | "cost", OcxRoutingUnknownEvidenceMode>; revision: string; } @@ -317,6 +318,11 @@ export function routingProfileIssues( || limits.maxEstimatedCostUsd < 0)) { issues.push({ path: ["limits", "maxEstimatedCostUsd"], message: "maxEstimatedCostUsd must be a non-negative number" }); } + if (limits.onUnknownCost !== undefined + && limits.onUnknownCost !== "allow" + && limits.onUnknownCost !== "exclude") { + issues.push({ path: ["limits", "onUnknownCost"], message: 'onUnknownCost must be "allow" or "exclude"' }); + } } } @@ -402,6 +408,9 @@ export function normalizeRoutingProfile(id: string, raw: OcxRoutingProfileConfig ...(raw.limits?.maxEstimatedCostUsd !== undefined ? { maxEstimatedCostUsd: raw.limits.maxEstimatedCostUsd } : {}), + ...(raw.limits?.onUnknownCost !== undefined + ? { onUnknownCost: raw.limits.onUnknownCost } + : {}), }, unknownEvidence: normalizedUnknownEvidence(raw), }; diff --git a/src/routing/trace.ts b/src/routing/trace.ts index daea4db9f2f..64cae992406 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -77,12 +77,29 @@ export interface RouteQuotaEvidence { source?: string; } +/** + * Stable wire outcome for `limits.maxEstimatedCostUsd` against this candidate's + * cost evidence. Present only when a cap is configured. Non-excluding outcomes + * (`satisfied`, `unknown-allowed`) must not be mirrored into `exclusions`. + */ +export type RouteCostCapOutcome = + | "satisfied" + | "exceeded" + | "unknown-allowed" + | "unknown-excluded"; + export interface RouteCostEvidence { estimatedUsd?: number; /** Stable wire code for the price source (e.g. "registry" | "expected"). */ priceSource?: string; incomplete?: boolean; limitUsd?: number; + /** + * Cap evaluation outcome when `limitUsd` / profile `maxEstimatedCostUsd` is set. + * Distinguishes "known under the cap" from "unknown cost allowed by policy" + * without treating the latter as an exclusion. + */ + capOutcome?: RouteCostCapOutcome; } export interface RouteScoreEvidence { @@ -528,6 +545,13 @@ function parseQuota(raw: unknown, caps: ParseCaps): RouteQuotaEvidence | undefin return out; } +const COST_CAP_OUTCOMES = new Set([ + "satisfied", + "exceeded", + "unknown-allowed", + "unknown-excluded", +]); + /** Whitelisted cost-evidence parse. */ function parseCost(raw: unknown, caps: ParseCaps): RouteCostEvidence | undefined { if (!isPlainRecord(raw)) return undefined; @@ -539,6 +563,11 @@ function parseCost(raw: unknown, caps: ParseCaps): RouteCostEvidence | undefined } if (typeof raw.incomplete === "boolean") out.incomplete = raw.incomplete; if (finiteNumber(raw.limitUsd)) out.limitUsd = raw.limitUsd; + if (out.limitUsd !== undefined + && typeof raw.capOutcome === "string" + && COST_CAP_OUTCOMES.has(raw.capOutcome as RouteCostCapOutcome)) { + out.capOutcome = raw.capOutcome as RouteCostCapOutcome; + } return Object.keys(out).length > 0 ? out : undefined; } diff --git a/src/types.ts b/src/types.ts index 49040ed6bcb..a194f35c2dc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -968,9 +968,24 @@ export interface OcxRoutingProfileOptimize { quota?: number; } +/** + * Policy for the hard cost ceiling when a candidate has no finite cost + * estimate. `"allow"` (default) preserves the documented dry-run contract: + * the cap only excludes evidence known to exceed it, and the candidate's + * `cost.capOutcome` is `"unknown-allowed"`. `"exclude"` makes the ceiling + * fail-closed (`cost-limit-unknown` + `capOutcome: "unknown-excluded"`). + */ +export type OcxRoutingUnknownCostCapMode = "allow" | "exclude"; + export interface OcxRoutingProfileLimits { /** Hard per-request estimated-cost ceiling in USD. */ maxEstimatedCostUsd?: number; + /** + * How `maxEstimatedCostUsd` behaves when the estimate is unknown. + * Defaults to `"allow"` (eligible + `cost.capOutcome: "unknown-allowed"`); + * opt in to `"exclude"` for a true hard ceiling. + */ + onUnknownCost?: OcxRoutingUnknownCostCapMode; } export interface OcxRoutingProfileUnknownEvidence { diff --git a/tests/cost-cap-unknown-evidence.test.ts b/tests/cost-cap-unknown-evidence.test.ts new file mode 100644 index 00000000000..e9444a8dfdb --- /dev/null +++ b/tests/cost-cap-unknown-evidence.test.ts @@ -0,0 +1,411 @@ +/** + * Regression coverage for issue #1181 — "Routing: define hard cost-cap behavior + * when runtime cost evidence is unknown". + * + * The hard ceiling `limits.maxEstimatedCostUsd` is documented as a hard + * per-request cap. In the live routing path it never fires by itself, because + * `router.ts` assembles cost evidence WITHOUT usage: + * + * costEvidenceForCandidate({ provider, model, limitUsd }) // no `usage` + * + * `costEvidenceForCandidate` then returns `{ limitUsd, incomplete: true }` + * with no `estimatedUsd`, and the evaluator's known-over-cap check requires a + * finite estimate. These tests pin both sides of `limits.onUnknownCost`: + * + * - default `"allow"` — candidate stays eligible and the trace stamps + * `cost.capOutcome: "unknown-allowed"` (not an exclusion) + * - opt-in `"exclude"` — ineligible with `cost-limit-unknown` and + * `capOutcome: "unknown-excluded"` + * + * Coverage includes the evaluator unit surface plus dry-run API and live + * `routeModel` acceptance paths so the operator-visible outcome stays + * consistent end-to-end. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { costEvidenceForCandidate } from "../src/routing/cost"; +import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import { normalizeRouteDecisionTrace } from "../src/routing/trace"; +import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; +import { handleManagementAPI } from "../src/server/management-api"; +import type { OcxConfig } from "../src/types"; +import { ManagementRequest } from "./helpers/management-auth"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-cost-cap-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (!testDir) return; + try { + rmSync(testDir, { recursive: true, force: true }); + } catch { + // Windows may keep a handle briefly after management/router I/O. + } +}); + +/** Mirrors the live routing path: a cap is configured, usage is NOT available. */ +function configWithCap(capUsd: number, overrides: Record = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "anthropic", + providers: { + anthropic: { + adapter: "anthropic", + baseUrl: "https://api.anthropic.com/v1", + apiKey: "kan", + models: ["claude-opus-5"], + }, + }, + routingProfiles: { + cost: { + candidates: [{ provider: "anthropic", model: "claude-opus-5" }], + optimize: { cost: 0.8 }, + limits: { maxEstimatedCostUsd: capUsd }, + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "allow" }, + ...overrides, + }, + }, + } as OcxConfig; +} + +function livePathEvidence(capUsd: number) { + return costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: capUsd, + }); +} + +describe("issue #1181 — hard cost cap under unknown evidence", () => { + test("default allow: live-path evidence stays eligible with unknown-allowed capOutcome", () => { + const evidence = livePathEvidence(0.000001); + expect(evidence.estimatedUsd).toBeUndefined(); + expect(evidence.incomplete).toBe(true); + expect(evidence.limitUsd).toBe(0.000001); + + const result = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: evidence, + }, + ]); + + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.exclusions.some(e => e.code === "cost-limit")).toBe(false); + expect(candidate.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(false); + expect(candidate.cost?.capOutcome).toBe("unknown-allowed"); + expect(result.selectedIndex).toBe(0); + expect(result.trace.candidates[0]?.cost?.capOutcome).toBe("unknown-allowed"); + expect(normalizeRouteDecisionTrace(result.trace)?.candidates[0]?.cost?.capOutcome) + .toBe("unknown-allowed"); + }); + + test("opt-in exclude: fail-closed cap excludes unknown-cost candidates", () => { + const evidence = livePathEvidence(0.000001); + const result = evaluatePolicyProfile( + configWithCap(0.000001, { limits: { maxEstimatedCostUsd: 0.000001, onUnknownCost: "exclude" } }), + "cost", + {}, + [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: evidence, + }, + ], + ); + + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(false); + expect(candidate.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(true); + expect(candidate.cost?.capOutcome).toBe("unknown-excluded"); + expect(result.selectedIndex).toBeNull(); + }); + + test("known estimate under the cap stamps satisfied without exclusions", () => { + const result = evaluatePolicyProfile(configWithCap(1), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: { estimatedUsd: 0.01, incomplete: false, limitUsd: 1 }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.exclusions).toEqual([]); + expect(candidate.cost?.capOutcome).toBe("satisfied"); + }); + + test("cap policy and unknownEvidence.cost are distinct mechanisms", () => { + const evidence = livePathEvidence(0.000001); + const scoringExcluded = evaluatePolicyProfile( + configWithCap(0.000001, { + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "exclude" }, + }), + "cost", + {}, + [{ provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }], + ); + + const codes = scoringExcluded.candidates[0]!.exclusions.map(e => e.code); + expect(codes).toContain("unknown-price"); + expect(codes).not.toContain("cost-limit-unknown"); + expect(scoringExcluded.candidates[0]!.eligible).toBe(false); + // Cap policy still stamps the allow-path outcome; scoring exclusion is separate. + expect(scoringExcluded.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); + }); + + test("no cap configured — onUnknownCost is inert", () => { + const evidence = costEvidenceForCandidate({ provider: "anthropic", model: "claude-opus-5" }); + const noCap = { + port: 10100, + defaultProvider: "anthropic", + providers: { + anthropic: { adapter: "anthropic", baseUrl: "https://api.anthropic.com/v1", apiKey: "kan", models: ["claude-opus-5"] }, + }, + routingProfiles: { + cost: { + candidates: [{ provider: "anthropic", model: "claude-opus-5" }], + optimize: { cost: 0.8 }, + limits: { onUnknownCost: "exclude" }, + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "allow" }, + }, + }, + } as unknown as OcxConfig; + + const result = evaluatePolicyProfile(noCap, "cost", {}, [ + { provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }, + ]); + + expect(result.candidates[0]!.eligible).toBe(true); + expect(result.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(false); + expect(result.candidates[0]!.cost?.capOutcome).toBeUndefined(); + }); + + test("trace stamps the profile cap even when evidence carries a different limitUsd", () => { + const result = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: { estimatedUsd: 0.01, incomplete: false, limitUsd: 1 }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(false); + expect(candidate.exclusions.some(e => e.code === "cost-limit")).toBe(true); + expect(candidate.cost?.limitUsd).toBe(0.000001); + expect(candidate.cost?.capOutcome).toBe("exceeded"); + expect(result.trace.candidates[0]?.cost?.limitUsd).toBe(0.000001); + }); + + test("trace limitUsd normalization does not change costScore for eligible candidates", () => { + // Profile cap 0.5; caller evidence carries limitUsd 1. Scoring must keep + // the caller's reference (1 - 0.4/1 = 0.6), not the stamped profile cap + // (1 - 0.4/0.5 = 0.2). + const result = evaluatePolicyProfile(configWithCap(0.5), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: { estimatedUsd: 0.4, incomplete: false, limitUsd: 1 }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.cost?.limitUsd).toBe(0.5); + expect(candidate.cost?.capOutcome).toBe("satisfied"); + expect(candidate.score?.components.cost).toBeCloseTo(0.6, 5); + }); + + test("synthesized cost evidence from missing input marks incomplete", () => { + const result = evaluatePolicyProfile(configWithCap(0.5), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.cost).toEqual({ + incomplete: true, + limitUsd: 0.5, + capOutcome: "unknown-allowed", + }); + }); + + test("non-finite estimatedUsd is stamped as incomplete unknown, not Infinity", () => { + const result = evaluatePolicyProfile(configWithCap(0.5), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: { estimatedUsd: Number.POSITIVE_INFINITY, incomplete: false, priceSource: "registry" }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.exclusions.some(e => e.code === "cost-limit")).toBe(false); + expect(candidate.cost?.estimatedUsd).toBeUndefined(); + expect(candidate.cost).toEqual({ + incomplete: true, + priceSource: "registry", + limitUsd: 0.5, + capOutcome: "unknown-allowed", + }); + expect(result.trace.candidates[0]?.cost?.estimatedUsd).toBeUndefined(); + }); + + test("normalizeRouteDecisionTrace drops capOutcome without a finite limitUsd", () => { + const dirty = { + version: 1 as const, + decisionId: "abcdef012345", + createdAt: 1, + requestedModel: "policy/cost", + routeKind: "policy" as const, + requirements: [], + candidates: [{ + provider: "anthropic", + model: "claude-opus-5", + eligible: true, + exclusions: [], + cost: { incomplete: true, capOutcome: "unknown-allowed" as const }, + }], + selected: { + candidateIndex: 0, + provider: "anthropic", + model: "claude-opus-5", + reason: "policy-selected", + }, + }; + const normalized = normalizeRouteDecisionTrace(dirty); + expect(normalized).not.toBeNull(); + expect(normalized?.candidates[0]?.cost?.capOutcome).toBeUndefined(); + expect(normalized?.candidates[0]?.cost?.incomplete).toBe(true); + }); + + test("explicit onUnknownCost allow matches omitted default", () => { + const evidence = livePathEvidence(0.000001); + const omitted = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }, + ]); + const explicit = evaluatePolicyProfile( + configWithCap(0.000001, { limits: { maxEstimatedCostUsd: 0.000001, onUnknownCost: "allow" } }), + "cost", + {}, + [{ provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }], + ); + + expect(omitted.candidates[0]!.eligible).toBe(true); + expect(explicit.candidates[0]!.eligible).toBe(true); + expect(omitted.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); + expect(explicit.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); + expect(omitted.selectedIndex).toBe(0); + expect(explicit.selectedIndex).toBe(0); + }); + + test("dry-run API and live routeModel share allow/exclude cap outcomes", async () => { + const allowConfig = configWithCap(0.000001); + const excludeConfig = configWithCap(0.000001, { + limits: { maxEstimatedCostUsd: 0.000001, onUnknownCost: "exclude" }, + }); + + const allowDry = await handleManagementAPI( + new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "cost" }), + }), + new URL("http://localhost/api/routing-profiles/dry-run"), + allowConfig, + { refreshCodexCatalog: async () => {} }, + ); + expect(allowDry!.status).toBe(200); + const allowBody = await allowDry!.json() as { + candidates: Array<{ + eligible: boolean; + exclusions: Array<{ code: string }>; + cost?: { capOutcome?: string }; + }>; + selectedIndex: number | null; + trace: { candidates: Array<{ cost?: { capOutcome?: string }; exclusions: Array<{ code: string }> }> }; + }; + expect(allowBody.candidates[0]!.eligible).toBe(true); + expect(allowBody.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); + expect(allowBody.trace.candidates[0]?.cost?.capOutcome).toBe("unknown-allowed"); + expect(allowBody.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(false); + + const allowLive = routeModel(allowConfig, "policy/cost"); + expect(allowLive.routeKind).toBe("policy"); + expect(allowLive.routeDecision?.candidates[0]?.eligible).toBe(true); + expect(allowLive.routeDecision?.candidates[0]?.cost?.capOutcome).toBe("unknown-allowed"); + expect(allowLive.routeDecision?.candidates[0]?.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(false); + + const excludeDry = await handleManagementAPI( + new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "cost" }), + }), + new URL("http://localhost/api/routing-profiles/dry-run"), + excludeConfig, + { refreshCodexCatalog: async () => {} }, + ); + expect(excludeDry!.status).toBe(200); + const excludeBody = await excludeDry!.json() as { + candidates: Array<{ + eligible: boolean; + exclusions: Array<{ code: string }>; + cost?: { capOutcome?: string }; + }>; + selectedIndex: number | null; + }; + expect(excludeBody.candidates[0]!.eligible).toBe(false); + expect(excludeBody.candidates[0]!.cost?.capOutcome).toBe("unknown-excluded"); + expect(excludeBody.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(true); + expect(excludeBody.selectedIndex).toBeNull(); + + expect(() => routeModel(excludeConfig, "policy/cost")).toThrow(NoEligiblePolicyCandidateError); + }); + + test("unknownEvidence.cost exclude stays distinguishable on the dry-run path", async () => { + const config = configWithCap(0.000001, { + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "exclude" }, + }); + const res = await handleManagementAPI( + new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "cost" }), + }), + new URL("http://localhost/api/routing-profiles/dry-run"), + config, + { refreshCodexCatalog: async () => {} }, + ); + expect(res!.status).toBe(200); + const body = await res!.json() as { + candidates: Array<{ exclusions: Array<{ code: string }>; cost?: { capOutcome?: string } }>; + }; + const codes = body.candidates[0]!.exclusions.map(e => e.code); + expect(codes).toContain("unknown-price"); + expect(codes).not.toContain("cost-limit-unknown"); + expect(body.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); + }); +});