From b534209bfa24466dbaf103108aff5d738db68ed6 Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Sun, 9 Aug 2026 01:28:14 -0700 Subject: [PATCH 1/8] fix(routing): define hard cost-cap behavior when cost evidence is unknown `limits.maxEstimatedCostUsd` is documented as a hard per-request ceiling, but it never fires on the live routing path. `evaluatePolicyProfile` only excludes a candidate when the estimate is a finite number (evaluator.ts), while `routeModel` assembles cost evidence without usage (router.ts), so `estimatedUsd` is always `undefined` live and any candidate silently passes a cap the operator configured as hard. The existing coverage passed only because it supplied `usage` directly, exercising a path production does not take. Fail-closed unconditionally is not safe either: with usage unwired, it would reject every live candidate whenever a cap is set, and it would change the documented dry-run contract. This adds an explicit, opt-in policy instead: limits.onUnknownCost: "allow" | "exclude" (default "allow") - "allow" preserves today's behavior and the documented contract exactly. - "exclude" makes the ceiling genuinely hard: a candidate whose cost cannot be proven under the cap is ineligible. Unknown-cost exclusions emit a distinct `cost-limit-unknown` code so a trace distinguishes "known above the cap" from "cost is unknown", which is the operator-facing distinction #1181 asks for. Kept separate from `unknownEvidence.cost`, which governs how an unknown-cost candidate is *scored* rather than whether the *ceiling* applies. The two mechanisms now emit distinct codes and are covered by a test asserting they stay distinguishable. Fixes #1181 Co-Authored-By: Claude Opus 5 (1M context) --- src/routing/evaluator.ts | 19 ++- src/routing/profile.ts | 11 +- src/types.ts | 14 ++ tests/cost-cap-unknown-evidence.test.ts | 216 ++++++++++++++++++++++++ 4 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 tests/cost-cap-unknown-evidence.test.ts diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index c1b3796d9c1..19465a1ea66 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -304,14 +304,25 @@ export function evaluatePolicyProfile( const excludedByUnknown = unknown && profile.unknownEvidence.capability === "exclude"; const costLimit = profile.limits.maxEstimatedCostUsd; const estimatedCost = evidence.cost?.estimatedUsd; + const costEstimateKnown = typeof estimatedCost === "number" && Number.isFinite(estimatedCost); const overCostLimit = costLimit !== undefined - && typeof estimatedCost === "number" - && Number.isFinite(estimatedCost) - && estimatedCost > costLimit; + && costEstimateKnown + && estimatedCost! > costLimit; if (overCostLimit) { exclusions.push({ code: "cost-limit", detail: "maxEstimatedCostUsd" }); } - let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit; + // A cap can only be *proven* satisfied when the estimate is known. The live + // routing path often has no usage evidence yet, so the default stays + // "allow" to preserve the documented dry-run contract; operators who need a + // genuine hard ceiling opt into "exclude". The distinct exclusion code lets + // a trace distinguish "known above the cap" from "cost is unknown". + const unknownCostBlocked = costLimit !== undefined + && !costEstimateKnown + && profile.limits.onUnknownCost === "exclude"; + if (unknownCostBlocked) { + exclusions.push({ code: "cost-limit-unknown", detail: "maxEstimatedCostUsd" }); + } + let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit && !unknownCostBlocked; // Health scoring (RI-06): live hard cooldown is authoritative and // excludes; unknown health follows the profile's unknownEvidence policy; diff --git a/src/routing/profile.ts b/src/routing/profile.ts index b35705954e6..80ca01ef9c1 100644 --- a/src/routing/profile.ts +++ b/src/routing/profile.ts @@ -9,6 +9,7 @@ import type { OcxConfig, OcxRoutingProfileConfig, OcxRoutingUnknownEvidenceMode, + OcxRoutingUnknownCostCapMode, } from "../types"; import { codexAccountNamespaceEntries } from "../codex/account-namespaces"; import { listComboIds, resolveComboId } from "../combos"; @@ -60,7 +61,7 @@ export interface NormalizedRoutingProfile { candidates: Array<{ provider: string; model: string }>; require: NormalizedRoutingProfileRequirements; optimize: { latency: number; health: number; cost: number; quota: number }; - limits: { maxEstimatedCostUsd?: number }; + limits: { maxEstimatedCostUsd?: number; onUnknownCost?: OcxRoutingUnknownCostCapMode }; unknownEvidence: Record<"capability" | "health" | "quota" | "cost", OcxRoutingUnknownEvidenceMode>; revision: string; } @@ -317,6 +318,11 @@ export function routingProfileIssues( || limits.maxEstimatedCostUsd < 0)) { issues.push({ path: ["limits", "maxEstimatedCostUsd"], message: "maxEstimatedCostUsd must be a non-negative number" }); } + if (limits.onUnknownCost !== undefined + && limits.onUnknownCost !== "allow" + && limits.onUnknownCost !== "exclude") { + issues.push({ path: ["limits", "onUnknownCost"], message: 'onUnknownCost must be "allow" or "exclude"' }); + } } } @@ -402,6 +408,9 @@ export function normalizeRoutingProfile(id: string, raw: OcxRoutingProfileConfig ...(raw.limits?.maxEstimatedCostUsd !== undefined ? { maxEstimatedCostUsd: raw.limits.maxEstimatedCostUsd } : {}), + ...(raw.limits?.onUnknownCost !== undefined + ? { onUnknownCost: raw.limits.onUnknownCost } + : {}), }, unknownEvidence: normalizedUnknownEvidence(raw), }; diff --git a/src/types.ts b/src/types.ts index 49040ed6bcb..1fd5a128baf 100644 --- a/src/types.ts +++ b/src/types.ts @@ -968,9 +968,23 @@ export interface OcxRoutingProfileOptimize { quota?: number; } +/** + * Policy for the hard cost ceiling when a candidate has no finite cost + * estimate. `"allow"` (default) preserves the documented dry-run contract: + * the cap only excludes evidence known to exceed it. `"exclude"` makes the + * ceiling fail-closed, so a candidate that cannot be proven under the cap is + * ineligible. + */ +export type OcxRoutingUnknownCostCapMode = "allow" | "exclude"; + export interface OcxRoutingProfileLimits { /** Hard per-request estimated-cost ceiling in USD. */ maxEstimatedCostUsd?: number; + /** + * How `maxEstimatedCostUsd` behaves when the estimate is unknown. + * Defaults to `"allow"`; opt in to `"exclude"` for a true hard ceiling. + */ + onUnknownCost?: OcxRoutingUnknownCostCapMode; } export interface OcxRoutingProfileUnknownEvidence { diff --git a/tests/cost-cap-unknown-evidence.test.ts b/tests/cost-cap-unknown-evidence.test.ts new file mode 100644 index 00000000000..e2e0e8c404a --- /dev/null +++ b/tests/cost-cap-unknown-evidence.test.ts @@ -0,0 +1,216 @@ +/** + * Reproduction for issue #1181 — "Routing: define hard cost-cap behavior when + * runtime cost evidence is unknown". + * + * The hard ceiling `limits.maxEstimatedCostUsd` is documented as a hard + * per-request cap. In the live routing path it never fires, because + * `router.ts` assembles cost evidence WITHOUT usage: + * + * costEvidenceForCandidate({ provider, model, limitUsd }) // no `usage` + * + * `costEvidenceForCandidate` then returns `{ limitUsd, incomplete: true }` + * with no `estimatedUsd`, and the evaluator's cap check + * (evaluator.ts:307-310) requires `typeof estimatedCost === "number"`, so an + * unknown estimate silently passes a cap the operator configured as hard. + * + * The existing test in cost-scoring.test.ts only exercises the cap with + * `usage: USAGE` supplied — i.e. on a code path production never takes. + * + * These tests pin both sides of `limits.onUnknownCost`: + * - the default `"allow"`, which preserves the documented dry-run contract + * and lets an unprovable candidate through, and + * - the opt-in `"exclude"`, which makes the ceiling genuinely hard and + * reports the distinct `cost-limit-unknown` exclusion. + * + * The first case was originally written as a failing reproduction before the + * evaluator change landed; it is retained to keep the fail-open default + * asserted rather than assumed. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { costEvidenceForCandidate } from "../src/routing/cost"; +import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-cost-cap-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +/** Mirrors the live routing path: a cap is configured, usage is NOT available. */ +function configWithCap(capUsd: number, overrides: Record = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "anthropic", + providers: { + anthropic: { + adapter: "anthropic", + baseUrl: "https://api.anthropic.com/v1", + apiKey: "kan", + models: ["claude-opus-5"], + }, + }, + routingProfiles: { + cost: { + candidates: [{ provider: "anthropic", model: "claude-opus-5" }], + optimize: { cost: 0.8 }, + limits: { maxEstimatedCostUsd: capUsd }, + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "allow" }, + ...overrides, + }, + }, + } as OcxConfig; +} + +describe("issue #1181 — hard cost cap under unknown evidence", () => { + test("default allow: live-path evidence carries no estimate, so the cap does not fire", async () => { + // Exactly how router.ts:515-519 builds it — no `usage` argument. + const evidence = costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: 0.000001, // an absurdly low cap; nothing should realistically pass + }); + + // The evidence is explicitly unknown, and correctly so. + expect(evidence.estimatedUsd).toBeUndefined(); + expect(evidence.incomplete).toBe(true); + expect(evidence.limitUsd).toBe(0.000001); + + const result = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: evidence, + }, + ]); + + // Current behaviour: the candidate is eligible and selected despite a cap + // of $0.000001. No `cost-limit` exclusion is recorded, and nothing in the + // trace distinguishes "known below cap" from "cost unknown". + expect(result.candidates[0]!.eligible).toBe(true); + expect( + result.candidates[0]!.exclusions.some(e => e.code === "cost-limit"), + ).toBe(false); + expect(result.selectedIndex).toBe(0); + }); + + test("opt-in exclude: fail-closed cap excludes unknown-cost candidates", async () => { + const evidence = costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: 0.000001, + }); + + // Proposed opt-in policy: limits.onUnknownCost = "exclude". + const result = evaluatePolicyProfile( + configWithCap(0.000001, { limits: { maxEstimatedCostUsd: 0.000001, onUnknownCost: "exclude" } }), + "cost", + {}, + [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: evidence, + }, + ], + ); + + expect(result.candidates[0]!.eligible).toBe(false); + // Distinct code so operators can tell "over a known cap" from "cost unknown". + expect( + result.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown"), + ).toBe(true); + expect(result.selectedIndex).toBeNull(); + }); + + test("cap policy and unknownEvidence.cost are distinct mechanisms", async () => { + // unknownEvidence.cost governs SCORING of an unknown-cost candidate; + // limits.onUnknownCost governs whether the hard CEILING applies to it. + // They must produce distinct exclusion codes so a trace stays diagnosable. + const evidence = costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: 0.000001, + }); + + const scoringExcluded = evaluatePolicyProfile( + configWithCap(0.000001, { + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "exclude" }, + }), + "cost", + {}, + [{ provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }], + ); + + const codes = scoringExcluded.candidates[0]!.exclusions.map(e => e.code); + expect(codes).toContain("unknown-price"); + expect(codes).not.toContain("cost-limit-unknown"); + expect(scoringExcluded.candidates[0]!.eligible).toBe(false); + }); + + test("no cap configured — onUnknownCost is inert", async () => { + const evidence = costEvidenceForCandidate({ provider: "anthropic", model: "claude-opus-5" }); + const noCap = { + port: 10100, + defaultProvider: "anthropic", + providers: { + anthropic: { adapter: "anthropic", baseUrl: "https://api.anthropic.com/v1", apiKey: "kan", models: ["claude-opus-5"] }, + }, + routingProfiles: { + cost: { + candidates: [{ provider: "anthropic", model: "claude-opus-5" }], + optimize: { cost: 0.8 }, + limits: { onUnknownCost: "exclude" }, // no maxEstimatedCostUsd + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "allow" }, + }, + }, + } as unknown as OcxConfig; + + const result = evaluatePolicyProfile(noCap, "cost", {}, [ + { provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }, + ]); + + // Without a ceiling there is nothing to fail closed against. + expect(result.candidates[0]!.eligible).toBe(true); + expect( + result.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown"), + ).toBe(false); + }); + + test("default stays allow — the documented contract is unchanged", async () => { + const evidence = costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: 0.000001, + }); + + // No onUnknownCost configured → must behave exactly as today (fail-open), + // so existing deployments do not lose every live route on upgrade. + const result = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: evidence, + }, + ]); + + expect(result.candidates[0]!.eligible).toBe(true); + expect(result.selectedIndex).toBe(0); + }); +}); From 0ed29aa055a6999f2d585dbdca81991fdf4eebbe Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:23:39 +0200 Subject: [PATCH 2/8] fix(routing): surface unknown cost-cap outcomes on allow path Default onUnknownCost stays fail-open, but stamp cost.capOutcome so dry-run and live traces distinguish known-under-cap from unknown-allowed; document and expose the policy in the profile editor. --- .../docs/guides/routing-profile-editor.md | 6 +- .../docs/reference/configuration/routing.md | 9 +- gui/src/i18n/en.ts | 1 + gui/src/pages/RoutingProfiles.tsx | 25 +- gui/src/routing-profile-editor-data.ts | 15 +- src/routing/evaluator.ts | 31 ++- src/routing/trace.ts | 27 ++ src/types.ts | 9 +- tests/cost-cap-unknown-evidence.test.ts | 250 ++++++++++++------ 9 files changed, 274 insertions(+), 99 deletions(-) diff --git a/docs-site/src/content/docs/guides/routing-profile-editor.md b/docs-site/src/content/docs/guides/routing-profile-editor.md index 2d48bcd7b18..58ac8901283 100644 --- a/docs-site/src/content/docs/guides/routing-profile-editor.md +++ b/docs-site/src/content/docs/guides/routing-profile-editor.md @@ -11,7 +11,7 @@ The **Models → Routing** tab in the OpenCodex dashboard can manage `config.rou 2. Select **Create profile**. 3. Enter an `id`. The canonical model id is `policy/`. 4. Add one or more explicit provider/model candidates. -5. Configure optional requirements, scoring weights, cost limits, and unknown-evidence behavior. +5. Configure optional requirements, scoring weights, cost limits (`maxEstimatedCostUsd`, optional `onUnknownCost`), and unknown-evidence behavior. 6. Save the profile. Profile ids are immutable after creation. To use a different id, create a new profile and remove the old one after updating callers. @@ -28,6 +28,8 @@ The dashboard sends the same profile object used by `config.routingProfiles` to A successful save persists the profile through the normal config writer, reconciles live state, and refreshes the model catalog. Validation failures leave the previous configuration unchanged and are shown in the editor. +`limits.onUnknownCost` defaults to `"allow"`: an unknown cost estimate stays eligible, and dry-run / live route-decision traces stamp `cost.capOutcome: "unknown-allowed"` so operators can tell the cap was not proven. Set `"exclude"` when the ceiling must fail closed (`cost-limit-unknown`). This is separate from `unknownEvidence.cost`, which only affects scoring. + ## Dry-run a saved profile Select a saved profile and use **Dry-run evaluation** to add request evidence such as context-window size, tool use, image input, or structured output. Dry-run evaluates eligibility and scoring but never sends an upstream model request. @@ -57,7 +59,7 @@ Example save payload: ], "require": { "tools": true, "minContextWindow": 128000 }, "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.1, "quota": 0.1 }, - "limits": { "maxEstimatedCostUsd": 0.5 }, + "limits": { "maxEstimatedCostUsd": 0.5, "onUnknownCost": "allow" }, "unknownEvidence": { "capability": "exclude", "health": "penalize", diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 18ca8a36905..49deee248ac 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -114,7 +114,7 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c | `alias?` | `string` | — | Optional public model id in place of `policy/`. | | `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). | | `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights, normalized deterministically. `health`, `quota`, and `cost` have score dimensions; the configured-priority share is `1 - health - quota - cost` (default 0.55), and `latency` folds into that priority share rather than scoring independently. | -| `limits?` | object | — | Hard limits, e.g. `maxEstimatedCostUsd` (enforced by the dry-run evaluator when candidate cost evidence is known). | +| `limits?` | object | — | Hard limits. `maxEstimatedCostUsd` excludes a candidate when its estimated cost is known and above the cap. `onUnknownCost` (`"allow"` default, or `"exclude"`) controls unknown estimates: allow keeps the candidate eligible and stamps `cost.capOutcome: "unknown-allowed"` on dry-run/live traces; exclude emits `cost-limit-unknown` and `capOutcome: "unknown-excluded"`. Separate from `unknownEvidence.cost`, which only affects scoring. | | `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | `require` supports: `minContextWindow` (positive integer), `minQuotaHeadroom` (0..1 fraction), @@ -145,7 +145,7 @@ candidate evidence is provided through the API (`POST /api/routing-profiles/dry- ], "require": { "tools": true, "minContextWindow": 128000 }, "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.10, "quota": 0.10 }, - "limits": { "maxEstimatedCostUsd": 0.50 }, + "limits": { "maxEstimatedCostUsd": 0.50, "onUnknownCost": "allow" }, "unknownEvidence": { "capability": "exclude", "health": "penalize", @@ -185,7 +185,10 @@ Both are virtual namespaces with aliases and collision validation; they differ i is chosen. Profile scoring combines the configured-priority component with the health (RI-06), quota (RI-07), and cost (RI-08) score dimensions where evidence is present; the `latency` weight folds into the priority share rather than scoring independently. Cost is also enforced through the -`limits.maxEstimatedCostUsd` cap (a candidate whose estimated cost exceeds the cap is excluded). +`limits.maxEstimatedCostUsd` cap: a candidate whose estimated cost is known and exceeds the cap is +excluded (`cost-limit`). When the estimate is unknown, the default `limits.onUnknownCost: "allow"` +keeps the candidate eligible and records `cost.capOutcome: "unknown-allowed"` on the route-decision +trace; set `onUnknownCost: "exclude"` for a fail-closed ceiling (`cost-limit-unknown`). Per-request route-decision traces are recorded when a policy profile executes. ### Catalog eligibility diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 7bdebfeb07c..6565140a30c 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -89,6 +89,7 @@ export const en = { "routing.candidate": "Candidate", "routing.eligible": "Eligible", "routing.exclusions": "Exclusions", + "routing.costCap": "Cost cap", "routing.score": "Score", "routing.selected": "selected", "routing.yes": "yes", diff --git a/gui/src/pages/RoutingProfiles.tsx b/gui/src/pages/RoutingProfiles.tsx index 56543c89196..f02bfa2a332 100644 --- a/gui/src/pages/RoutingProfiles.tsx +++ b/gui/src/pages/RoutingProfiles.tsx @@ -11,6 +11,7 @@ import { type OptionalBoolean, type RoutingProfileDraft, type RoutingProfileDto, + type UnknownCostCapMode, type UnknownEvidenceMode, } from "../routing-profile-editor-data"; import { readJsonIfOk } from "../fetch-json"; @@ -23,6 +24,7 @@ type DryRunCandidate = { eligible: boolean; exclusions: Array<{ code: string; detail?: string }>; score?: { total: number; components: Record }; + cost?: { capOutcome?: string; estimatedUsd?: number; incomplete?: boolean; limitUsd?: number }; }; type Analytics = { @@ -69,6 +71,7 @@ const NUMERIC_REQUIREMENTS = Object.keys(NUMERIC_REQUIREMENT_SPEC) as Array setDraft(current => current ? { ...current, - limits: { maxEstimatedCostUsd: event.target.value }, + limits: { ...current.limits, maxEstimatedCostUsd: event.target.value }, } : current)} /> +
@@ -817,6 +838,7 @@ export default function RoutingProfiles({ {t("routing.candidate")} {t("routing.eligible")} {t("routing.exclusions")} + {t("routing.costCap")} {t("routing.score")} @@ -829,6 +851,7 @@ export default function RoutingProfiles({ {candidate.eligible ? t("routing.yes") : t("routing.no")} {candidate.exclusions.map(exclusion => exclusion.code).join(", ") || t("routing.none")} + {candidate.cost?.capOutcome ?? t("routing.none")} {candidate.score ? candidate.score.total.toFixed(3) : unavailable} ))} diff --git a/gui/src/routing-profile-editor-data.ts b/gui/src/routing-profile-editor-data.ts index 030825bf1b9..c178019ec06 100644 --- a/gui/src/routing-profile-editor-data.ts +++ b/gui/src/routing-profile-editor-data.ts @@ -1,4 +1,5 @@ export type UnknownEvidenceMode = "allow" | "penalize" | "exclude"; +export type UnknownCostCapMode = "allow" | "exclude"; export type OptionalBoolean = "" | "true" | "false"; export type RoutingProfileCandidate = { @@ -52,6 +53,7 @@ export type RoutingProfileDto = { }; limits: { maxEstimatedCostUsd?: number; + onUnknownCost?: UnknownCostCapMode; }; unknownEvidence: Record<"capability" | "health" | "quota" | "cost", UnknownEvidenceMode>; }; @@ -80,6 +82,7 @@ export type RoutingProfileDraft = { }; limits: { maxEstimatedCostUsd: string; + onUnknownCost: UnknownCostCapMode; }; unknownEvidence: Record<"capability" | "health" | "quota" | "cost", UnknownEvidenceMode>; }; @@ -134,7 +137,7 @@ export function newRoutingProfileDraft( encryptedCodexTasks: "", }, optimize: { ...DEFAULT_OPTIMIZE }, - limits: { maxEstimatedCostUsd: "" }, + limits: { maxEstimatedCostUsd: "", onUnknownCost: "allow" }, unknownEvidence: { ...DEFAULT_UNKNOWN_EVIDENCE }, }; } @@ -164,6 +167,7 @@ export function routingProfileDraftFromDto(profile: RoutingProfileDto): RoutingP }, limits: { maxEstimatedCostUsd: numberInput(profile.limits.maxEstimatedCostUsd), + onUnknownCost: profile.limits.onUnknownCost === "exclude" ? "exclude" : "allow", }, unknownEvidence: { ...profile.unknownEvidence }, }; @@ -209,6 +213,11 @@ export function routingProfilePutBody( encryptedCodexTasks: draftBoolean(draft.require.encryptedCodexTasks), }); const maxEstimatedCostUsd = optionalNumber(draft.limits.maxEstimatedCostUsd); + const onUnknownCost = draft.limits.onUnknownCost === "exclude" ? "exclude" as const : undefined; + const limits = compactRecord({ + maxEstimatedCostUsd, + onUnknownCost, + }); return { mode, @@ -227,9 +236,7 @@ export function routingProfilePutBody( cost: Number(draft.optimize.cost), quota: Number(draft.optimize.quota), }, - ...(maxEstimatedCostUsd !== undefined - ? { limits: { maxEstimatedCostUsd } } - : {}), + ...(Object.keys(limits).length > 0 ? { limits } : {}), unknownEvidence: { ...draft.unknownEvidence }, }, }; diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 19465a1ea66..30a1e4f6a75 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -314,16 +314,35 @@ export function evaluatePolicyProfile( // A cap can only be *proven* satisfied when the estimate is known. The live // routing path often has no usage evidence yet, so the default stays // "allow" to preserve the documented dry-run contract; operators who need a - // genuine hard ceiling opt into "exclude". The distinct exclusion code lets - // a trace distinguish "known above the cap" from "cost is unknown". - const unknownCostBlocked = costLimit !== undefined - && !costEstimateKnown + // genuine hard ceiling opt into "exclude". Exclusions cover only the + // fail-closed path; the allow path stamps `cost.capOutcome` so operators + // can still distinguish "known under the cap" from "unknown cost allowed". + const unknownCostUnderCap = costLimit !== undefined && !costEstimateKnown; + const unknownCostBlocked = unknownCostUnderCap && profile.limits.onUnknownCost === "exclude"; if (unknownCostBlocked) { exclusions.push({ code: "cost-limit-unknown", detail: "maxEstimatedCostUsd" }); } let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit && !unknownCostBlocked; + // Stamp a copy of cost evidence with the cap outcome so traces/dry-run + // carry operator-visible status without mutating caller-supplied objects. + let costForCandidate = evidence.cost; + if (costLimit !== undefined) { + const capOutcome = overCostLimit + ? "exceeded" as const + : unknownCostBlocked + ? "unknown-excluded" as const + : unknownCostUnderCap + ? "unknown-allowed" as const + : "satisfied" as const; + costForCandidate = { + ...(evidence.cost ?? {}), + limitUsd: evidence.cost?.limitUsd ?? costLimit, + capOutcome, + }; + } + // Health scoring (RI-06): live hard cooldown is authoritative and // excludes; unknown health follows the profile's unknownEvidence policy; // historical health never overrides explicit ineligibility. @@ -358,7 +377,7 @@ export function evaluatePolicyProfile( // Cost scoring (RI-08): the hard per-request ceiling was already checked // above; unknown cost follows the profile's unknownEvidence policy. - const cost = evidence.cost; + const cost = costForCandidate; let costValue = cost ? costScore(cost) : null; if (costValue === null && profile.unknownEvidence.cost === "exclude") { exclusions.push({ code: "unknown-price" }); @@ -405,7 +424,7 @@ export function evaluatePolicyProfile( ...(evidence.capability ? { capability: evidence.capability } : {}), ...(evidence.health ? { health: evidence.health } : {}), ...(evidence.quota ? { quota: evidence.quota } : {}), - ...(evidence.cost ? { cost: evidence.cost } : {}), + ...(costForCandidate ? { cost: costForCandidate } : {}), score, }; candidates.push(evaluated); diff --git a/src/routing/trace.ts b/src/routing/trace.ts index daea4db9f2f..6497428a065 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -77,12 +77,29 @@ export interface RouteQuotaEvidence { source?: string; } +/** + * Stable wire outcome for `limits.maxEstimatedCostUsd` against this candidate's + * cost evidence. Present only when a cap is configured. Non-excluding outcomes + * (`satisfied`, `unknown-allowed`) must not be mirrored into `exclusions`. + */ +export type RouteCostCapOutcome = + | "satisfied" + | "exceeded" + | "unknown-allowed" + | "unknown-excluded"; + export interface RouteCostEvidence { estimatedUsd?: number; /** Stable wire code for the price source (e.g. "registry" | "expected"). */ priceSource?: string; incomplete?: boolean; limitUsd?: number; + /** + * Cap evaluation outcome when `limitUsd` / profile `maxEstimatedCostUsd` is set. + * Distinguishes "known under the cap" from "unknown cost allowed by policy" + * without treating the latter as an exclusion. + */ + capOutcome?: RouteCostCapOutcome; } export interface RouteScoreEvidence { @@ -528,6 +545,13 @@ function parseQuota(raw: unknown, caps: ParseCaps): RouteQuotaEvidence | undefin return out; } +const COST_CAP_OUTCOMES = new Set([ + "satisfied", + "exceeded", + "unknown-allowed", + "unknown-excluded", +]); + /** Whitelisted cost-evidence parse. */ function parseCost(raw: unknown, caps: ParseCaps): RouteCostEvidence | undefined { if (!isPlainRecord(raw)) return undefined; @@ -539,6 +563,9 @@ function parseCost(raw: unknown, caps: ParseCaps): RouteCostEvidence | undefined } if (typeof raw.incomplete === "boolean") out.incomplete = raw.incomplete; if (finiteNumber(raw.limitUsd)) out.limitUsd = raw.limitUsd; + if (typeof raw.capOutcome === "string" && COST_CAP_OUTCOMES.has(raw.capOutcome as RouteCostCapOutcome)) { + out.capOutcome = raw.capOutcome as RouteCostCapOutcome; + } return Object.keys(out).length > 0 ? out : undefined; } diff --git a/src/types.ts b/src/types.ts index 1fd5a128baf..a194f35c2dc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -971,9 +971,9 @@ export interface OcxRoutingProfileOptimize { /** * Policy for the hard cost ceiling when a candidate has no finite cost * estimate. `"allow"` (default) preserves the documented dry-run contract: - * the cap only excludes evidence known to exceed it. `"exclude"` makes the - * ceiling fail-closed, so a candidate that cannot be proven under the cap is - * ineligible. + * the cap only excludes evidence known to exceed it, and the candidate's + * `cost.capOutcome` is `"unknown-allowed"`. `"exclude"` makes the ceiling + * fail-closed (`cost-limit-unknown` + `capOutcome: "unknown-excluded"`). */ export type OcxRoutingUnknownCostCapMode = "allow" | "exclude"; @@ -982,7 +982,8 @@ export interface OcxRoutingProfileLimits { maxEstimatedCostUsd?: number; /** * How `maxEstimatedCostUsd` behaves when the estimate is unknown. - * Defaults to `"allow"`; opt in to `"exclude"` for a true hard ceiling. + * Defaults to `"allow"` (eligible + `cost.capOutcome: "unknown-allowed"`); + * opt in to `"exclude"` for a true hard ceiling. */ onUnknownCost?: OcxRoutingUnknownCostCapMode; } diff --git a/tests/cost-cap-unknown-evidence.test.ts b/tests/cost-cap-unknown-evidence.test.ts index e2e0e8c404a..28f6d1ce96d 100644 --- a/tests/cost-cap-unknown-evidence.test.ts +++ b/tests/cost-cap-unknown-evidence.test.ts @@ -1,30 +1,25 @@ /** - * Reproduction for issue #1181 — "Routing: define hard cost-cap behavior when - * runtime cost evidence is unknown". + * Regression coverage for issue #1181 — "Routing: define hard cost-cap behavior + * when runtime cost evidence is unknown". * * The hard ceiling `limits.maxEstimatedCostUsd` is documented as a hard - * per-request cap. In the live routing path it never fires, because + * per-request cap. In the live routing path it never fires by itself, because * `router.ts` assembles cost evidence WITHOUT usage: * * costEvidenceForCandidate({ provider, model, limitUsd }) // no `usage` * * `costEvidenceForCandidate` then returns `{ limitUsd, incomplete: true }` - * with no `estimatedUsd`, and the evaluator's cap check - * (evaluator.ts:307-310) requires `typeof estimatedCost === "number"`, so an - * unknown estimate silently passes a cap the operator configured as hard. + * with no `estimatedUsd`, and the evaluator's known-over-cap check requires a + * finite estimate. These tests pin both sides of `limits.onUnknownCost`: * - * The existing test in cost-scoring.test.ts only exercises the cap with - * `usage: USAGE` supplied — i.e. on a code path production never takes. + * - default `"allow"` — candidate stays eligible and the trace stamps + * `cost.capOutcome: "unknown-allowed"` (not an exclusion) + * - opt-in `"exclude"` — ineligible with `cost-limit-unknown` and + * `capOutcome: "unknown-excluded"` * - * These tests pin both sides of `limits.onUnknownCost`: - * - the default `"allow"`, which preserves the documented dry-run contract - * and lets an unprovable candidate through, and - * - the opt-in `"exclude"`, which makes the ceiling genuinely hard and - * reports the distinct `cost-limit-unknown` exclusion. - * - * The first case was originally written as a failing reproduction before the - * evaluator change landed; it is retained to keep the fail-open default - * asserted rather than assumed. + * Coverage includes the evaluator unit surface plus dry-run API and live + * `routeModel` acceptance paths so the operator-visible outcome stays + * consistent end-to-end. */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; @@ -33,7 +28,11 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { costEvidenceForCandidate } from "../src/routing/cost"; import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import { normalizeRouteDecisionTrace } from "../src/routing/trace"; +import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; +import { handleManagementAPI } from "../src/server/management-api"; import type { OcxConfig } from "../src/types"; +import { ManagementRequest } from "./helpers/management-auth"; let testDir = ""; let previousHome: string | undefined; @@ -47,7 +46,12 @@ beforeEach(() => { afterEach(() => { if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; - if (testDir) rmSync(testDir, { recursive: true, force: true }); + if (!testDir) return; + try { + rmSync(testDir, { recursive: true, force: true }); + } catch { + // Windows may keep a handle briefly after management/router I/O. + } }); /** Mirrors the live routing path: a cap is configured, usage is NOT available. */ @@ -75,16 +79,17 @@ function configWithCap(capUsd: number, overrides: Record = {}): } as OcxConfig; } -describe("issue #1181 — hard cost cap under unknown evidence", () => { - test("default allow: live-path evidence carries no estimate, so the cap does not fire", async () => { - // Exactly how router.ts:515-519 builds it — no `usage` argument. - const evidence = costEvidenceForCandidate({ - provider: "anthropic", - model: "claude-opus-5", - limitUsd: 0.000001, // an absurdly low cap; nothing should realistically pass - }); +function livePathEvidence(capUsd: number) { + return costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: capUsd, + }); +} - // The evidence is explicitly unknown, and correctly so. +describe("issue #1181 — hard cost cap under unknown evidence", () => { + test("default allow: live-path evidence stays eligible with unknown-allowed capOutcome", () => { + const evidence = livePathEvidence(0.000001); expect(evidence.estimatedUsd).toBeUndefined(); expect(evidence.incomplete).toBe(true); expect(evidence.limitUsd).toBe(0.000001); @@ -98,24 +103,19 @@ describe("issue #1181 — hard cost cap under unknown evidence", () => { }, ]); - // Current behaviour: the candidate is eligible and selected despite a cap - // of $0.000001. No `cost-limit` exclusion is recorded, and nothing in the - // trace distinguishes "known below cap" from "cost unknown". - expect(result.candidates[0]!.eligible).toBe(true); - expect( - result.candidates[0]!.exclusions.some(e => e.code === "cost-limit"), - ).toBe(false); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.exclusions.some(e => e.code === "cost-limit")).toBe(false); + expect(candidate.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(false); + expect(candidate.cost?.capOutcome).toBe("unknown-allowed"); expect(result.selectedIndex).toBe(0); + expect(result.trace.candidates[0]?.cost?.capOutcome).toBe("unknown-allowed"); + expect(normalizeRouteDecisionTrace(result.trace)?.candidates[0]?.cost?.capOutcome) + .toBe("unknown-allowed"); }); - test("opt-in exclude: fail-closed cap excludes unknown-cost candidates", async () => { - const evidence = costEvidenceForCandidate({ - provider: "anthropic", - model: "claude-opus-5", - limitUsd: 0.000001, - }); - - // Proposed opt-in policy: limits.onUnknownCost = "exclude". + test("opt-in exclude: fail-closed cap excludes unknown-cost candidates", () => { + const evidence = livePathEvidence(0.000001); const result = evaluatePolicyProfile( configWithCap(0.000001, { limits: { maxEstimatedCostUsd: 0.000001, onUnknownCost: "exclude" } }), "cost", @@ -130,24 +130,30 @@ describe("issue #1181 — hard cost cap under unknown evidence", () => { ], ); - expect(result.candidates[0]!.eligible).toBe(false); - // Distinct code so operators can tell "over a known cap" from "cost unknown". - expect( - result.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown"), - ).toBe(true); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(false); + expect(candidate.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(true); + expect(candidate.cost?.capOutcome).toBe("unknown-excluded"); expect(result.selectedIndex).toBeNull(); }); - test("cap policy and unknownEvidence.cost are distinct mechanisms", async () => { - // unknownEvidence.cost governs SCORING of an unknown-cost candidate; - // limits.onUnknownCost governs whether the hard CEILING applies to it. - // They must produce distinct exclusion codes so a trace stays diagnosable. - const evidence = costEvidenceForCandidate({ - provider: "anthropic", - model: "claude-opus-5", - limitUsd: 0.000001, - }); + test("known estimate under the cap stamps satisfied without exclusions", () => { + const result = evaluatePolicyProfile(configWithCap(1), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: { estimatedUsd: 0.01, incomplete: false, limitUsd: 1 }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.exclusions).toEqual([]); + expect(candidate.cost?.capOutcome).toBe("satisfied"); + }); + test("cap policy and unknownEvidence.cost are distinct mechanisms", () => { + const evidence = livePathEvidence(0.000001); const scoringExcluded = evaluatePolicyProfile( configWithCap(0.000001, { unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "exclude" }, @@ -161,9 +167,11 @@ describe("issue #1181 — hard cost cap under unknown evidence", () => { expect(codes).toContain("unknown-price"); expect(codes).not.toContain("cost-limit-unknown"); expect(scoringExcluded.candidates[0]!.eligible).toBe(false); + // Cap policy still stamps the allow-path outcome; scoring exclusion is separate. + expect(scoringExcluded.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); }); - test("no cap configured — onUnknownCost is inert", async () => { + test("no cap configured — onUnknownCost is inert", () => { const evidence = costEvidenceForCandidate({ provider: "anthropic", model: "claude-opus-5" }); const noCap = { port: 10100, @@ -175,7 +183,7 @@ describe("issue #1181 — hard cost cap under unknown evidence", () => { cost: { candidates: [{ provider: "anthropic", model: "claude-opus-5" }], optimize: { cost: 0.8 }, - limits: { onUnknownCost: "exclude" }, // no maxEstimatedCostUsd + limits: { onUnknownCost: "exclude" }, unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "allow" }, }, }, @@ -185,32 +193,116 @@ describe("issue #1181 — hard cost cap under unknown evidence", () => { { provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }, ]); - // Without a ceiling there is nothing to fail closed against. expect(result.candidates[0]!.eligible).toBe(true); - expect( - result.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown"), - ).toBe(false); + expect(result.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(false); + expect(result.candidates[0]!.cost?.capOutcome).toBeUndefined(); }); - test("default stays allow — the documented contract is unchanged", async () => { - const evidence = costEvidenceForCandidate({ - provider: "anthropic", - model: "claude-opus-5", - limitUsd: 0.000001, + test("explicit onUnknownCost allow matches omitted default", () => { + const evidence = livePathEvidence(0.000001); + const omitted = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }, + ]); + const explicit = evaluatePolicyProfile( + configWithCap(0.000001, { limits: { maxEstimatedCostUsd: 0.000001, onUnknownCost: "allow" } }), + "cost", + {}, + [{ provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }], + ); + + expect(omitted.candidates[0]!.eligible).toBe(true); + expect(explicit.candidates[0]!.eligible).toBe(true); + expect(omitted.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); + expect(explicit.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); + expect(omitted.selectedIndex).toBe(0); + expect(explicit.selectedIndex).toBe(0); + }); + + test("dry-run API and live routeModel share allow/exclude cap outcomes", async () => { + const allowConfig = configWithCap(0.000001); + const excludeConfig = configWithCap(0.000001, { + limits: { maxEstimatedCostUsd: 0.000001, onUnknownCost: "exclude" }, }); - // No onUnknownCost configured → must behave exactly as today (fail-open), - // so existing deployments do not lose every live route on upgrade. - const result = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ - { - provider: "anthropic", - model: "claude-opus-5", - capability: { contextWindow: 200000 }, - cost: evidence, - }, - ]); + const allowDry = await handleManagementAPI( + new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "cost" }), + }), + new URL("http://localhost/api/routing-profiles/dry-run"), + allowConfig, + { refreshCodexCatalog: async () => {} }, + ); + expect(allowDry!.status).toBe(200); + const allowBody = await allowDry!.json() as { + candidates: Array<{ + eligible: boolean; + exclusions: Array<{ code: string }>; + cost?: { capOutcome?: string }; + }>; + selectedIndex: number | null; + trace: { candidates: Array<{ cost?: { capOutcome?: string }; exclusions: Array<{ code: string }> }> }; + }; + expect(allowBody.candidates[0]!.eligible).toBe(true); + expect(allowBody.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); + expect(allowBody.trace.candidates[0]?.cost?.capOutcome).toBe("unknown-allowed"); + expect(allowBody.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(false); - expect(result.candidates[0]!.eligible).toBe(true); - expect(result.selectedIndex).toBe(0); + const allowLive = routeModel(allowConfig, "policy/cost"); + expect(allowLive.routeKind).toBe("policy"); + expect(allowLive.routeDecision?.candidates[0]?.eligible).toBe(true); + expect(allowLive.routeDecision?.candidates[0]?.cost?.capOutcome).toBe("unknown-allowed"); + expect(allowLive.routeDecision?.candidates[0]?.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(false); + + const excludeDry = await handleManagementAPI( + new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "cost" }), + }), + new URL("http://localhost/api/routing-profiles/dry-run"), + excludeConfig, + { refreshCodexCatalog: async () => {} }, + ); + expect(excludeDry!.status).toBe(200); + const excludeBody = await excludeDry!.json() as { + candidates: Array<{ + eligible: boolean; + exclusions: Array<{ code: string }>; + cost?: { capOutcome?: string }; + }>; + selectedIndex: number | null; + }; + expect(excludeBody.candidates[0]!.eligible).toBe(false); + expect(excludeBody.candidates[0]!.cost?.capOutcome).toBe("unknown-excluded"); + expect(excludeBody.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown")).toBe(true); + expect(excludeBody.selectedIndex).toBeNull(); + + expect(() => routeModel(excludeConfig, "policy/cost")).toThrow(NoEligiblePolicyCandidateError); + }); + + test("unknownEvidence.cost exclude stays distinguishable on the dry-run path", async () => { + const config = configWithCap(0.000001, { + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "exclude" }, + }); + const res = await handleManagementAPI( + new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "cost" }), + }), + new URL("http://localhost/api/routing-profiles/dry-run"), + config, + { refreshCodexCatalog: async () => {} }, + ); + expect(res!.status).toBe(200); + const body = await res!.json() as { + candidates: Array<{ exclusions: Array<{ code: string }>; cost?: { capOutcome?: string } }>; + }; + const codes = body.candidates[0]!.exclusions.map(e => e.code); + expect(codes).toContain("unknown-price"); + expect(codes).not.toContain("cost-limit-unknown"); + expect(body.candidates[0]!.cost?.capOutcome).toBe("unknown-allowed"); }); }); From f42060c5629c3f85690efa4e504945376c365dce Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:28:27 +0200 Subject: [PATCH 3/8] fix(gui): add routing.costCap i18n keys for all locales Keep locale Records complete so GUI typecheck/CI does not fail on the new dry-run cost-cap column label. --- gui/src/i18n/de.ts | 1 + gui/src/i18n/ja.ts | 1 + gui/src/i18n/ko.ts | 1 + gui/src/i18n/ru.ts | 1 + gui/src/i18n/tr.ts | 1 + gui/src/i18n/zh.ts | 1 + 6 files changed, 6 insertions(+) diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 917a92f563a..9b94fd7f7bc 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -46,6 +46,7 @@ export const de: Record = { "routing.candidate": "Kandidat", "routing.eligible": "Geeignet", "routing.exclusions": "Ausschlüsse", + "routing.costCap": "Kostenobergrenze", "routing.score": "Punktzahl", "routing.selected": "ausgewählt", "routing.yes": "ja", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 4d57ca67013..4945236e3ee 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -46,6 +46,7 @@ export const ja: Record = { "routing.candidate": "候補", "routing.eligible": "対象", "routing.exclusions": "除外", + "routing.costCap": "コスト上限", "routing.score": "スコア", "routing.selected": "選択済み", "routing.yes": "はい", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index bf842b922f7..a760759686d 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -46,6 +46,7 @@ export const ko: Record = { "routing.candidate": "후보", "routing.eligible": "적격", "routing.exclusions": "제외", + "routing.costCap": "비용 상한", "routing.score": "점수", "routing.selected": "선택됨", "routing.yes": "예", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 48a584e5e23..ff41b779818 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -46,6 +46,7 @@ export const ru: Record = { "routing.candidate": "Кандидат", "routing.eligible": "Допустим", "routing.exclusions": "Исключения", + "routing.costCap": "Лимит стоимости", "routing.score": "Оценка", "routing.selected": "выбран", "routing.yes": "да", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 494fa072117..b3f64506544 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -88,6 +88,7 @@ export const tr: Record = { "routing.candidate": "Aday", "routing.eligible": "Uygun", "routing.exclusions": "Hariç Tutulanlar", + "routing.costCap": "Maliyet tavanı", "routing.score": "Puan", "routing.selected": "seçildi", "routing.yes": "evet", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index f1f0eb62b3b..c1b63283485 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -46,6 +46,7 @@ export const zh: Record = { "routing.candidate": "候选", "routing.eligible": "合格", "routing.exclusions": "排除项", + "routing.costCap": "成本上限", "routing.score": "分数", "routing.selected": "已选择", "routing.yes": "是", From c248690040cbffdfa0415dafab9e2b635754966f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:35:50 +0200 Subject: [PATCH 4/8] fix(routing): address CodeRabbit contract nits on cost-cap outcomes Align docs with unknownEvidence.cost eligibility, stamp the applied profile cap, drop orphaned capOutcome without limitUsd, and localize dry-run labels. --- .../docs/guides/routing-profile-editor.md | 7 ++- .../docs/reference/configuration/routing.md | 9 ++-- gui/src/i18n/de.ts | 4 ++ gui/src/i18n/en.ts | 4 ++ gui/src/i18n/ja.ts | 4 ++ gui/src/i18n/ko.ts | 4 ++ gui/src/i18n/ru.ts | 4 ++ gui/src/i18n/tr.ts | 4 ++ gui/src/i18n/zh.ts | 4 ++ gui/src/pages/RoutingProfiles.tsx | 21 ++++++++- src/routing/evaluator.ts | 2 +- src/routing/trace.ts | 4 +- tests/cost-cap-unknown-evidence.test.ts | 45 +++++++++++++++++++ 13 files changed, 108 insertions(+), 8 deletions(-) diff --git a/docs-site/src/content/docs/guides/routing-profile-editor.md b/docs-site/src/content/docs/guides/routing-profile-editor.md index 58ac8901283..e777e6f7838 100644 --- a/docs-site/src/content/docs/guides/routing-profile-editor.md +++ b/docs-site/src/content/docs/guides/routing-profile-editor.md @@ -28,7 +28,12 @@ The dashboard sends the same profile object used by `config.routingProfiles` to A successful save persists the profile through the normal config writer, reconciles live state, and refreshes the model catalog. Validation failures leave the previous configuration unchanged and are shown in the editor. -`limits.onUnknownCost` defaults to `"allow"`: an unknown cost estimate stays eligible, and dry-run / live route-decision traces stamp `cost.capOutcome: "unknown-allowed"` so operators can tell the cap was not proven. Set `"exclude"` when the ceiling must fail closed (`cost-limit-unknown`). This is separate from `unknownEvidence.cost`, which only affects scoring. +`limits.onUnknownCost` defaults to `"allow"`: an unknown cost estimate does not get a +cap-specific exclusion, and dry-run / live route-decision traces stamp +`cost.capOutcome: "unknown-allowed"` so operators can tell the cap was not proven. Set `"exclude"` +when the ceiling must fail closed (`cost-limit-unknown`). This is separate from +`unknownEvidence.cost`, which can still exclude or penalize unknown prices independently of the +cap outcome. ## Dry-run a saved profile diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 49deee248ac..39420785cef 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -114,7 +114,7 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c | `alias?` | `string` | — | Optional public model id in place of `policy/`. | | `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). | | `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights, normalized deterministically. `health`, `quota`, and `cost` have score dimensions; the configured-priority share is `1 - health - quota - cost` (default 0.55), and `latency` folds into that priority share rather than scoring independently. | -| `limits?` | object | — | Hard limits. `maxEstimatedCostUsd` excludes a candidate when its estimated cost is known and above the cap. `onUnknownCost` (`"allow"` default, or `"exclude"`) controls unknown estimates: allow keeps the candidate eligible and stamps `cost.capOutcome: "unknown-allowed"` on dry-run/live traces; exclude emits `cost-limit-unknown` and `capOutcome: "unknown-excluded"`. Separate from `unknownEvidence.cost`, which only affects scoring. | +| `limits?` | object | — | Hard limits. `maxEstimatedCostUsd` excludes a candidate when its estimated cost is known and above the cap. `onUnknownCost` (`"allow"` default, or `"exclude"`) controls unknown estimates for that ceiling: allow prevents a cap-specific exclusion and records `cost.capOutcome: "unknown-allowed"`; exclude emits `cost-limit-unknown` and `capOutcome: "unknown-excluded"`. Separate from `unknownEvidence.cost`, which can still exclude or penalize unknown prices via `unknown-price` / scoring. | | `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | `require` supports: `minContextWindow` (positive integer), `minQuotaHeadroom` (0..1 fraction), @@ -187,9 +187,10 @@ quota (RI-07), and cost (RI-08) score dimensions where evidence is present; the folds into the priority share rather than scoring independently. Cost is also enforced through the `limits.maxEstimatedCostUsd` cap: a candidate whose estimated cost is known and exceeds the cap is excluded (`cost-limit`). When the estimate is unknown, the default `limits.onUnknownCost: "allow"` -keeps the candidate eligible and records `cost.capOutcome: "unknown-allowed"` on the route-decision -trace; set `onUnknownCost: "exclude"` for a fail-closed ceiling (`cost-limit-unknown`). -Per-request route-decision traces are recorded when a policy profile executes. +records `cost.capOutcome: "unknown-allowed"` on the route-decision trace without a cap exclusion; +set `onUnknownCost: "exclude"` for a fail-closed ceiling (`cost-limit-unknown`). Cap outcome is not +overall eligibility — `unknownEvidence.cost: "exclude"` can still add `unknown-price` and mark the +candidate ineligible. Per-request route-decision traces are recorded when a policy profile executes. ### Catalog eligibility diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 9b94fd7f7bc..7d1cea568e9 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -47,6 +47,10 @@ export const de: Record = { "routing.eligible": "Geeignet", "routing.exclusions": "Ausschlüsse", "routing.costCap": "Kostenobergrenze", + "routing.capOutcome.satisfied": "innerhalb Limit", + "routing.capOutcome.exceeded": "über Limit", + "routing.capOutcome.unknown-allowed": "unbekannt (erlaubt)", + "routing.capOutcome.unknown-excluded": "unbekannt (ausgeschlossen)", "routing.score": "Punktzahl", "routing.selected": "ausgewählt", "routing.yes": "ja", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 6565140a30c..68e7baa6c20 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -90,6 +90,10 @@ export const en = { "routing.eligible": "Eligible", "routing.exclusions": "Exclusions", "routing.costCap": "Cost cap", + "routing.capOutcome.satisfied": "within limit", + "routing.capOutcome.exceeded": "over limit", + "routing.capOutcome.unknown-allowed": "unknown (allowed)", + "routing.capOutcome.unknown-excluded": "unknown (excluded)", "routing.score": "Score", "routing.selected": "selected", "routing.yes": "yes", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 4945236e3ee..1afdda1e097 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -47,6 +47,10 @@ export const ja: Record = { "routing.eligible": "対象", "routing.exclusions": "除外", "routing.costCap": "コスト上限", + "routing.capOutcome.satisfied": "上限内", + "routing.capOutcome.exceeded": "上限超過", + "routing.capOutcome.unknown-allowed": "不明(許可)", + "routing.capOutcome.unknown-excluded": "不明(除外)", "routing.score": "スコア", "routing.selected": "選択済み", "routing.yes": "はい", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index a760759686d..14bd7a8e449 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -47,6 +47,10 @@ export const ko: Record = { "routing.eligible": "적격", "routing.exclusions": "제외", "routing.costCap": "비용 상한", + "routing.capOutcome.satisfied": "한도 이내", + "routing.capOutcome.exceeded": "한도 초과", + "routing.capOutcome.unknown-allowed": "알 수 없음(허용)", + "routing.capOutcome.unknown-excluded": "알 수 없음(제외)", "routing.score": "점수", "routing.selected": "선택됨", "routing.yes": "예", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index ff41b779818..719cd70a56a 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -47,6 +47,10 @@ export const ru: Record = { "routing.eligible": "Допустим", "routing.exclusions": "Исключения", "routing.costCap": "Лимит стоимости", + "routing.capOutcome.satisfied": "в пределах лимита", + "routing.capOutcome.exceeded": "сверх лимита", + "routing.capOutcome.unknown-allowed": "неизвестно (разрешено)", + "routing.capOutcome.unknown-excluded": "неизвестно (исключено)", "routing.score": "Оценка", "routing.selected": "выбран", "routing.yes": "да", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index b3f64506544..5947fbb8590 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -89,6 +89,10 @@ export const tr: Record = { "routing.eligible": "Uygun", "routing.exclusions": "Hariç Tutulanlar", "routing.costCap": "Maliyet tavanı", + "routing.capOutcome.satisfied": "limit içinde", + "routing.capOutcome.exceeded": "limit aşıldı", + "routing.capOutcome.unknown-allowed": "bilinmiyor (izinli)", + "routing.capOutcome.unknown-excluded": "bilinmiyor (hariç)", "routing.score": "Puan", "routing.selected": "seçildi", "routing.yes": "evet", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index c1b63283485..3a06a25feac 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -47,6 +47,10 @@ export const zh: Record = { "routing.eligible": "合格", "routing.exclusions": "排除项", "routing.costCap": "成本上限", + "routing.capOutcome.satisfied": "未超限", + "routing.capOutcome.exceeded": "已超限", + "routing.capOutcome.unknown-allowed": "未知(允许)", + "routing.capOutcome.unknown-excluded": "未知(排除)", "routing.score": "分数", "routing.selected": "已选择", "routing.yes": "是", diff --git a/gui/src/pages/RoutingProfiles.tsx b/gui/src/pages/RoutingProfiles.tsx index f02bfa2a332..763b8f738e8 100644 --- a/gui/src/pages/RoutingProfiles.tsx +++ b/gui/src/pages/RoutingProfiles.tsx @@ -81,6 +81,25 @@ function fmtRate(value: number | null | undefined, unavailable: string): string return value === null || value === undefined ? unavailable : `${Math.round(value * 100)}%`; } +function fmtCapOutcome( + value: string | undefined, + t: ReturnType, + unavailable: string, +): string { + switch (value) { + case "satisfied": + return t("routing.capOutcome.satisfied"); + case "exceeded": + return t("routing.capOutcome.exceeded"); + case "unknown-allowed": + return t("routing.capOutcome.unknown-allowed"); + case "unknown-excluded": + return t("routing.capOutcome.unknown-excluded"); + default: + return unavailable; + } +} + function isPlainObject(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -851,7 +870,7 @@ export default function RoutingProfiles({ {candidate.eligible ? t("routing.yes") : t("routing.no")} {candidate.exclusions.map(exclusion => exclusion.code).join(", ") || t("routing.none")} - {candidate.cost?.capOutcome ?? t("routing.none")} + {fmtCapOutcome(candidate.cost?.capOutcome, t, unavailable)} {candidate.score ? candidate.score.total.toFixed(3) : unavailable} ))} diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 30a1e4f6a75..a06449edd51 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -338,7 +338,7 @@ export function evaluatePolicyProfile( : "satisfied" as const; costForCandidate = { ...(evidence.cost ?? {}), - limitUsd: evidence.cost?.limitUsd ?? costLimit, + limitUsd: costLimit, capOutcome, }; } diff --git a/src/routing/trace.ts b/src/routing/trace.ts index 6497428a065..64cae992406 100644 --- a/src/routing/trace.ts +++ b/src/routing/trace.ts @@ -563,7 +563,9 @@ function parseCost(raw: unknown, caps: ParseCaps): RouteCostEvidence | undefined } if (typeof raw.incomplete === "boolean") out.incomplete = raw.incomplete; if (finiteNumber(raw.limitUsd)) out.limitUsd = raw.limitUsd; - if (typeof raw.capOutcome === "string" && COST_CAP_OUTCOMES.has(raw.capOutcome as RouteCostCapOutcome)) { + if (out.limitUsd !== undefined + && typeof raw.capOutcome === "string" + && COST_CAP_OUTCOMES.has(raw.capOutcome as RouteCostCapOutcome)) { out.capOutcome = raw.capOutcome as RouteCostCapOutcome; } return Object.keys(out).length > 0 ? out : undefined; diff --git a/tests/cost-cap-unknown-evidence.test.ts b/tests/cost-cap-unknown-evidence.test.ts index 28f6d1ce96d..926fe8e1cf7 100644 --- a/tests/cost-cap-unknown-evidence.test.ts +++ b/tests/cost-cap-unknown-evidence.test.ts @@ -198,6 +198,51 @@ describe("issue #1181 — hard cost cap under unknown evidence", () => { expect(result.candidates[0]!.cost?.capOutcome).toBeUndefined(); }); + test("trace stamps the profile cap even when evidence carries a different limitUsd", () => { + const result = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: { estimatedUsd: 0.01, incomplete: false, limitUsd: 1 }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(false); + expect(candidate.exclusions.some(e => e.code === "cost-limit")).toBe(true); + expect(candidate.cost?.limitUsd).toBe(0.000001); + expect(candidate.cost?.capOutcome).toBe("exceeded"); + expect(result.trace.candidates[0]?.cost?.limitUsd).toBe(0.000001); + }); + + test("normalizeRouteDecisionTrace drops capOutcome without a finite limitUsd", () => { + const dirty = { + version: 1 as const, + decisionId: "abcdef012345", + createdAt: 1, + requestedModel: "policy/cost", + routeKind: "policy" as const, + requirements: [], + candidates: [{ + provider: "anthropic", + model: "claude-opus-5", + eligible: true, + exclusions: [], + cost: { incomplete: true, capOutcome: "unknown-allowed" as const }, + }], + selected: { + candidateIndex: 0, + provider: "anthropic", + model: "claude-opus-5", + reason: "policy-selected", + }, + }; + const normalized = normalizeRouteDecisionTrace(dirty); + expect(normalized).not.toBeNull(); + expect(normalized?.candidates[0]?.cost?.capOutcome).toBeUndefined(); + expect(normalized?.candidates[0]?.cost?.incomplete).toBe(true); + }); + test("explicit onUnknownCost allow matches omitted default", () => { const evidence = livePathEvidence(0.000001); const omitted = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ From 6be095951cfea2833a5863a43b6d16878f884e72 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 08:12:40 +0200 Subject: [PATCH 5/8] fix(routing): keep costScore free of cap-outcome stamping Trace/dry-run still report the profile limit and incomplete synthesis, but scoring uses the caller's original cost evidence so ranking cannot drift. --- src/routing/evaluator.ts | 13 ++++++--- tests/cost-cap-unknown-evidence.test.ts | 36 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index a06449edd51..1720c26f02d 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -325,8 +325,10 @@ export function evaluatePolicyProfile( } let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit && !unknownCostBlocked; - // Stamp a copy of cost evidence with the cap outcome so traces/dry-run - // carry operator-visible status without mutating caller-supplied objects. + // Trace/dry-run copy only: report the profile cap that was applied and the + // operator-visible outcome. Do not feed this copy into costScore() — that + // would silently change ranking when a caller supplied a different + // limitUsd (costScore uses limitUsd as its reference denominator). let costForCandidate = evidence.cost; if (costLimit !== undefined) { const capOutcome = overCostLimit @@ -336,8 +338,11 @@ export function evaluatePolicyProfile( : unknownCostUnderCap ? "unknown-allowed" as const : "satisfied" as const; + const synthesizedFromNothing = evidence.cost === undefined; costForCandidate = { ...(evidence.cost ?? {}), + // Match costEvidenceForCandidate: absent usage/price is incomplete. + ...(synthesizedFromNothing ? { incomplete: true } : {}), limitUsd: costLimit, capOutcome, }; @@ -377,7 +382,9 @@ export function evaluatePolicyProfile( // Cost scoring (RI-08): the hard per-request ceiling was already checked // above; unknown cost follows the profile's unknownEvidence policy. - const cost = costForCandidate; + // Score against the caller's original evidence so trace stamping cannot + // move the costScore reference / ranking. + const cost = evidence.cost; let costValue = cost ? costScore(cost) : null; if (costValue === null && profile.unknownEvidence.cost === "exclude") { exclusions.push({ code: "unknown-price" }); diff --git a/tests/cost-cap-unknown-evidence.test.ts b/tests/cost-cap-unknown-evidence.test.ts index 926fe8e1cf7..2950755b791 100644 --- a/tests/cost-cap-unknown-evidence.test.ts +++ b/tests/cost-cap-unknown-evidence.test.ts @@ -215,6 +215,42 @@ describe("issue #1181 — hard cost cap under unknown evidence", () => { expect(result.trace.candidates[0]?.cost?.limitUsd).toBe(0.000001); }); + test("trace limitUsd normalization does not change costScore for eligible candidates", () => { + // Profile cap 0.5; caller evidence carries limitUsd 1. Scoring must keep + // the caller's reference (1 - 0.4/1 = 0.6), not the stamped profile cap + // (1 - 0.4/0.5 = 0.2). + const result = evaluatePolicyProfile(configWithCap(0.5), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: { estimatedUsd: 0.4, incomplete: false, limitUsd: 1 }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.cost?.limitUsd).toBe(0.5); + expect(candidate.cost?.capOutcome).toBe("satisfied"); + expect(candidate.score?.components.cost).toBeCloseTo(0.6, 5); + }); + + test("synthesized cost evidence from missing input marks incomplete", () => { + const result = evaluatePolicyProfile(configWithCap(0.5), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.cost).toEqual({ + incomplete: true, + limitUsd: 0.5, + capOutcome: "unknown-allowed", + }); + }); + test("normalizeRouteDecisionTrace drops capOutcome without a finite limitUsd", () => { const dirty = { version: 1 as const, From d4850e2dec1fffd4f4e72814b5ea2348e7f7a583 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 08:16:42 +0200 Subject: [PATCH 6/8] fix(gui): address CodeRabbit cost-cap docs and i18n nits Qualify onUnknownCost docs with the required cap, fix DE/TR labels, and localize dry-run exclusion codes including cost-limit-unknown. --- .../docs/guides/routing-profile-editor.md | 4 +-- .../docs/reference/configuration/routing.md | 4 +-- gui/src/i18n/de.ts | 13 ++++++++-- gui/src/i18n/en.ts | 9 +++++++ gui/src/i18n/ja.ts | 9 +++++++ gui/src/i18n/ko.ts | 9 +++++++ gui/src/i18n/ru.ts | 9 +++++++ gui/src/i18n/tr.ts | 11 +++++++- gui/src/i18n/zh.ts | 9 +++++++ gui/src/pages/RoutingProfiles.tsx | 25 ++++++++++++++++++- gui/tests/routing-profiles.test.tsx | 2 +- 11 files changed, 95 insertions(+), 9 deletions(-) diff --git a/docs-site/src/content/docs/guides/routing-profile-editor.md b/docs-site/src/content/docs/guides/routing-profile-editor.md index e777e6f7838..9d7f883518d 100644 --- a/docs-site/src/content/docs/guides/routing-profile-editor.md +++ b/docs-site/src/content/docs/guides/routing-profile-editor.md @@ -28,10 +28,10 @@ The dashboard sends the same profile object used by `config.routingProfiles` to A successful save persists the profile through the normal config writer, reconciles live state, and refreshes the model catalog. Validation failures leave the previous configuration unchanged and are shown in the editor. -`limits.onUnknownCost` defaults to `"allow"`: an unknown cost estimate does not get a +When `limits.maxEstimatedCostUsd` is configured, `limits.onUnknownCost` defaults to `"allow"`: an unknown cost estimate does not get a cap-specific exclusion, and dry-run / live route-decision traces stamp `cost.capOutcome: "unknown-allowed"` so operators can tell the cap was not proven. Set `"exclude"` -when the ceiling must fail closed (`cost-limit-unknown`). This is separate from +when the ceiling must fail closed (`cost-limit-unknown`). Configuring `onUnknownCost` alone is inert and does not emit a cap outcome. This is separate from `unknownEvidence.cost`, which can still exclude or penalize unknown prices independently of the cap outcome. diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 39420785cef..8a2b039c26e 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -114,7 +114,7 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c | `alias?` | `string` | — | Optional public model id in place of `policy/`. | | `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). | | `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights, normalized deterministically. `health`, `quota`, and `cost` have score dimensions; the configured-priority share is `1 - health - quota - cost` (default 0.55), and `latency` folds into that priority share rather than scoring independently. | -| `limits?` | object | — | Hard limits. `maxEstimatedCostUsd` excludes a candidate when its estimated cost is known and above the cap. `onUnknownCost` (`"allow"` default, or `"exclude"`) controls unknown estimates for that ceiling: allow prevents a cap-specific exclusion and records `cost.capOutcome: "unknown-allowed"`; exclude emits `cost-limit-unknown` and `capOutcome: "unknown-excluded"`. Separate from `unknownEvidence.cost`, which can still exclude or penalize unknown prices via `unknown-price` / scoring. | +| `limits?` | object | — | Hard limits. `maxEstimatedCostUsd` excludes a candidate when its estimated cost is known and above the cap. When that cap is set, `onUnknownCost` (`"allow"` default, or `"exclude"`) controls unknown estimates: allow prevents a cap-specific exclusion and records `cost.capOutcome: "unknown-allowed"`; exclude emits `cost-limit-unknown` and `capOutcome: "unknown-excluded"`. `onUnknownCost` alone (no cap) is inert. Separate from `unknownEvidence.cost`, which can still exclude or penalize unknown prices via `unknown-price` / scoring. | | `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. | `require` supports: `minContextWindow` (positive integer), `minQuotaHeadroom` (0..1 fraction), @@ -186,7 +186,7 @@ is chosen. Profile scoring combines the configured-priority component with the h quota (RI-07), and cost (RI-08) score dimensions where evidence is present; the `latency` weight folds into the priority share rather than scoring independently. Cost is also enforced through the `limits.maxEstimatedCostUsd` cap: a candidate whose estimated cost is known and exceeds the cap is -excluded (`cost-limit`). When the estimate is unknown, the default `limits.onUnknownCost: "allow"` +excluded (`cost-limit`). When a cap is configured and the estimate is unknown, the default `limits.onUnknownCost: "allow"` records `cost.capOutcome: "unknown-allowed"` on the route-decision trace without a cap exclusion; set `onUnknownCost: "exclude"` for a fail-closed ceiling (`cost-limit-unknown`). Cap outcome is not overall eligibility — `unknownEvidence.cost: "exclude"` can still add `unknown-price` and mark the diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 7d1cea568e9..31e85574669 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -47,10 +47,19 @@ export const de: Record = { "routing.eligible": "Geeignet", "routing.exclusions": "Ausschlüsse", "routing.costCap": "Kostenobergrenze", - "routing.capOutcome.satisfied": "innerhalb Limit", - "routing.capOutcome.exceeded": "über Limit", + "routing.capOutcome.satisfied": "innerhalb des Limits", + "routing.capOutcome.exceeded": "über dem Limit", "routing.capOutcome.unknown-allowed": "unbekannt (erlaubt)", "routing.capOutcome.unknown-excluded": "unbekannt (ausgeschlossen)", + "routing.exclusion.capability-unsatisfied": "Anforderung nicht erfüllt", + "routing.exclusion.unknown-capability": "unbekannte Fähigkeit", + "routing.exclusion.cost-limit": "über Kostenobergrenze", + "routing.exclusion.cost-limit-unknown": "unbekannte Kosten unter Obergrenze", + "routing.exclusion.cooldown": "Abklingzeit", + "routing.exclusion.unknown-health": "unbekannter Zustand", + "routing.exclusion.unknown-quota": "unbekanntes Kontingent", + "routing.exclusion.unknown-price": "unbekannter Preis", + "routing.exclusion.other": "Ausschluss: {code}", "routing.score": "Punktzahl", "routing.selected": "ausgewählt", "routing.yes": "ja", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 68e7baa6c20..2dc6a0793a0 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -94,6 +94,15 @@ export const en = { "routing.capOutcome.exceeded": "over limit", "routing.capOutcome.unknown-allowed": "unknown (allowed)", "routing.capOutcome.unknown-excluded": "unknown (excluded)", + "routing.exclusion.capability-unsatisfied": "capability not met", + "routing.exclusion.unknown-capability": "unknown capability", + "routing.exclusion.cost-limit": "over cost cap", + "routing.exclusion.cost-limit-unknown": "unknown cost under cap", + "routing.exclusion.cooldown": "cooldown", + "routing.exclusion.unknown-health": "unknown health", + "routing.exclusion.unknown-quota": "unknown quota", + "routing.exclusion.unknown-price": "unknown price", + "routing.exclusion.other": "exclusion: {code}", "routing.score": "Score", "routing.selected": "selected", "routing.yes": "yes", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 1afdda1e097..c097ec8b43f 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -51,6 +51,15 @@ export const ja: Record = { "routing.capOutcome.exceeded": "上限超過", "routing.capOutcome.unknown-allowed": "不明(許可)", "routing.capOutcome.unknown-excluded": "不明(除外)", + "routing.exclusion.capability-unsatisfied": "能力要件未達", + "routing.exclusion.unknown-capability": "能力不明", + "routing.exclusion.cost-limit": "コスト上限超過", + "routing.exclusion.cost-limit-unknown": "上限下でコスト不明", + "routing.exclusion.cooldown": "クールダウン", + "routing.exclusion.unknown-health": "健全性不明", + "routing.exclusion.unknown-quota": "割当不明", + "routing.exclusion.unknown-price": "価格不明", + "routing.exclusion.other": "除外: {code}", "routing.score": "スコア", "routing.selected": "選択済み", "routing.yes": "はい", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 14bd7a8e449..d81b1543135 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -51,6 +51,15 @@ export const ko: Record = { "routing.capOutcome.exceeded": "한도 초과", "routing.capOutcome.unknown-allowed": "알 수 없음(허용)", "routing.capOutcome.unknown-excluded": "알 수 없음(제외)", + "routing.exclusion.capability-unsatisfied": "기능 미충족", + "routing.exclusion.unknown-capability": "알 수 없는 기능", + "routing.exclusion.cost-limit": "비용 상한 초과", + "routing.exclusion.cost-limit-unknown": "상한 이하 비용 불명", + "routing.exclusion.cooldown": "쿨다운", + "routing.exclusion.unknown-health": "상태 불명", + "routing.exclusion.unknown-quota": "할당량 불명", + "routing.exclusion.unknown-price": "가격 불명", + "routing.exclusion.other": "제외: {code}", "routing.score": "점수", "routing.selected": "선택됨", "routing.yes": "예", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 719cd70a56a..07089f6e4de 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -51,6 +51,15 @@ export const ru: Record = { "routing.capOutcome.exceeded": "сверх лимита", "routing.capOutcome.unknown-allowed": "неизвестно (разрешено)", "routing.capOutcome.unknown-excluded": "неизвестно (исключено)", + "routing.exclusion.capability-unsatisfied": "требование не выполнено", + "routing.exclusion.unknown-capability": "неизвестная возможность", + "routing.exclusion.cost-limit": "сверх лимита стоимости", + "routing.exclusion.cost-limit-unknown": "неизвестная стоимость при лимите", + "routing.exclusion.cooldown": "пауза", + "routing.exclusion.unknown-health": "неизвестное состояние", + "routing.exclusion.unknown-quota": "неизвестная квота", + "routing.exclusion.unknown-price": "неизвестная цена", + "routing.exclusion.other": "исключение: {code}", "routing.score": "Оценка", "routing.selected": "выбран", "routing.yes": "да", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 5947fbb8590..546aebead9e 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -92,7 +92,16 @@ export const tr: Record = { "routing.capOutcome.satisfied": "limit içinde", "routing.capOutcome.exceeded": "limit aşıldı", "routing.capOutcome.unknown-allowed": "bilinmiyor (izinli)", - "routing.capOutcome.unknown-excluded": "bilinmiyor (hariç)", + "routing.capOutcome.unknown-excluded": "bilinmiyor (hariç tutuldu)", + "routing.exclusion.capability-unsatisfied": "yetenek karşılanmadı", + "routing.exclusion.unknown-capability": "bilinmeyen yetenek", + "routing.exclusion.cost-limit": "maliyet tavanı aşıldı", + "routing.exclusion.cost-limit-unknown": "tavan altında bilinmeyen maliyet", + "routing.exclusion.cooldown": "soğuma süresi", + "routing.exclusion.unknown-health": "bilinmeyen sağlık", + "routing.exclusion.unknown-quota": "bilinmeyen kota", + "routing.exclusion.unknown-price": "bilinmeyen fiyat", + "routing.exclusion.other": "hariç tutma: {code}", "routing.score": "Puan", "routing.selected": "seçildi", "routing.yes": "evet", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 3a06a25feac..71c41897a0c 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -51,6 +51,15 @@ export const zh: Record = { "routing.capOutcome.exceeded": "已超限", "routing.capOutcome.unknown-allowed": "未知(允许)", "routing.capOutcome.unknown-excluded": "未知(排除)", + "routing.exclusion.capability-unsatisfied": "能力未满足", + "routing.exclusion.unknown-capability": "能力未知", + "routing.exclusion.cost-limit": "超出成本上限", + "routing.exclusion.cost-limit-unknown": "上限下成本未知", + "routing.exclusion.cooldown": "冷却中", + "routing.exclusion.unknown-health": "健康状态未知", + "routing.exclusion.unknown-quota": "配额未知", + "routing.exclusion.unknown-price": "价格未知", + "routing.exclusion.other": "排除项:{code}", "routing.score": "分数", "routing.selected": "已选择", "routing.yes": "是", diff --git a/gui/src/pages/RoutingProfiles.tsx b/gui/src/pages/RoutingProfiles.tsx index 763b8f738e8..b6a19ac3cec 100644 --- a/gui/src/pages/RoutingProfiles.tsx +++ b/gui/src/pages/RoutingProfiles.tsx @@ -100,6 +100,29 @@ function fmtCapOutcome( } } +function fmtExclusion(code: string, t: ReturnType): string { + switch (code) { + case "capability-unsatisfied": + return t("routing.exclusion.capability-unsatisfied"); + case "unknown-capability": + return t("routing.exclusion.unknown-capability"); + case "cost-limit": + return t("routing.exclusion.cost-limit"); + case "cost-limit-unknown": + return t("routing.exclusion.cost-limit-unknown"); + case "cooldown": + return t("routing.exclusion.cooldown"); + case "unknown-health": + return t("routing.exclusion.unknown-health"); + case "unknown-quota": + return t("routing.exclusion.unknown-quota"); + case "unknown-price": + return t("routing.exclusion.unknown-price"); + default: + return t("routing.exclusion.other", { code }); + } +} + function isPlainObject(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -869,7 +892,7 @@ export default function RoutingProfiles({ {index === dryRunResult.selectedIndex ? ` ✓ (${t("routing.selected")})` : ""} {candidate.eligible ? t("routing.yes") : t("routing.no")} - {candidate.exclusions.map(exclusion => exclusion.code).join(", ") || t("routing.none")} + {candidate.exclusions.map(exclusion => fmtExclusion(exclusion.code, t)).join(", ") || t("routing.none")} {fmtCapOutcome(candidate.cost?.capOutcome, t, unavailable)} {candidate.score ? candidate.score.total.toFixed(3) : unavailable} diff --git a/gui/tests/routing-profiles.test.tsx b/gui/tests/routing-profiles.test.tsx index 9262c937f5b..7250402b137 100644 --- a/gui/tests/routing-profiles.test.tsx +++ b/gui/tests/routing-profiles.test.tsx @@ -196,7 +196,7 @@ test("routing page loads profiles, analytics, and marks the dry-run selection", expect(dryRunBodies).toEqual([{ profile: "balanced", evidence: {} }]); expect(container.textContent).toContain("selected"); - expect(container.textContent).toContain("unknown-capability"); + expect(container.textContent).toContain("unknown capability"); expect(container.textContent).toContain("0.910"); } finally { await act(async () => { root.unmount(); }); From 51b1bbe68db4ae84ed63838a765852be2784d9d5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 08:22:05 +0200 Subject: [PATCH 7/8] fix(routing): strip non-finite cost estimates from cap traces Treat missing and non-finite estimatedUsd the same on the stamped copy: mark incomplete and omit Infinity/NaN so unknown-allowed traces stay honest. --- src/routing/evaluator.ts | 25 +++++++++++++++++-------- tests/cost-cap-unknown-evidence.test.ts | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 1720c26f02d..6554434f95b 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -338,14 +338,23 @@ export function evaluatePolicyProfile( : unknownCostUnderCap ? "unknown-allowed" as const : "satisfied" as const; - const synthesizedFromNothing = evidence.cost === undefined; - costForCandidate = { - ...(evidence.cost ?? {}), - // Match costEvidenceForCandidate: absent usage/price is incomplete. - ...(synthesizedFromNothing ? { incomplete: true } : {}), - limitUsd: costLimit, - capOutcome, - }; + if (!costEstimateKnown) { + // Missing or non-finite estimates are the same unknown: never stamp + // Infinity/NaN into the trace, and always mark incomplete. + const { estimatedUsd: _nonFiniteOrMissing, ...rest } = evidence.cost ?? {}; + costForCandidate = { + ...rest, + incomplete: true, + limitUsd: costLimit, + capOutcome, + }; + } else { + costForCandidate = { + ...evidence.cost!, + limitUsd: costLimit, + capOutcome, + }; + } } // Health scoring (RI-06): live hard cooldown is authoritative and diff --git a/tests/cost-cap-unknown-evidence.test.ts b/tests/cost-cap-unknown-evidence.test.ts index 2950755b791..e9444a8dfdb 100644 --- a/tests/cost-cap-unknown-evidence.test.ts +++ b/tests/cost-cap-unknown-evidence.test.ts @@ -251,6 +251,28 @@ describe("issue #1181 — hard cost cap under unknown evidence", () => { }); }); + test("non-finite estimatedUsd is stamped as incomplete unknown, not Infinity", () => { + const result = evaluatePolicyProfile(configWithCap(0.5), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: { estimatedUsd: Number.POSITIVE_INFINITY, incomplete: false, priceSource: "registry" }, + }, + ]); + const candidate = result.candidates[0]!; + expect(candidate.eligible).toBe(true); + expect(candidate.exclusions.some(e => e.code === "cost-limit")).toBe(false); + expect(candidate.cost?.estimatedUsd).toBeUndefined(); + expect(candidate.cost).toEqual({ + incomplete: true, + priceSource: "registry", + limitUsd: 0.5, + capOutcome: "unknown-allowed", + }); + expect(result.trace.candidates[0]?.cost?.estimatedUsd).toBeUndefined(); + }); + test("normalizeRouteDecisionTrace drops capOutcome without a finite limitUsd", () => { const dirty = { version: 1 as const, From 56aa51cb6bacb870ec28283d83bee27f8698c09d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 08:30:18 +0200 Subject: [PATCH 8/8] docs(routing): document unknown-excluded on fail-closed cost cap --- docs-site/src/content/docs/guides/routing-profile-editor.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs-site/src/content/docs/guides/routing-profile-editor.md b/docs-site/src/content/docs/guides/routing-profile-editor.md index 9d7f883518d..5cf5fc6d71e 100644 --- a/docs-site/src/content/docs/guides/routing-profile-editor.md +++ b/docs-site/src/content/docs/guides/routing-profile-editor.md @@ -31,7 +31,8 @@ A successful save persists the profile through the normal config writer, reconci When `limits.maxEstimatedCostUsd` is configured, `limits.onUnknownCost` defaults to `"allow"`: an unknown cost estimate does not get a cap-specific exclusion, and dry-run / live route-decision traces stamp `cost.capOutcome: "unknown-allowed"` so operators can tell the cap was not proven. Set `"exclude"` -when the ceiling must fail closed (`cost-limit-unknown`). Configuring `onUnknownCost` alone is inert and does not emit a cap outcome. This is separate from +when the ceiling must fail closed (`cost-limit-unknown`, with +`cost.capOutcome: "unknown-excluded"`). Configuring `onUnknownCost` alone is inert and does not emit a cap outcome. This is separate from `unknownEvidence.cost`, which can still exclude or penalize unknown prices independently of the cap outcome.