From 2e234c1dbd4565b856ea55f53dc39a8740706ed4 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 9 Oct 2026 10:41:58 +0900 Subject: [PATCH] fix(jev): apply initial effort to native Chat dispatch Carry of #6750 onto current dev with review fixes. A validated JEV choice now owns the first native Chat dispatch, including an explicit no-effort choice; provider pins, caps and wire normalization still apply downstream, and later targets do not ask the judge again. Review fixes on top of the contributor's change: - collapse adjacent identical segments in the Combo requested-effort label, so a provider pin that kept the JEV effort no longer logs high->low->low - assert the native lane actually ran in the initial-effort wire matrix Supersedes #6750 Co-authored-by: GeunwooJun <313474999+geunwoojun99@users.noreply.github.com> --- docs-site/src/content/docs/guides/combos.md | 13 +- scripts/test-layout/layout.json | 2 +- .../responses/combo-requested-effort.ts | 4 +- src/server/responses/core-combo-native.ts | 23 +- src/server/responses/core-combo.ts | 8 +- structure/clients/claude-desktop.md | 2 +- structure/dashboard-and-usage.md | 2 +- structure/data-planes/images.md | 2 +- structure/ops/service-and-sidecars.md | 2 +- structure/providers/jev-decision.md | 17 +- structure/providers/xai-grok.md | 2 +- structure/subagents.md | 2 +- structure/transports/responses.md | 6 +- structure/transports/streaming-health.md | 2 +- tests/fixtures/test-layout-expected.json | 2 +- tests/responses/chat-native-combo.test.ts | 8 +- .../responses/jev-initial-effort-wire.test.ts | 299 ++++++++++++++++++ tests/server/combo-requested-effort.test.ts | 1 + 18 files changed, 359 insertions(+), 38 deletions(-) create mode 100644 tests/responses/jev-initial-effort-wire.test.ts diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index deff7eb095c..60e3db08e6b 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -512,12 +512,13 @@ target/effort, a coarse decision gate, latency, optional confidence/probability, Automated tests use mocked TypeSafe responses plus a no-key fail-open smoke; a live TypeSafe decision requires an operator-supplied key and is not run implicitly. -When JEV's selected effort is applied to the Responses child, request and attempt logs show a -differing effort as `high->xhigh` or `high->low`. Later effort adjustments remain in the chain, -such as `high->xhigh->medium`. An unchanged effort or a target without effort control keeps the -existing label; fallback attempts do not inherit the initial JEV effort. With the opt-in -`protocols.rollout.nativeChatCombos` path, native Chat children currently do not apply JEV's -selected effort; their labels retain the caller's effort and any native child adjustments instead. +JEV's selected effort applies to the first Responses child and to native Chat children on the opt-in +`protocols.rollout.nativeChatCombos` path. It replaces caller effort and removes conflicting thinking +controls and caller `service_tier`; an explicit no-effort choice removes effort even when the Combo has +a forced default. Provider effort pins, caps and wire normalization still apply afterward. +Request and attempt logs show a differing applied effort as `high->xhigh` or `high->low`, retaining later +adjustments such as `high->xhigh->medium`. An unchanged effort or a no-effort choice keeps the existing +label. Later fallback targets use ordinary Combo policy from the original request, without another JEV call. After the Combo has served requests, open **Models → Combos → jev-auto → Stats** to inspect JEV's picks without replacing the normal model picker or Usage page. The tab separates backend-reported decision diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 5f226f55b41..3014b56dc86 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -308,7 +308,7 @@ "protocol-direct-encoders-stall-disabled.test.ts": "responses", "protocol-direct-encoders-local-upstream.test.ts": "responses", "protocol-direct-encoders-messages.test.ts": "responses", - "chat-native-combo.test.ts": "responses", + "chat-native-combo.test.ts": "responses", "jev-initial-effort-wire.test.ts": "responses", "messages-native-eligibility.test.ts": "responses", "messages-native-bridge-policy.test.ts": "responses", "messages-native-oauth-eligibility.test.ts": "responses", diff --git a/src/server/responses/combo-requested-effort.ts b/src/server/responses/combo-requested-effort.ts index 926d22d6da0..7d468bfb9d1 100644 --- a/src/server/responses/combo-requested-effort.ts +++ b/src/server/responses/combo-requested-effort.ts @@ -10,5 +10,7 @@ export function comboRequestedEffortLabel( const forcedTransition = forcedEffort != null && forcedEffort !== originalRequestedEffort ? `->${forcedEffort}` : ""; - return `${originalRequestedEffort}${forcedTransition}${laterTransitions}`; + const segments = `${originalRequestedEffort}${forcedTransition}${laterTransitions}`.split("->"); + // A pin that kept the effort it was given is not a transition; drop the repeated segment. + return segments.filter((segment, index) => index === 0 || segment !== segments[index - 1]).join("->"); } diff --git a/src/server/responses/core-combo-native.ts b/src/server/responses/core-combo-native.ts index b9da29ee0fe..e57dabd1abf 100644 --- a/src/server/responses/core-combo-native.ts +++ b/src/server/responses/core-combo-native.ts @@ -32,7 +32,7 @@ import { type RequestLogContext, } from "../request-log"; import { captureRouteStaticPolicy, routeConcreteModel } from "../../router"; -import { comboDefaultEffort, concreteComboRequestBody, getCombo } from "../../combos"; +import { comboDefaultEffort, concreteComboRequestBody, getCombo, type JevDecision } from "../../combos"; import { supportedLadderFor } from "../effort-policy"; import { applyDroidReasoningDefault } from "../droid-reasoning-default"; import { resolveWireProtocolOverride } from "../adapter-resolve"; @@ -94,6 +94,7 @@ export interface ComboProtocolLanes { target: ComboTarget, targetRoute: RouteResult, targetSendBudget: TransientSendBudget | undefined, + initialDecision?: Pick, ): NativeComboChildPlan | undefined; } @@ -101,6 +102,7 @@ interface CandidateVerdict { skip?: ProtocolFeature[]; } +/** Build opt-in native eligibility and child plans; only the initial plan applies a JEV choice. */ export function createComboProtocolLanes(input: { source: ComboProtocolSource | undefined; req: Request; @@ -203,7 +205,7 @@ export function createComboProtocolLanes(input: { 400, unrepresentableMessage(features), "invalid_request_error", "unsupported_feature", ), "chat"); }, - nativeChild: (target, targetRoute, targetSendBudget) => { + nativeChild: (target, targetRoute, targetSendBudget, initialDecision) => { // Without the request's execution budget a native child could not share its sends, so it // keeps the bridge rather than opening a tracker of its own. if (!isRequestExecutionBudget(targetSendBudget)) return undefined; @@ -214,7 +216,22 @@ export function createComboProtocolLanes(input: { provider: targetRoute.provider, modelId: targetRoute.modelId, }); - applyComboEffort(body, config, comboId, target, targetRoute); + if (initialDecision) { + // A validated JEV choice owns only this first dispatch, including explicit no-effort. + // Do not run ordinary force defaults for null: they require a non-null default. + delete body.service_tier; + delete body.thinking_budget; + delete body.thinking; + const reasoning = body.reasoning; + if (reasoning && typeof reasoning === "object" && !Array.isArray(reasoning)) { + const preserved = { ...reasoning as Rec }; + delete preserved.effort; + if (Object.keys(preserved).length > 0) body.reasoning = preserved; + else delete body.reasoning; + } + if (initialDecision.effort === null) delete body.reasoning_effort; + else body.reasoning_effort = initialDecision.effort; + } else applyComboEffort(body, config, comboId, target, targetRoute); return { route, body, sendBudget: targetSendBudget }; }, }; diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 846c3b1688a..53ed28c2086 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -764,7 +764,7 @@ export async function executeComboResponses( initialJevDecision ? initialJevDecision.effort : comboDefaultEffort(config, comboId), initialJevDecision?.effort === null ? [] : targetReasoningEfforts, combo.reasoningEffortMode, - initialJevDecision !== undefined && initialJevDecision.effort !== null ? "force" : combo.defaultEffortMode, + initialJevDecision ? (initialJevDecision.effort === null ? "fallback" : "force") : combo.defaultEffortMode, ); if (initialJevDecision) { delete childBody.service_tier; @@ -860,7 +860,7 @@ export async function executeComboResponses( && targetEligible(target) && !isComboTargetInCooldown(comboId, target), ); - const nativeChild = protocolLanes?.nativeChild(pick.target, targetRoute, targetSendBudget); + const nativeChild = protocolLanes?.nativeChild(pick.target, targetRoute, targetSendBudget, initialJevDecision); response = nativeChild ? await dispatchNativeComboChild({ source: options.protocolSource!, plan: nativeChild, @@ -909,8 +909,8 @@ export async function executeComboResponses( onNativePassthroughCancel: callbackGate.onCancel, onResponseComplete: callbackGate.onResponseComplete, }); - // Native Chat rebuilds its own body and does not apply the initial JEV effort. - restoreOriginalRequestedEffort(childLog, nativeChild ? undefined : initialJevDecision?.effort); + // Both lanes applied the initial choice before provider pins/caps; retain those transitions. + restoreOriginalRequestedEffort(childLog, initialJevDecision?.effort); } catch (error) { callbackGate.discard(); if (options.abortSignal?.aborted) { diff --git a/structure/clients/claude-desktop.md b/structure/clients/claude-desktop.md index aa94003a72c..19544d85f0d 100644 --- a/structure/clients/claude-desktop.md +++ b/structure/clients/claude-desktop.md @@ -514,7 +514,7 @@ The account history response can include a [low-confidence effective capacity es Account quota surfaces use [safe probe diagnostics](../transports/inventory.md#account-quota-failure-diagnostics) separately from quota validity, credential health and routing authority. -Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](../transports/responses.md) documents this boundary, and native Chat removes effort only for an explicit empty declaration or no-reasoning model. +Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](../transports/responses.md) documents this boundary, and native Chat capability stripping applies only for an explicit empty declaration or no-reasoning model; an initial JEV null choice separately strips caller effort. Live sideband admission and its bounded upstream handshake follow the [runtime contract](../runtime.md#live-sideband-handshake); the ordinary Responses WebSocket exchange remains separate. diff --git a/structure/dashboard-and-usage.md b/structure/dashboard-and-usage.md index ae4ea9ee028..560fc6da79b 100644 --- a/structure/dashboard-and-usage.md +++ b/structure/dashboard-and-usage.md @@ -497,7 +497,7 @@ The existing dashboard file-client maps include Cline CLI and reuse its committe Native Chat applies qualifying effort ceilings independently of model pins; pin selection precedes the cap and only pins or cap rewrites enter wire mapping. The [catalog effort contract](catalog.md#ultra-reasoning-level) records the V1/compaction exemptions and caller-preservation boundary. Pool quota producers and account commands follow the [bounded raw-observation contract](providers/openai-accounts.md#bounded-pool-quota-observations), separate from the latest display snapshot and capacity estimates. The account history response can include a [low-confidence effective capacity estimate](providers/openai-accounts.md#observed-effective-token-capacity); usage normalization retains local-answer provenance so local responses cannot supply samples. Account quota surfaces use [safe probe diagnostics](transports/inventory.md#account-quota-failure-diagnostics) separately from quota validity, credential health and routing authority. -Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](transports/responses.md) documents this boundary, and native Chat removes effort only for an explicit empty declaration or no-reasoning model. +Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](transports/responses.md) documents this boundary, and native Chat capability stripping applies only for an explicit empty declaration or no-reasoning model; an initial JEV null choice separately strips caller effort. Live sideband admission and its bounded upstream handshake follow the [runtime contract](runtime.md#live-sideband-handshake); the ordinary Responses WebSocket exchange remains separate. Dashboard overview polling observes authorization failures independently of stalled or rejected peer requests, cancels remaining child requests after a decisive result, and exposes resource-level deadline failures without rewriting them as authentication failures. diff --git a/structure/data-planes/images.md b/structure/data-planes/images.md index 588209c22a9..0f3a82ca3ae 100644 --- a/structure/data-planes/images.md +++ b/structure/data-planes/images.md @@ -179,7 +179,7 @@ Pool quota producers and account commands follow the [bounded raw-observation co Account quota surfaces use [safe probe diagnostics](../transports/inventory.md#account-quota-failure-diagnostics) separately from quota validity, credential health and routing authority. -Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](../transports/responses.md) documents this boundary, and native Chat removes effort only for an explicit empty declaration or no-reasoning model. +Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](../transports/responses.md) documents this boundary, and native Chat capability stripping applies only for an explicit empty declaration or no-reasoning model; an initial JEV null choice separately strips caller effort. Live sideband admission and its bounded upstream handshake follow the [runtime contract](../runtime.md#live-sideband-handshake); the ordinary Responses WebSocket exchange remains separate. diff --git a/structure/ops/service-and-sidecars.md b/structure/ops/service-and-sidecars.md index af0419f014b..bcaee14a8b3 100644 --- a/structure/ops/service-and-sidecars.md +++ b/structure/ops/service-and-sidecars.md @@ -329,7 +329,7 @@ Pool quota producers and account commands follow the [bounded raw-observation co Account quota surfaces use [safe probe diagnostics](../transports/inventory.md#account-quota-failure-diagnostics) separately from quota validity, credential health and routing authority. -Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](../transports/responses.md) documents this boundary, and native Chat removes effort only for an explicit empty declaration or no-reasoning model. +Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](../transports/responses.md) documents this boundary, and native Chat capability stripping applies only for an explicit empty declaration or no-reasoning model; an initial JEV null choice separately strips caller effort. Live sideband admission and its bounded upstream handshake follow the [runtime contract](../runtime.md#live-sideband-handshake); the ordinary Responses WebSocket exchange remains separate. diff --git a/structure/providers/jev-decision.md b/structure/providers/jev-decision.md index 041ade80770..894b5be7d36 100644 --- a/structure/providers/jev-decision.md +++ b/structure/providers/jev-decision.md @@ -78,15 +78,16 @@ target/effort allowlist stay authoritative. The note reaches TypeSafe with each applicable decision, so operators must keep secrets and private paths out of it. An absent note leaves the prior decision payload shape intact. -`src/server/responses/core-combo.ts` computes current eligibility, asks JEV once for the initial pick, -applies the validated effort, and removes caller `service_tier` on the Responses child body. The -opt-in native Chat lane builds a separate body in `src/server/responses/core-combo-native.ts` and -does not apply that JEV effort override. A retryable child failure re-enters the ordinary Combo -fallback loop from the untouched request without another JEV call. Request and attempt -`requestedEffort` labels retain an applied differing JEV effort as `caller->forced`, followed by -later child transitions; native children omit the unapplied override and preserve their own -transitions. Absent or unchanged forced efforts keep the existing label. +`src/server/responses/core-combo.ts` computes current eligibility and asks JEV once for the initial pick. +The validated effort shapes the first Responses child or the separate opt-in native Chat body in +`src/server/responses/core-combo-native.ts`; both remove caller `service_tier` and conflicting effort controls. +Explicit null strips effort using a non-force path, even when the Combo has a forced default; the ordinary +force-default invariant is unchanged. Provider pins, caps and wire normalization still run afterward. +A retryable failure rebuilds later targets from the untouched request under ordinary Combo policy, without +another JEV call. Request and attempt `requestedEffort` labels retain a differing applied JEV effort as +`caller->forced`, followed by child transitions on either lane; null or unchanged effort keeps the existing label. `src/server/responses/combo-requested-effort.ts` owns this pure label calculation. +`tests/responses/jev-initial-effort-wire.test.ts` captures both lanes across all three decision backends. Each target may carry an optional non-empty `reasoningEfforts` allowlist. Omission keeps the backward-compatible all-advertised behavior; a present list is intersected with current capabilities, and an empty intersection removes that target from the JEV choice map rather than broadening it. diff --git a/structure/providers/xai-grok.md b/structure/providers/xai-grok.md index 11667bbf65a..c2c4dedb222 100644 --- a/structure/providers/xai-grok.md +++ b/structure/providers/xai-grok.md @@ -215,7 +215,7 @@ do not alter xAI discovery, model metadata, or transport behavior. Native Chat applies qualifying effort ceilings independently of model pins; pin selection precedes the cap and only pins or cap rewrites enter wire mapping. The [catalog effort contract](../catalog.md#ultra-reasoning-level) records the V1/compaction exemptions and caller-preservation boundary. -Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](../transports/responses.md) documents this boundary, and native Chat removes effort only for an explicit empty declaration or no-reasoning model. +Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](../transports/responses.md) documents this boundary, and native Chat capability stripping applies only for an explicit empty declaration or no-reasoning model; an initial JEV null choice separately strips caller effort. The [explicit model-capability contract](../config.md#explicit-per-model-capability-declarations) preserves operator declarations through provider storage and catalog capture; it does not infer upstream capability or change this surface's routing behavior. diff --git a/structure/subagents.md b/structure/subagents.md index ee615a61e22..6fed00fe0b5 100644 --- a/structure/subagents.md +++ b/structure/subagents.md @@ -503,7 +503,7 @@ The account history response can include a [low-confidence effective capacity es Account quota surfaces use [safe probe diagnostics](transports/inventory.md#account-quota-failure-diagnostics) separately from quota validity, credential health and routing authority. -Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](transports/responses.md) documents this boundary, and native Chat removes effort only for an explicit empty declaration or no-reasoning model. +Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](transports/responses.md) documents this boundary, and native Chat capability stripping applies only for an explicit empty declaration or no-reasoning model; an initial JEV null choice separately strips caller effort. Live sideband admission and its bounded upstream handshake follow the [runtime contract](runtime.md#live-sideband-handshake); the ordinary Responses WebSocket exchange remains separate. diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 9520b144d5d..53acb4c279c 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -541,9 +541,9 @@ not `openai-chat` is decided without a copy. An eligible child is dispatched thr already opened (its opening stays hand-rolled: the ordinal comes from the parent context while the active attempt and requested effort land on the child context, which `beginInferenceAttempt` does not express). The child gets the combo's per-target send budget, the client's abort signal -and the turn lease, and the configured combo effort policy mapped onto `reasoning_effort` through `concreteComboRequestBody`. -The initial JEV override changes only the separate Responses body; native labels omit it and keep native child transitions -([JEV Decision Routing](../providers/jev-decision.md)). Its attempt path is native (`[chat, chat]`) and its answer is marked `chat`. +and the turn lease. Ordinary Combo policy maps onto `reasoning_effort` through `concreteComboRequestBody`; the first JEV choice overrides it, including explicit null. +Initial shaping removes conflicting controls and caller `service_tier` before provider pins/caps and wire normalization; labels retain applied transitions ([JEV Decision Routing](../providers/jev-decision.md)). +Later targets use ordinary policy from the original body. Its attempt path is native (`[chat, chat]`) and its answer is marked `chat`. Send accounting: the combo's hop reservation already booked the target's first send, so the native child opens no spend tracker; it reports each physical send to the target budget (the first diff --git a/structure/transports/streaming-health.md b/structure/transports/streaming-health.md index 3e01e3894d2..3d4f57ab9ac 100644 --- a/structure/transports/streaming-health.md +++ b/structure/transports/streaming-health.md @@ -324,7 +324,7 @@ Pool quota producers and account commands follow the [bounded raw-observation co Account quota surfaces use [safe probe diagnostics](inventory.md#account-quota-failure-diagnostics) separately from quota validity, credential health and routing authority. -Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](responses.md) documents this boundary, and native Chat removes effort only for an explicit empty declaration or no-reasoning model. +Combo child requests normalize effort and thinking controls against the selected target while retaining reasoning summaries; strict unknown targets preserve caller controls. The [Responses transport owner](responses.md) documents this boundary, and native Chat capability stripping applies only for an explicit empty declaration or no-reasoning model; an initial JEV null choice separately strips caller effort. Live sideband admission and its bounded upstream handshake follow the [runtime contract](../runtime.md#live-sideband-handshake); the ordinary Responses WebSocket exchange remains separate. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index e7953e2947a..9d212697c7a 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -224,7 +224,7 @@ "protocol-ingress-guard.test.ts": "responses", "protocol-direct-encoders-chat.test.ts": "responses", "protocol-direct-encoders-stall-disabled.test.ts": "responses", "protocol-direct-encoders-local-upstream.test.ts": "responses", - "protocol-direct-encoders-messages.test.ts": "responses", "chat-native-combo.test.ts": "responses", + "protocol-direct-encoders-messages.test.ts": "responses", "chat-native-combo.test.ts": "responses", "jev-initial-effort-wire.test.ts": "responses", "messages-native-eligibility.test.ts": "responses", "messages-native-bridge-policy.test.ts": "responses", "messages-native-oauth-eligibility.test.ts": "responses", "chat-inbound-reasoning-none.test.ts": "responses", "chat-native-decline-reason.test.ts": "responses", "chat-inbound-reasoning-replay.test.ts": "responses", diff --git a/tests/responses/chat-native-combo.test.ts b/tests/responses/chat-native-combo.test.ts index 37a109d2bed..1c17d98191b 100644 --- a/tests/responses/chat-native-combo.test.ts +++ b/tests/responses/chat-native-combo.test.ts @@ -177,10 +177,10 @@ describe("native Chat candidates in a combo", () => { }); test.each([ - ["native", "low", "unset", "high"], - ["native", "xhigh", "unset", "high"], + ["native", "low", "unset", "high->low"], + ["native", "xhigh", "unset", "high->xhigh"], ["native", "high", "unset", "high"], - ["native", "low", "medium", "high->medium"], + ["native", "low", "medium", "high->low->medium"], ["bridge", "low", "unset", "high->low"], ["bridge", "xhigh", "unset", "high->xhigh"], ["bridge", "low", "medium", "high->low->medium"], @@ -205,7 +205,7 @@ describe("native Chat candidates in a combo", () => { expect(response.status).toBe(200); expect(a.bodies).toHaveLength(1); - expect(a.bodies[0]!.reasoning_effort).toBe(pin ?? (native ? "high" : selected)); + expect(a.bodies[0]!.reasoning_effort).toBe(pin ?? selected); expect(rows).toHaveLength(1); expect(rows[0]!.jevDecision?.selected).toEqual({ provider: "a", model: "m1", effort: selected }); expect(rows[0]!.requestedEffort).toBe(label); diff --git a/tests/responses/jev-initial-effort-wire.test.ts b/tests/responses/jev-initial-effort-wire.test.ts new file mode 100644 index 00000000000..513aeb406df --- /dev/null +++ b/tests/responses/jev-initial-effort-wire.test.ts @@ -0,0 +1,299 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleChatCompletions } from "../../src/server/chat-completions"; +import { handleResponses } from "../../src/server/responses/core"; +import { getActiveTurnCount } from "../../src/server/lifecycle"; +import type { DataPlaneAdmission } from "../../src/server/auth-cors"; +import { clearComboSelectionState, clearComboTargetCooldowns, concreteComboRequestBody } from "../../src/combos"; +import { getRequestLogEntries, type RequestLogContext } from "../../src/server/request-log"; +import { closeRequestHistoryIndex } from "../../src/routing/history/indexer"; +import { clearResponseStateForTests, flushResponseState } from "../../src/responses/state"; +import { clearKeyCooldowns } from "../../src/providers/key-failover"; +import { resetProviderRequestPacingForTest } from "../../src/providers/request-pacing"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { chatStream, chatSuccess } from "../helpers/combo-failover-upstream"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; + +type Rec = Record; +type Lane = "native" | "responses"; +type Backend = "typesafe" | "systemone" | "model"; +let home: string; +let prior: string | undefined; +let codex: IsolatedCodexHome; +let release: () => void; +const servers: Array> = []; +beforeEach(() => { + prior = process.env.OPENCODEX_HOME; + home = mkdtempSync(join(tmpdir(), "jev-effort-wire-")); + process.env.OPENCODEX_HOME = home; + codex = installIsolatedCodexHome("jev-effort-codex-"); + release = acquireOwnedSpendHome(); + clearComboSelectionState(); clearComboTargetCooldowns(); clearKeyCooldowns(); +}); +afterEach(async () => { + release(); + for (const server of servers.splice(0)) await server.stop(true); + closeRequestHistoryIndex(); + await flushResponseState(); clearResponseStateForTests(); + clearComboSelectionState(); clearComboTargetCooldowns(); clearKeyCooldowns(); + resetProviderRequestPacingForTest(); + if (prior === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = prior; + codex.restore(); removeTreeWithRetry(home); +}); + +/** Start a teardown-owned loopback upstream that records bodies and headers before replying. */ +function upstream(reply: (body: Rec) => Response | Promise = body => body.stream === true ? chatStream("fixture answer") : chatSuccess("fixture answer")) { + const bodies: Rec[] = []; + const headers: Headers[] = []; + const server = Bun.serve({ hostname: "127.0.0.1", port: 0, async fetch(req) { + const body = await req.json() as Rec; + bodies.push(body); headers.push(req.headers); + return reply(body); + } }); + servers.push(server); + return { bodies, headers, baseUrl: new URL("/v1", server.url).href }; +} +/** Build a synthetic Chat provider with an overridable effort ladder and loopback transport. */ +function provider(baseUrl: string, extra: Partial = {}): OcxProviderConfig { + return { adapter: "openai-chat", baseUrl, allowPrivateNetwork: true, apiKey: "fixture-inference-key", + authMode: "key", liveModels: false, models: ["m"], reasoningEfforts: ["low", "medium", "high"], ...extra }; +} +/** Configure one JEV backend and ingress lane with captured inference and decision requests. */ +function fixture(lane: Lane, backend: Backend, effort: "low" | "high" | null, failure = false) { + const inference = upstream(); + const choice = `a/m:${effort ?? "none"}`; + const judge = upstream(body => failure + ? Response.json({ error: { message: "fixture judge unavailable" } }, { status: 500 }) + : backend === "model" ? chatStream(JSON.stringify({ choice })) + : Response.json({ answers: { route: { choice } } })); + const decisionBodies: Rec[] = []; + const config: OcxConfig = { + port: 0, defaultProvider: "a", + providers: { a: provider(inference.baseUrl, effort === null ? { reasoningEfforts: [] } : {}) }, + combos: { auto: { strategy: "jev", targets: [{ provider: "a", model: "m" }] } }, + ...(lane === "native" ? { protocols: { rollout: { nativeChatCombos: true } } } : {}), + }; + if (backend === "model") { + config.providers.judge = provider(judge.baseUrl, { apiKey: "fixture-judge-key", reasoningEfforts: ["low"] }); + config.combos!.auto!.decisionModel = "judge/m"; + } else { + const id = backend === "typesafe" ? "jev" : "decider"; + config.providers[id] = { adapter: "jev-decision", liveModels: false, apiKey: "fixture-service-key", + baseUrl: backend === "typesafe" ? "https://api.typesafe.ai/v1/systemone" : new URL("systemone", judge.baseUrl + "/").href, + ...(backend === "systemone" ? { defaultModel: "fixture-judge", allowPrivateNetwork: true } : {}), + ...(backend === "typesafe" ? { fetch: (async (_url: RequestInfo | URL, init?: RequestInit) => { + decisionBodies.push(JSON.parse(String(init?.body)) as Rec); + return failure ? Response.json({ error: {} }, { status: 500 }) : Response.json({ answers: { route: { choice } } }); + }) as unknown as typeof fetch } : {}), + }; + if (backend === "systemone") { + config.combos!.auto!.decisionProvider = id; + // System One requires two expanded options; a second no-effort target is still eligible. + config.providers.b = provider(inference.baseUrl, effort === null ? { reasoningEfforts: [] } : {}); + config.combos!.auto!.targets.push({ provider: "b", model: "m" }); + } + } + return { config, inference, judge, decisionBodies }; +} +/** Dispatch and drain a fixture request, returning its response and lane-owned log metadata. */ +async function send(lane: Lane, config: OcxConfig, extra: Rec = {}, signal?: AbortSignal, headers: Rec = {}, admission?: DataPlaneAdmission) { + const body = lane === "native" + ? { model: "combo/auto", messages: [{ role: "user", content: "Solve the fixture task." }], reasoning_effort: "high", stream: false, ...extra } + : { model: "combo/auto", input: "Solve the fixture task.", reasoning: { effort: "high", summary: "auto" }, stream: false, ...extra }; + const request = new Request(`http://localhost/v1/${lane === "native" ? "chat/completions" : "responses"}`, { + method: "POST", headers: { "content-type": "application/json", authorization: "Bearer fixture-caller-key", + "x-codex-parent-thread-id": "fixture-parent-thread", ...headers } as HeadersInit, + body: JSON.stringify(body), ...(signal ? { signal } : {}), + }); + const log: RequestLogContext = { model: "", provider: "" }; + const requestId = crypto.randomUUID(); + const response = lane === "native" + ? await handleChatCompletions(request, config, log, { requestId, start: Date.now(), admission }) + : await handleResponses(request, config, log, { admission, abortSignal: request.signal }); + await response.text(); + const row = lane === "native" ? getRequestLogEntries().find(row => row.requestId === requestId)! : log; + return { response, row }; +} + +for (const lane of ["native", "responses"] as const) { + for (const backend of ["typesafe", "systemone", "model"] as const) { + for (const effort of ["low", "high", null] as const) { + test(`${lane}/${backend}: actual initial wire effort ${effort}`, async () => { + const f = fixture(lane, backend, effort); + const { response, row } = await send(lane, f.config, { service_tier: "priority", thinking: { type: "enabled" }, thinking_budget: 999, + ...(lane === "native" ? { reasoning: { effort: "medium", summary: "auto" }, reasoning_effort: "none" } : { reasoning_effort: "none" }) }); + expect(response.status).toBe(200); + expect(f.inference.bodies).toHaveLength(1); + const wire = f.inference.bodies[0]!; + expect(wire.reasoning_effort).toBe(effort ?? undefined); + expect(wire.service_tier).toBeUndefined(); + expect(wire.thinking).toBeUndefined(); expect(wire.thinking_budget).toBeUndefined(); + expect((wire.reasoning as Rec | undefined)?.effort).toBeUndefined(); + expect(row.jevDecision).toMatchObject({ backend, gate: "apply", selected: { provider: "a", model: "m", effort } }); + expect(row.attempts).toHaveLength(1); expect(row.attempts![0]!.sendCount).toBe(1); + // row is RequestLogEntry | RequestLogContext; only the persisted entry carries protocolTrace. + if (lane === "native") expect("protocolTrace" in row ? row.protocolTrace?.attempts?.[0]?.mode : undefined).toBe("native"); + if (backend === "model") { + expect(f.judge.bodies).toHaveLength(1); + expect(f.judge.headers[0]!.get("authorization")).toBe("Bearer fixture-judge-key"); + expect(f.judge.headers[0]!.get("x-codex-parent-thread-id")).toBeNull(); + expect(f.judge.bodies[0]!.tools).toBeUndefined(); + expect(f.judge.bodies[0]!.max_tokens ?? f.judge.bodies[0]!.max_completion_tokens).toBe(1024); + } else expect(backend === "typesafe" ? f.decisionBodies : f.judge.bodies).toHaveLength(1); + }, 30_000); + } + } + test(`${lane}: unknown effort ladder supports global fail-open null without force defaults`, async () => { + const f = fixture(lane, "model", null, true); + delete f.config.providers.a!.reasoningEfforts; + Object.assign(f.config.combos!.auto!, { defaultEffort: "high", defaultEffortMode: "force" }); + const { response, row } = await send(lane, f.config); + expect(response.status).toBe(200); expect(f.inference.bodies).toHaveLength(1); + expect(f.inference.bodies[0]!.reasoning_effort).toBeUndefined(); + expect(row.jevDecision).toMatchObject({ gate: "http", selected: { effort: null } }); + }); + for (const failure of [false, true]) { + test(`${lane}: force-default plus ${failure ? "failed judge global" : "selected"} null dispatches`, async () => { + const f = fixture(lane, "model", null, failure); + Object.assign(f.config.combos!.auto!, { defaultEffort: "high", defaultEffortMode: "force" }); + const { response, row } = await send(lane, f.config); + expect(response.status).toBe(200); + expect(f.inference.bodies).toHaveLength(1); + expect(f.inference.bodies[0]!.reasoning_effort).toBeUndefined(); + expect(row.jevDecision).toMatchObject({ gate: failure ? "http" : "apply", selected: { effort: null } }); + }, 30_000); + } +} + +test.each(["none", "minimal"])("native caller sentinel %s cannot override the initial decision", async caller => { + const f = fixture("native", "typesafe", "high"); + const { response, row } = await send("native", f.config, { reasoning_effort: caller }); + expect(response.status).toBe(200); expect(f.inference.bodies[0]!.reasoning_effort).toBe("high"); + expect(row.requestedEffort).toBe(`${caller}->high`); +}); + +test.each(["effortCap", "subagentEffortCap"] as const)("native JEV effort retains provider pin, %s and honest applied transitions", async cap => { + const f = fixture("native", "model", "low"); + f.config.providers.a!.pinnedReasoningEffort = "high"; + f.config[cap] = "medium"; + const { row } = await send("native", f.config, {}, undefined, { "x-openai-subagent": "collab_spawn" }); + expect(f.inference.bodies[0]!.reasoning_effort).toBe("medium"); + expect(row.jevDecision?.selected.effort).toBe("low"); + expect(row.requestedEffort).toBe("high->low->high->medium"); + expect(row.attempts![0]!.requestedEffort).toBe(row.requestedEffort); +}); + +test("native initial null still permits a downstream provider pin on an unknown ladder", async () => { + const f = fixture("native", "model", null); + delete f.config.providers.a!.reasoningEfforts; + f.config.providers.a!.pinnedReasoningEffort = "high"; + const { response, row } = await send("native", f.config); + expect(response.status).toBe(200); expect(f.inference.bodies[0]!.reasoning_effort).toBe("high"); + expect(row.jevDecision?.selected.effort).toBeNull(); expect(row.requestedEffort).toBe("high"); +}); + +test("native provider gateway effort normalization still runs after the JEV choice", async () => { + const f = fixture("native", "typesafe", "low"); + f.config.providers.a!.reasoningWireFormat = "gateway-object"; + await send("native", f.config); + expect(f.inference.bodies[0]!.reasoning_effort).toBeUndefined(); + expect(f.inference.bodies[0]!.reasoning).toEqual({ enabled: true, effort: "low" }); +}); + +for (const lane of ["native", "responses"] as const) { + test(`${lane}: retry targets use original effort and only one classification`, async () => { + const f = fixture(lane, "model", "low"); + const first = upstream(() => Response.json({ error: { message: "fixture unavailable" } }, { status: 503 })); + f.config.providers.a!.baseUrl = first.baseUrl; + f.config.providers.a!.transientRetryOn5xx = { attempts: 1 }; + f.config.providers.b = provider(f.inference.baseUrl); + f.config.combos!.auto!.targets.push({ provider: "b", model: "m" }); + const { response, row } = await send(lane, f.config); + expect(response.status).toBe(200); expect(f.judge.bodies).toHaveLength(1); + expect(first.bodies.length).toBeGreaterThan(0); + for (const body of first.bodies) expect(body.reasoning_effort).toBe("low"); + expect(f.inference.bodies).toHaveLength(1); expect(f.inference.bodies[0]!.reasoning_effort).toBe("high"); + // Direct Responses ingress has no outer final-log owner to finish the accepted attempt. + expect(row.attempts!.map(attempt => attempt.status)).toEqual([503, lane === "native" ? 200 : 0]); + expect(row.attempts![0]!.sendCount).toBe(first.bodies.length); + expect(row.attempts![1]!.sendCount).toBe(1); + }, 30_000); +} + +test("ordinary force invariant is not relaxed", () => { + expect(() => concreteComboRequestBody({}, { provider: "a", model: "m" }, null, [], "strict", "force")) + .toThrow("force combo default effort requires a valid defaultEffort"); +}); + +for (const lane of ["native", "responses"] as const) { + test(`${lane}: unallowlisted service effort fails open without applying the rejected effort`, async () => { + const f = fixture(lane, "typesafe", "high"); + f.config.combos!.auto!.targets[0]!.reasoningEfforts = ["low"]; + const { row } = await send(lane, f.config); + expect(row.jevDecision).toMatchObject({ gate: "invalid", selected: { effort: "low" } }); + expect(f.inference.bodies[0]!.reasoning_effort).toBe("low"); + }); + test(`${lane}: restrictive empty intersection never receives a decision effort`, async () => { + const f = fixture(lane, "model", "high"); + f.config.providers.a!.reasoningEfforts = []; + f.config.combos!.auto!.targets[0]!.reasoningEfforts = ["high"]; + f.config.providers.b = provider(f.inference.baseUrl, { reasoningEfforts: ["low"] }); + f.config.combos!.auto!.targets.push({ provider: "b", model: "m" }); + const { row } = await send(lane, f.config); + expect(row.jevDecision).toMatchObject({ gate: "invalid", selected: { provider: "b", effort: "low" } }); + expect(f.inference.bodies).toHaveLength(1); + expect(f.inference.bodies[0]!.reasoning_effort).toBe("low"); + }); + test(`${lane}: actual detached model admission denial sends no judge bytes`, async () => { + const f = fixture(lane, "model", "high"); + f.config.apiKeys = [{ id: "fixture-scope", key: "fixture-scoped-key", name: "fixture", createdAt: "2026-01-01T00:00:00Z", allowedProviders: ["a"] }]; + const { response, row } = await send(lane, f.config, {}, undefined, {}, { kind: "configured", keyId: "fixture-scope", source: "bearer" }); + expect(response.status).toBe(200); expect(f.judge.bodies).toHaveLength(0); + expect(f.inference.bodies).toHaveLength(1); expect(f.inference.bodies[0]!.reasoning_effort).toBe("medium"); + expect(row.jevDecision).toMatchObject({ backend: "model", gate: "http", selected: { effort: "medium" } }); + expect(getActiveTurnCount()).toBe(0); + }); + test(`${lane}: cancellation during detached judge prevents inference and releases its turn`, async () => { + const f = fixture(lane, "model", "low"); + let entered!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + let finish!: () => void; + const hold = new Promise(resolve => { finish = resolve; }); + const judge = upstream(async () => { entered(); await hold; return chatStream('{"choice":"a/m:low"}'); }); + f.config.providers.judge!.baseUrl = judge.baseUrl; + const controller = new AbortController(); + const pending = send(lane, f.config, {}, controller.signal); + await started; + const reason = new Error("fixture caller cancellation"); + controller.abort(reason); finish(); + const { response } = await pending; + expect(controller.signal.reason).toBe(reason); expect(response.status).toBe(499); + expect(f.inference.bodies).toHaveLength(0); expect(judge.bodies).toHaveLength(1); + expect(getActiveTurnCount()).toBe(0); + }, 30_000); +} + +test("native model judge with explicit send policy and inference each settle one distinct physical spend", async () => { + const f = fixture("native", "model", "low"); + // Generic Chat adapters report physical sends through the explicit transient policy path. + // Unconfigured generic-adapter spend reporting is outside this initial-effort repair. + f.config.providers.judge!.transientRetryOn5xx = { attempts: 1 }; + const { row } = await send("native", f.config); + const records = readFileSync(join(home, "spend-ledger.jsonl"), "utf8").trim().split("\n").map(line => JSON.parse(line) as Rec); + // One-send turns keep their newest booking open until terminal settlement; a separate + // dispatch record is only required once a later send confirms an older reservation. + const reserved = records.filter(record => record.kind === "reserve"); + const settled = records.filter(record => record.kind === "settle"); + expect(f.judge.bodies).toHaveLength(1); expect(f.inference.bodies).toHaveLength(1); + expect(reserved).toHaveLength(2); expect(settled).toHaveLength(2); + expect(new Set(settled.map(record => record.send)).size).toBe(2); + expect(settled.map(record => record.send)).toEqual(reserved.map(record => record.send)); + expect(settled.map(record => record.tokens)).toEqual([3, 3]); + expect(row.attempts).toHaveLength(1); expect(row.attempts![0]!.sendCount).toBe(1); + expect(getActiveTurnCount()).toBe(0); +}); diff --git a/tests/server/combo-requested-effort.test.ts b/tests/server/combo-requested-effort.test.ts index 6a937174203..db7e44a17c6 100644 --- a/tests/server/combo-requested-effort.test.ts +++ b/tests/server/combo-requested-effort.test.ts @@ -12,6 +12,7 @@ describe("Combo requested effort labels", () => { ["high", "low->high", "low", "high->low->high"], ["high", "high", "high", "high"], ["high", "high->low", "high", "high->low"], + ["high", "low->low", "low", "high->low"], ["high", "medium", undefined, "high"], ["high", undefined, undefined, "high"], ["high", "medium->low", undefined, "high->low"],