Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs-site/src/content/docs/guides/combos.md
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,13 @@ target/effort, a coarse decision gate, latency, optional confidence/probability,
Automated tests use mocked TypeSafe responses plus a no-key fail-open smoke; a live TypeSafe decision
requires an operator-supplied key and is not run implicitly.

When JEV's selected effort is applied to the Responses child, request and attempt logs show a
differing effort as `high->xhigh` or `high->low`. Later effort adjustments remain in the chain,
such as `high->xhigh->medium`. An unchanged effort or a target without effort control keeps the
existing label; fallback attempts do not inherit the initial JEV effort. With the opt-in
`protocols.rollout.nativeChatCombos` path, native Chat children currently do not apply JEV's
selected effort; their labels retain the caller's effort and any native child adjustments instead.

After the Combo has served requests, open **Models → Combos → jev-auto → Stats** to inspect JEV's
picks without replacing the normal model picker or Usage page. The tab separates backend-reported decision
tokens from tokens reported by physical model sends, and shows decision gates, fail-open picks,
Expand Down
2 changes: 1 addition & 1 deletion scripts/test-layout/layout.json
Original file line number Diff line number Diff line change
Expand Up @@ -1674,7 +1674,7 @@
"server-images-pool-admission.test.ts": "server",
"server-images.test.ts": "server",
"server-key-failover-e2e.test.ts": "server",
"server-jev-combo-e2e.test.ts": "server",
"server-jev-combo-e2e.test.ts": "server", "combo-requested-effort.test.ts": "server",
"server-kiro-completion-e2e.test.ts": "server",
"server-kiro-device-login.test.ts": "server", "server-kiro-oauth-401-replay.test.ts": "server",
"server-kiro-refusal-e2e.test.ts": "server", "server-live-existing-call.test.ts": "server",
Expand Down
14 changes: 14 additions & 0 deletions src/server/responses/combo-requested-effort.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/** Restore the caller label while retaining JEV's forced effort and later child transitions. */
export function comboRequestedEffortLabel(
originalRequestedEffort: string | undefined,
childRequestedEffort: string | undefined,
forcedEffort?: string | null,
): string | undefined {
if (originalRequestedEffort === undefined) return childRequestedEffort;
const transitionIndex = childRequestedEffort?.indexOf("->") ?? -1;
const laterTransitions = transitionIndex >= 0 ? childRequestedEffort!.slice(transitionIndex) : "";
const forcedTransition = forcedEffort != null && forcedEffort !== originalRequestedEffort
? `->${forcedEffort}`
: "";
return `${originalRequestedEffort}${forcedTransition}${laterTransitions}`;
}
12 changes: 5 additions & 7 deletions src/server/responses/core-combo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ import { mandatoryResponsesReasoningReplayUnavailable } from "./core-replay";
import { settleOperatorReplacement } from "../../lib/upstream-retry";
import { createComboProtocolLanes, dispatchNativeComboChild } from "./core-combo-native";
import { createJevModelInvoker } from "./jev-model-invoke";
import { comboRequestedEffortLabel } from "./combo-requested-effort";
import { clientWireOf } from "../inference/client-wire";

/**
Expand Down Expand Up @@ -656,13 +657,9 @@ export async function executeComboResponses(
&& isDeclaredReasoningEffort(originalRequestedEffortValue)
? originalRequestedEffortValue
: undefined;
const restoreOriginalRequestedEffort = (childLog: RequestLogContext): void => {
const restoreOriginalRequestedEffort = (childLog: RequestLogContext, forcedEffort?: string | null): void => {
if (originalRequestedEffort === undefined) return;
const normalizedRequestedEffort = childLog.requestedEffort;
const transitionIndex = normalizedRequestedEffort?.indexOf("->") ?? -1;
childLog.requestedEffort = transitionIndex >= 0
? `${originalRequestedEffort}${normalizedRequestedEffort!.slice(transitionIndex)}`
: originalRequestedEffort;
childLog.requestedEffort = comboRequestedEffortLabel(originalRequestedEffort, childLog.requestedEffort, forcedEffort);
recordAttemptRequestedEffort(childLog);
};

Expand Down Expand Up @@ -907,7 +904,8 @@ export async function executeComboResponses(
onNativePassthroughCancel: callbackGate.onCancel,
onResponseComplete: callbackGate.onResponseComplete,
});
restoreOriginalRequestedEffort(childLog);
// Native Chat rebuilds its own body and does not apply the initial JEV effort.
restoreOriginalRequestedEffort(childLog, nativeChild ? undefined : initialJevDecision?.effort);
} catch (error) {
callbackGate.discard();
if (options.abortSignal?.aborted) {
Expand Down
12 changes: 9 additions & 3 deletions structure/providers/jev-decision.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,15 @@ applicable decision, so operators must keep secrets and private paths out of it.
An absent note leaves the prior decision payload shape intact.

`src/server/responses/core-combo.ts` computes current eligibility, asks JEV once for the initial pick,
applies the validated effort, and removes caller `service_tier` for that child. A retryable child
failure re-enters the ordinary Combo fallback loop from the untouched request without another JEV
call. Each target may carry an optional non-empty `reasoningEfforts` allowlist. Omission keeps the
applies the validated effort, and removes caller `service_tier` on the Responses child body. The
opt-in native Chat lane builds a separate body in `src/server/responses/core-combo-native.ts` and
does not apply that JEV effort override. A retryable child failure re-enters the ordinary Combo
fallback loop from the untouched request without another JEV call. Request and attempt
`requestedEffort` labels retain an applied differing JEV effort as `caller->forced`, followed by
later child transitions; native children omit the unapplied override and preserve their own
transitions. Absent or unchanged forced efforts keep the existing label.
`src/server/responses/combo-requested-effort.ts` owns this pure label calculation.
Each target may carry an optional non-empty `reasoningEfforts` allowlist. Omission keeps the
backward-compatible all-advertised behavior; a present list is intersected with current capabilities,
and an empty intersection removes that target from the JEV choice map rather than broadening it.
Direct models and every other Combo strategy bypass this path. The shared Combo editor owns the GUI
Expand Down
8 changes: 4 additions & 4 deletions structure/transports/responses.md
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ is composed from the following owners in `src/server/responses/`; none is a gene
| `adapter-continuation.ts`, `adapter-delivery.ts` | Continuation event sources and final streaming/buffered bridging; a streamed turn with a `clientEncoder` option is handed to `src/server/inference/client-encoder-delivery.ts` instead of the bridge. |

Reusable helpers live in `core-auth.ts`, `core-codex-account.ts`, `core-combo.ts`,
`core-combo-failure.ts`, `core-combo-native.ts`, `core-errors.ts`, `core-lifetime.ts`, `core-normalize.ts`,
`core-combo-failure.ts`, `core-combo-native.ts`, `combo-requested-effort.ts`, `core-errors.ts`, `core-lifetime.ts`, `core-normalize.ts`,
`core-opaque-recovery.ts` and `core-replay.ts`. `core-options.ts` owns the public option types
and small composition contracts, including [finite model-refusal evidence](../providers-and-adapters.md#combo-model-refusal-evidence). Existing public helper names are re-exported by `core.ts`.
Adapter construction remains with the existing registry; `fetch-helpers.ts` remains a leaf. For Kiro OAuth with load settings, `request-transport.ts` acquires a lease on the admitted account and transfers it before a reactive replacement send; cancellation permanently fences the request holder so recovery cannot install a late lease after abort cleanup. `core.ts` and `core-lifetime.ts` release the lease on returned-body completion, error, or cancellation, outside the inner admission `finally`.
Expand Down Expand Up @@ -541,9 +541,9 @@ not `openai-chat` is decided without a copy. An eligible child is dispatched thr
already opened (its opening stays hand-rolled: the ordinal comes from the parent context while the
active attempt and requested effort land on the child context, which `beginInferenceAttempt`
does not express). The child gets the combo's per-target send budget, the client's abort signal
and the turn lease, and the combo's reasoning-effort policy mapped onto `reasoning_effort`
through `concreteComboRequestBody`, so the two lanes cannot disagree on effort. It records its
attempt path as native (`[chat, chat]`) and its answer is marked `chat`.
and the turn lease, and the configured combo effort policy mapped onto `reasoning_effort` through `concreteComboRequestBody`.
The initial JEV override changes only the separate Responses body; native labels omit it and keep native child transitions
([JEV Decision Routing](../providers/jev-decision.md)). Its attempt path is native (`[chat, chat]`) and its answer is marked `chat`.

Send accounting: the combo's hop reservation already booked the target's first send, so the native
child opens no spend tracker; it reports each physical send to the target budget (the first
Expand Down
1 change: 1 addition & 0 deletions tests/fixtures/test-layout-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -953,6 +953,7 @@
"server-images-bodyless-content-length.test.ts": "server", "server-images-pool-admission.test.ts": "server",
"server-images.test.ts": "server", "server-key-failover-e2e.test.ts": "server",
"server-jev-combo-e2e.test.ts": "server", "server-kiro-completion-e2e.test.ts": "server",
"combo-requested-effort.test.ts": "server",
"server-kiro-device-login.test.ts": "server", "server-kiro-oauth-401-replay.test.ts": "server",
"server-kiro-refusal-e2e.test.ts": "server", "server-live-existing-call.test.ts": "server",
"server-live-frame-log.test.ts": "server",
Expand Down
1 change: 1 addition & 0 deletions tests/helpers/responses-core-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export const RESPONSES_CORE_MODULES = [
"core-auth.ts",
"core-normalize.ts",
"core-combo.ts",
"combo-requested-effort.ts",
"core-combo-native.ts",
// Reached from core-combo.ts: the JEV model backend's internal decision turn.
"jev-model-invoke.ts",
Expand Down
38 changes: 38 additions & 0 deletions tests/responses/chat-native-combo.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,44 @@ describe("native Chat candidates in a combo", () => {
});
});

test.each([
["native", "low", "unset", "high"],
["native", "xhigh", "unset", "high"],
["native", "high", "unset", "high"],
["native", "low", "medium", "high->medium"],
["bridge", "low", "unset", "high->low"],
["bridge", "xhigh", "unset", "high->xhigh"],
["bridge", "low", "medium", "high->low->medium"],
] as const)("JEV wire effort: lane=%s, selected=%s, pin=%s, label=%s", async (lane, selected, pinned, label) => {
const native = lane === "native";
const pin = pinned === "unset" ? undefined : pinned;
const a = upstream(body => body.stream === true ? chatStream("effort fixture") : twoChoiceCompletion());
const config = comboConfig({
a: provider("openai-chat", a.baseUrl, {
models: ["m1"], liveModels: false, reasoningEfforts: ["low", "medium", "high", "xhigh"],
...(pin ? { pinnedReasoningEffort: pin } : {}),
}),
jev: {
adapter: "jev-decision", baseUrl: "https://api.typesafe.ai/v1/systemone",
authMode: "key", apiKey: "decision-fixture", liveModels: false,
fetch: (async () => Response.json({ answers: { route: { choice: `a/m1:${selected}` } } })) as typeof fetch,
},
}, [{ provider: "a", model: "m1" }], native ? NATIVE_ON : undefined);
config.combos!.pair!.strategy = "jev";

const { response, rows } = await send(config, { stream: false, reasoning_effort: "high" });

expect(response.status).toBe(200);
expect(a.bodies).toHaveLength(1);
expect(a.bodies[0]!.reasoning_effort).toBe(pin ?? (native ? "high" : selected));
expect(rows).toHaveLength(1);
expect(rows[0]!.jevDecision?.selected).toEqual({ provider: "a", model: "m1", effort: selected });
expect(rows[0]!.requestedEffort).toBe(label);
expect(rows[0]!.attempts).toHaveLength(1);
expect(rows[0]!.attempts![0]!.requestedEffort).toBe(label);
expect(rows[0]!.protocolTrace!.attempts![0]!.mode).toBe(native ? "native" : "legacy-bridge");
});

test("a failed native candidate fails over to the bridge within the shared send budget", async () => {
// Five sends on its own ladder, but the combo's per-target budget holds one back for the
// second declared target: the native child may reach its upstream three times, not five.
Expand Down
26 changes: 26 additions & 0 deletions tests/server/combo-requested-effort.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { describe, expect, test } from "bun:test";
import { comboRequestedEffortLabel } from "../../src/server/responses/combo-requested-effort";

describe("Combo requested effort labels", () => {
test.each([
["high", "xhigh", "xhigh", "high->xhigh"],
["high", "low", "low", "high->low"],
["none", "high", "high", "none->high"],
["high", undefined, "xhigh", "high->xhigh"],
["high", "xhigh->medium", "xhigh", "high->xhigh->medium"],
["high", "xhigh->medium->low", "xhigh", "high->xhigh->medium->low"],
["high", "low->high", "low", "high->low->high"],
["high", "high", "high", "high"],
["high", "high->low", "high", "high->low"],
["high", "medium", undefined, "high"],
["high", undefined, undefined, "high"],
["high", "medium->low", undefined, "high->low"],
["high", "medium->low->none", null, "high->low->none"],
["high", undefined, null, "high"],
[undefined, "xhigh", "xhigh", "xhigh"],
[undefined, "xhigh->low", "xhigh", "xhigh->low"],
[undefined, undefined, "xhigh", undefined],
] as const)("caller %s, child %s, forced %s produces %s", (original, child, forced, expected) => {
expect(comboRequestedEffortLabel(original, child, forced)).toBe(expected);
});
});
72 changes: 72 additions & 0 deletions tests/server/server-jev-combo-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,78 @@ describe("JEV Combo runtime", () => {
});
});

test.each(["xhigh", "low", "high"])("records caller high and JEV's forced %s on the request and attempt", async forced => {
const config = makeConfig({ jevFetch: choiceFetch(`sol/gpt-5.6-sol:${forced}`) });
const parentLog: RequestLogContext = { model: "", provider: "" };
const childBodies: Record<string, unknown>[] = [];

const response = await execute(config, (body, childLog) => {
childBodies.push(body);
childLog.requestedEffort = (body.reasoning as { effort: string }).effort;
return success(String(body.model));
}, { reasoning: { effort: "high", summary: "auto" } }, undefined, parentLog);

expect(response.status).toBe(200);
expect(childBodies).toEqual([expect.objectContaining({ reasoning: { effort: forced, summary: "auto" } })]);
const expected = forced === "high" ? "high" : `high->${forced}`;
expect(parentLog.requestedEffort).toBe(expected);
expect(parentLog.attempts).toHaveLength(1);
expect(parentLog.attempts![0]!.requestedEffort).toBe(expected);
});

test("preserves a later child effort transition after JEV's forced effort", async () => {
const config = makeConfig({ jevFetch: choiceFetch("sol/gpt-5.6-sol:xhigh") });
const parentLog: RequestLogContext = { model: "", provider: "" };

const response = await execute(config, (body, childLog) => {
expect(body.reasoning).toMatchObject({ effort: "xhigh" });
childLog.requestedEffort = "xhigh->medium->low";
return success(String(body.model));
}, { reasoning: { effort: "high" } }, undefined, parentLog);

expect(response.status).toBe(200);
expect(parentLog.requestedEffort).toBe("high->xhigh->medium->low");
expect(parentLog.attempts![0]!.requestedEffort).toBe("high->xhigh->medium->low");
});

test("keeps JEV's forced effort on the failed attempt without leaking it into fallback", async () => {
const config = makeConfig({ jevFetch: choiceFetch("sol/gpt-5.6-sol:low") });
const parentLog: RequestLogContext = { model: "", provider: "" };
const efforts: string[] = [];

const response = await execute(config, (body, childLog) => {
const effort = (body.reasoning as { effort: string }).effort;
efforts.push(effort);
childLog.requestedEffort = effort;
return String(body.model).startsWith("sol/")
? Response.json({ error: { message: "temporary outage" } }, { status: 503 })
: success(String(body.model));
}, { reasoning: { effort: "high" } }, undefined, parentLog);

expect(response.status).toBe(200);
expect(efforts).toEqual(["low", "high"]);
expect(parentLog.requestedEffort).toBe("high");
expect(parentLog.attempts?.map(attempt => attempt.requestedEffort)).toEqual(["high->low", "high"]);
});

test("retains today's label when the JEV target has no effort control", async () => {
const config = makeConfig({
jevFetch: choiceFetch("sol/gpt-5.6-sol:none"),
providerOverrides: { sol: { modelReasoningEfforts: { "gpt-5.6-sol": [] } } },
});
const parentLog: RequestLogContext = { model: "", provider: "" };

const response = await execute(config, (body, childLog) => {
expect(body.reasoning).toEqual({ summary: "auto" });
childLog.requestedEffort = undefined;
return success(String(body.model));
}, { reasoning: { effort: "high", summary: "auto" } }, undefined, parentLog);

expect(response.status).toBe(200);
expect(parentLog.requestedEffort).toBe("high");
expect(parentLog.attempts![0]!.requestedEffort).toBe("high");
});

test("routes the initial call to JEV's allowlisted target and keeps direct/catalog rows", async () => {
const jevRequests: Array<Record<string, unknown>> = [];
const config = makeConfig({
Expand Down
Loading