diff --git a/servers/gateway/voice/turn.js b/servers/gateway/voice/turn.js index 481dca76d..b0049390a 100644 --- a/servers/gateway/voice/turn.js +++ b/servers/gateway/voice/turn.js @@ -531,6 +531,28 @@ export function createVoiceTurnRunner(deps) { return { reason: "cold_timeout" }; } + // A primary the box will not serve for its own reasons is retried on the everyday role's own fallback + // roles (voice-keys.js escalateFallbacks, in order). A cold start, an abort or every fallback failing + // keeps the primary's result: the turn degrades exactly as before. + const ESCALATION_FALLBACK_REASONS = new Set(["box_reserved", "serving_class", "acquire_failed", "unresolved"]); + async function readyEscalationChain(keys, primaryKey, db, signal) { + const primary = await readyEscalation(primaryKey, db, signal); + if (primary.adapter) return { ...primary, key: primaryKey }; + if (!ESCALATION_FALLBACK_REASONS.has(primary.reason)) return primary; + const list = Array.isArray(keys?.escalateFallbacks) ? keys.escalateFallbacks : []; + for (const key of list) { + if (typeof key !== "string" || !key || key === primaryKey) continue; + const ready = await readyEscalation(key, db, signal); + if (ready.adapter) return { ...ready, key, fellBackFrom: primary.reason }; + } + return primary; + } + + /** The kiosk metrics line's mark that a fallback (not the primary escalation key) served the turn. */ + const noteEscalationFallback = (timings, ready) => { + if (ready?.fellBackFrom) timings.escalation_fallback = String(ready.key).split("/")[0]; + }; + /** STT only (the kiosk's early transcription, lever D). The WAV is handed to the adapter and dropped. */ async function transcribe({ db, device, audio, signal, sttModel, sttPrompt, sttHotwords }) { const sttProfile = await deps.getSttProfile(db, device); @@ -1030,6 +1052,7 @@ export function createVoiceTurnRunner(deps) { // A quick-model turn that will end on the no-tool can't-do below makes no adapter at all (no model is touched). const noToolAhead = decision.route !== "escalate" && actionAsked && !displayBacks && !quickTools.some((t) => !displayTool(t.name)); let chat = noToolAhead ? null : await deps.createChatAdapter(fastKey, db); + let escalationKeyUsed = null; // the escalation model that really served this turn (primary or a fallback) result.route = "fast"; if (decision.route === "escalate") { // The filler runs alongside the (real-timer) readiness wait. A handler is attached NOW @@ -1037,13 +1060,19 @@ export function createVoiceTurnRunner(deps) { // gateway); the error is kept, logged below, and the turn carries on without the filler. let fillerErr = null; const filler = say.filler().catch((err) => { fillerErr = err; }); - const ready = await readyEscalation(decision.key, db, llmSignal); + const ready = await readyEscalationChain(keys, decision.key, db, llmSignal); await filler; if (fillerErr && !aborted() && !mute.signal.aborted) { console.warn(`[voice-turn] filler TTS failed: ${fillerErr.message}`); timings.filler_error = String(fillerErr.message || fillerErr); } - if (ready.adapter) { chat = ready.adapter; result.route = "escalate"; result.escalated = true; } + if (ready.adapter) { + chat = ready.adapter; + result.route = "escalate"; + result.escalated = true; + escalationKeyUsed = ready.key || decision.key; + noteEscalationFallback(timings, ready); + } // Qwen chat templates reject a system message anywhere but first (review C1): // the note joins the leading system message. else if (budgetHit) result.degraded = "budget"; // cut by the first-audio budget, not a barge-in @@ -1103,7 +1132,7 @@ export function createVoiceTurnRunner(deps) { // 4b. Prompt fit, against the model this turn really uses. A bound bot's skills are inlined // in the system message; a general assistant with many skills was ~41k tokens against the // 4B's 8,192 and every turn failed. Full prompt → the one without skill bodies → no call. - const modelKey = result.escalated ? decision.key : fastKey; + const modelKey = result.escalated ? (escalationKeyUsed || decision.key) : fastKey; // Decided on allTools (every extra counted, as the panel does), so an extra tool that is // hidden this turn never flips the level — and the system message — between turns. const fit = await promptFit({ db, bot, tools: allTools, promptSuffix: opts.promptSuffix, key: modelKey, full: withSuffix(system, opts.promptSuffix) }); @@ -1442,7 +1471,7 @@ export function createVoiceTurnRunner(deps) { steppedUp = true; let fillerErr = null; const filler = say.filler().catch((err) => { fillerErr = err; }); - const ready = await readyEscalation(keys.escalate ?? decision.key, db, llmSignal); + const ready = await readyEscalationChain(keys, keys.escalate ?? decision.key, db, llmSignal); await filler; if (fillerErr && !aborted() && !mute.signal.aborted) { log(`[voice-turn] ${device.id} filler TTS failed: ${fillerErr.message}`); @@ -1452,6 +1481,7 @@ export function createVoiceTurnRunner(deps) { chat = ready.adapter; result.route = "escalate"; result.escalated = true; + noteEscalationFallback(timings, ready); timings.claim_retry = "step_up"; } else forcing = rf; retryMode = mode; @@ -1843,7 +1873,7 @@ export function createVoiceTurnRunner(deps) { steppedUp = true; let fillerErr = null; const filler = say.filler().catch((err) => { fillerErr = err; }); - const ready = await readyEscalation(keys.escalate ?? decision.key, db, llmSignal); + const ready = await readyEscalationChain(keys, keys.escalate ?? decision.key, db, llmSignal); await filler; if (fillerErr && !aborted() && !mute.signal.aborted) { log(`[voice-turn] ${device.id} filler TTS failed: ${fillerErr.message}`); @@ -1854,6 +1884,7 @@ export function createVoiceTurnRunner(deps) { chat = ready.adapter; result.route = "escalate"; result.escalated = true; + noteEscalationFallback(timings, ready); timings.step_up = "own_tool"; tools = baseTools; if (fam) famAll = true; diff --git a/servers/shared/voice-keys.js b/servers/shared/voice-keys.js index b9f273ce3..8e2eeffed 100644 --- a/servers/shared/voice-keys.js +++ b/servers/shared/voice-keys.js @@ -52,11 +52,14 @@ export async function strictVoiceCloud(dbc, key, consumer = "voice") { /** * The voice route's keys for THIS request. Per side: the env var (read now), else the CHOSEN role binding * (role:voice for fast, role:everyday for escalate; a seeded or missing binding is not chosen), else the raw - * constant. A chosen role that resolves to nothing (its model not configured; voice has no fallback chain) falls - * to the raw constant, never to another role's model. Never throws. - * → { fast, escalate, fastRole, escalateRole, fastCloud, escalateCloud } + * constant. A chosen role that resolves to nothing (its model not configured) falls to the raw constant, never + * to another role's model. `escalateFallbacks` is the one explicit chain: the everyday binding's own `fallback` + * role ids, resolved in order, which voice may try when the everyday model cannot be readied; it is empty when + * the env var pins the escalation model, and no failure ever makes it anything but []. Never throws. + * → { fast, escalate, escalateFallbacks, fastRole, escalateRole, fastCloud, escalateCloud } * *Role the role id when the role gave the key, else null * *Cloud the key is not proven local (strictVoiceCloud), whatever gave it: env, role or raw constant + * escalateFallbacks raw keys, in order, de-duplicated, without the fast/escalate keys */ export async function resolveVoiceKeys(dbc, { env = process.env, resolveRoleFn = defaultResolveRole } = {}) { const side = async (envName, roleId, raw) => { @@ -75,6 +78,36 @@ export async function resolveVoiceKeys(dbc, { env = process.env, resolveRoleFn = side("COMPANION_FAST_MODEL", "voice", FAST_KEY), side("COMPANION_ESCALATION_MODEL", "everyday", ESC_KEY), ]); - const [fastCloud, escalateCloud] = await Promise.all([strictVoiceCloud(dbc, f.key), strictVoiceCloud(dbc, e.key)]); - return { fast: f.key, escalate: e.key, fastRole: f.role, escalateRole: e.role, fastCloud, escalateCloud }; + const [fastCloud, escalateCloud, escalateFallbacks] = await Promise.all([ + strictVoiceCloud(dbc, f.key), strictVoiceCloud(dbc, e.key), escalationFallbackKeys(dbc, env, resolveRoleFn, f.key, e.key), + ]); + return { fast: f.key, escalate: e.key, escalateFallbacks, fastRole: f.role, escalateRole: e.role, fastCloud, escalateCloud }; +} + +/** + * The escalation model's fallback chain: the everyday binding's `fallback` role ids, each resolved with + * resolveRoleFn (role:, need chat) in order. Excludes the escalate and fast keys, duplicates dropped. + * [] when the escalation env var is set (a pinned key has no chain) and on any failure: this never throws. + */ +async function escalationFallbackKeys(dbc, env, resolveRoleFn, fastKey, escalateKey) { + const pinned = env?.COMPANION_ESCALATION_MODEL; + if (typeof pinned === "string" && pinned) return []; + try { + const { readBindings } = await import("./model-roles-store.js"); + const bindings = await readBindings(dbc); + const ids = Array.isArray(bindings?.everyday?.fallback) ? bindings.everyday.fallback : []; + const out = []; + for (const id of ids) { + if (typeof id !== "string" || !id) continue; + try { + const r = await resolveRoleFn(dbc, `role:${id}`, { need: "chat" }); + const key = r && typeof r.key === "string" ? r.key : null; + if (!key || key === escalateKey || key === fastKey || out.includes(key)) continue; + out.push(key); + } catch { /* one bad fallback does not drop the others */ } + } + return out; + } catch { + return []; + } } diff --git a/tests/llm-router-voice-roles.test.js b/tests/llm-router-voice-roles.test.js index 05a71ded7..d981f0e2d 100644 --- a/tests/llm-router-voice-roles.test.js +++ b/tests/llm-router-voice-roles.test.js @@ -64,7 +64,7 @@ test("unedited: no binding gives the raw constants and no roles", async () => { try { await baseProviders(db); // The raw constants have no provider rows here, so neither is proven local (strict, Task 20). - assert.deepEqual(await resolveVoiceKeys(db, { env: NO_ENV }), { fast: FAST_KEY, escalate: ESC_KEY, fastRole: null, escalateRole: null, fastCloud: true, escalateCloud: true }); + assert.deepEqual(await resolveVoiceKeys(db, { env: NO_ENV }), { fast: FAST_KEY, escalate: ESC_KEY, fastRole: null, escalateRole: null, fastCloud: true, escalateCloud: true, escalateFallbacks: [] }); } finally { cleanup(); } }); @@ -97,7 +97,7 @@ test("env beats role", async () => { assert.equal(k.escalateRole, null); // Positive control: without the env the chosen bindings win. const c = await resolveVoiceKeys(db, { env: NO_ENV }); - assert.deepEqual(c, { fast: "local-voice/m2", escalate: "local-chat/m1", fastRole: "voice", escalateRole: "everyday", fastCloud: false, escalateCloud: false }); + assert.deepEqual(c, { fast: "local-voice/m2", escalate: "local-chat/m1", fastRole: "voice", escalateRole: "everyday", fastCloud: false, escalateCloud: false, escalateFallbacks: [] }); } finally { cleanup(); } }); @@ -126,7 +126,7 @@ test("resolveVoiceKeys never throws: a failing resolver gives the raw constants" try { k = await resolveVoiceKeys({}, { env: NO_ENV, resolveRoleFn: async () => { throw new Error("boom"); } }); } finally { console.warn = warn; } // No role context could be read: nothing is proven local. - assert.deepEqual(k, { fast: FAST_KEY, escalate: ESC_KEY, fastRole: null, escalateRole: null, fastCloud: true, escalateCloud: true }); + assert.deepEqual(k, { fast: FAST_KEY, escalate: ESC_KEY, fastRole: null, escalateRole: null, fastCloud: true, escalateCloud: true, escalateFallbacks: [] }); assert.deepEqual(warns.sort(), [ "[llm-router] role everyday: boom; using the consumer default", "[llm-router] role voice: boom; using the consumer default", diff --git a/tests/voice-keys-fallback.test.js b/tests/voice-keys-fallback.test.js new file mode 100644 index 000000000..b5e0b6c08 --- /dev/null +++ b/tests/voice-keys-fallback.test.js @@ -0,0 +1,84 @@ +/** + * The voice escalation chain (voice-keys.js): resolveVoiceKeys reads the + * everyday role binding's optional `fallback` role ids and returns their raw + * keys as `escalateFallbacks`, in order, de-duplicated, without the + * fast/escalate keys, empty when COMPANION_ESCALATION_MODEL pins the + * escalation model or on any failure. The role resolver is stubbed and the + * db is a plain object, so no model server or scratch database is needed. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { resolveVoiceKeys, VOICE_RAW_KEYS } from "../servers/shared/voice-keys.js"; +import { bindingKey } from "../servers/shared/model-roles.js"; + +const EVERYDAY = bindingKey("everyday"); + +/** A db stand-in: the dashboard_settings rows asked for, empty for everything else. */ +function rowsDb(rows, { fail = false } = {}) { + return { + async execute(q) { + if (fail) throw new Error("db down"); + const sql = typeof q === "string" ? q : q?.sql || ""; + return { rows: sql.includes("FROM dashboard_settings") ? rows : [] }; + }, + }; +} + +const everydayBinding = (fallback) => ({ key: EVERYDAY, value: JSON.stringify({ model: "crow-chat/m", fallback }) }); + +/** resolveRoleFn(dbc, "role:", opts) from a role id → key map. */ +const resolver = (map) => async (_dbc, key) => { + const id = String(key).replace(/^role:/, ""); + return Object.hasOwn(map, id) ? { key: map[id], role: id, reason: "bound" } : { key: null, role: id, reason: "not_configured" }; +}; + +test("escalateFallbacks: the everyday binding's fallback roles, resolved in order", async () => { + const db = rowsDb([everydayBinding(["coordinator"])]); + const k = await resolveVoiceKeys(db, { + env: {}, + resolveRoleFn: resolver({ + everyday: "crow-chat/m", + coordinator: "raven/qwen3.8-flash-next", + voice: VOICE_RAW_KEYS.fast, + }), + }); + assert.equal(k.escalate, "crow-chat/m"); + assert.deepEqual(k.escalateFallbacks, ["raven/qwen3.8-flash-next"]); +}); + +test("escalateFallbacks: duplicates and the fast/escalate keys themselves are excluded", async () => { + const db = rowsDb([everydayBinding(["voice", "coordinator", "coordinator", "everyday"])]); + const k = await resolveVoiceKeys(db, { + env: {}, + resolveRoleFn: resolver({ + everyday: "crow-chat/m", + voice: VOICE_RAW_KEYS.fast, + coordinator: "raven/flash", + }), + }); + assert.deepEqual(k.escalateFallbacks, ["raven/flash"]); +}); + +test("escalateFallbacks: [] when COMPANION_ESCALATION_MODEL pins the escalation model", async () => { + const db = rowsDb([everydayBinding(["coordinator"])]); + const k = await resolveVoiceKeys(db, { + env: { COMPANION_ESCALATION_MODEL: "raven/flash" }, + resolveRoleFn: resolver({ everyday: "crow-chat/m", coordinator: "raven/flash", voice: VOICE_RAW_KEYS.fast }), + }); + assert.equal(k.escalate, "raven/flash"); + assert.deepEqual(k.escalateFallbacks, []); +}); + +test("escalateFallbacks: [] when nothing is configured, and never throws when the store fails", async () => { + const empty = await resolveVoiceKeys(rowsDb([]), { + env: {}, + resolveRoleFn: resolver({ voice: VOICE_RAW_KEYS.fast }), + }); + assert.deepEqual(empty.escalateFallbacks, []); + + const broken = await resolveVoiceKeys(rowsDb([], { fail: true }), { + env: {}, + resolveRoleFn: resolver({ everyday: "crow-chat/m", coordinator: "raven/flash", voice: VOICE_RAW_KEYS.fast }), + }); + assert.deepEqual(broken.escalateFallbacks, []); +}); \ No newline at end of file diff --git a/tests/voice-turn.test.js b/tests/voice-turn.test.js index 798ff0b9b..cff04400d 100644 --- a/tests/voice-turn.test.js +++ b/tests/voice-turn.test.js @@ -88,6 +88,40 @@ test("box reserved → immediate fallback, no 8 s wait", async () => { assert.ok(h.calls.sleeps <= 1); }); +test("escalation fallback: the everyday role's fallback serves a box_reserved primary", async () => { + const h = harness({ route: "escalate", probe: () => true, + acquire: async (p) => { if (p === "crow-chat") throw Object.assign(new Error("reserved"), { code: "box_reserved" }); return null; } }); + h.deps.resolveVoiceKeys = async () => ({ fast: "crow-voice/qwen3.5-4b", escalate: "crow-chat/qwen3.6-35b-a3b", escalateFallbacks: ["raven/flash"] }); + const r = await h.runner.runVoiceTurn({ db: {}, device: h.device, transcript: "play a random album", sink: h.sink }); + assert.equal(r.route, "escalate"); + assert.equal(r.escalated, true); + assert.equal(r.degraded, null); + assert.deepEqual(h.calls.acquired, ["crow-chat", "raven"], "the primary first, then the fallback"); + assert.deepEqual(h.calls.chatKeys, ["crow-voice/qwen3.5-4b", "raven/flash"], "the fallback's adapter answers the turn"); + assert.equal(r.timings.escalation_fallback, "raven", "the metrics line names the provider that served it"); +}); + +test("escalation fallback: with no fallbacks a box_reserved primary degrades exactly as before", async () => { + const h = harness({ route: "escalate", acquire: async () => { throw Object.assign(new Error("reserved"), { code: "box_reserved" }); } }); + h.deps.resolveVoiceKeys = async () => ({ fast: "crow-voice/qwen3.5-4b", escalate: "crow-chat/qwen3.6-35b-a3b", escalateFallbacks: [] }); + const r = await h.runner.runVoiceTurn({ db: {}, device: h.device, transcript: "play a random album", sink: h.sink }); + assert.equal(r.degraded, "box_reserved"); + assert.equal(r.escalated, false); + assert.deepEqual(h.calls.chatKeys, ["crow-voice/qwen3.5-4b"]); + assert.equal(r.timings.escalation_fallback, undefined); +}); + +test("escalation fallback: a cold primary does not try the fallbacks", async () => { + const h = harness({ route: "escalate", probe: () => false }); + h.deps.resolveVoiceKeys = async () => ({ fast: "crow-voice/qwen3.5-4b", escalate: "crow-chat/qwen3.6-35b-a3b", escalateFallbacks: ["raven/flash"] }); + const r = await h.runner.runVoiceTurn({ db: {}, device: h.device, transcript: "play a random album", sink: h.sink }); + assert.equal(r.degraded, "cold_timeout"); + assert.equal(r.escalated, false); + assert.deepEqual(h.calls.acquired, ["crow-chat"], "only the primary was asked for"); + assert.deepEqual(h.calls.chatKeys, ["crow-voice/qwen3.5-4b"]); + assert.equal(r.timings.escalation_fallback, undefined); +}); + test("destructive tool: two-turn spoken confirmation", async () => { const del = { type: "tool_call", id: "d1", name: "crow_delete_post", arguments: { id: 7 } }; const h = harness({ chatTools: ["crow_blog"], rounds: [[del, { type: "done" }], [{ type: "content_delta", text: "Are you sure?" }, { type: "done" }], [del, { type: "done" }], [{ type: "content_delta", text: "Deleted." }, { type: "done" }]] });