From 29af053968f1cd4f3a7e4af4b0eecf46b50077b2 Mon Sep 17 00:00:00 2001 From: kh0pper <93635715+kh0pper@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:56:17 -0500 Subject: [PATCH 1/2] fix(voice): an escalation the box refuses tries the everyday role's fallback models before the fast model A kiosk turn escalated to the local 35B while a GPU job held the box; the orchestrator refused it and the turn fell to the 4B, which ran out of time, though another capable model was up. resolveVoiceKeys now returns escalateFallbacks (the everyday binding's fallback roles, resolved), and an escalation refused for box_reserved / serving_class / acquire_failed / unresolved tries them in order. A cold start does not. The kiosk metrics line records which fallback served (escalation_fallback). Implemented by the Pi coding agent (deepseek-v4.1-flash). Claude-Session: https://claude.ai/code/session_01N9fXzvNy8rCEArj1y5p6fp --- servers/gateway/voice/turn.js | 41 +++++++++++++-- servers/shared/voice-keys.js | 43 ++++++++++++++-- tests/voice-keys-fallback.test.js | 84 +++++++++++++++++++++++++++++++ tests/voice-turn.test.js | 34 +++++++++++++ 4 files changed, 192 insertions(+), 10 deletions(-) create mode 100644 tests/voice-keys-fallback.test.js diff --git a/servers/gateway/voice/turn.js b/servers/gateway/voice/turn.js index 481dca76d..b0049390a 100644 --- a/servers/gateway/voice/turn.js +++ b/servers/gateway/voice/turn.js @@ -531,6 +531,28 @@ export function createVoiceTurnRunner(deps) { return { reason: "cold_timeout" }; } + // A primary the box will not serve for its own reasons is retried on the everyday role's own fallback + // roles (voice-keys.js escalateFallbacks, in order). A cold start, an abort or every fallback failing + // keeps the primary's result: the turn degrades exactly as before. + const ESCALATION_FALLBACK_REASONS = new Set(["box_reserved", "serving_class", "acquire_failed", "unresolved"]); + async function readyEscalationChain(keys, primaryKey, db, signal) { + const primary = await readyEscalation(primaryKey, db, signal); + if (primary.adapter) return { ...primary, key: primaryKey }; + if (!ESCALATION_FALLBACK_REASONS.has(primary.reason)) return primary; + const list = Array.isArray(keys?.escalateFallbacks) ? keys.escalateFallbacks : []; + for (const key of list) { + if (typeof key !== "string" || !key || key === primaryKey) continue; + const ready = await readyEscalation(key, db, signal); + if (ready.adapter) return { ...ready, key, fellBackFrom: primary.reason }; + } + return primary; + } + + /** The kiosk metrics line's mark that a fallback (not the primary escalation key) served the turn. */ + const noteEscalationFallback = (timings, ready) => { + if (ready?.fellBackFrom) timings.escalation_fallback = String(ready.key).split("/")[0]; + }; + /** STT only (the kiosk's early transcription, lever D). The WAV is handed to the adapter and dropped. */ async function transcribe({ db, device, audio, signal, sttModel, sttPrompt, sttHotwords }) { const sttProfile = await deps.getSttProfile(db, device); @@ -1030,6 +1052,7 @@ export function createVoiceTurnRunner(deps) { // A quick-model turn that will end on the no-tool can't-do below makes no adapter at all (no model is touched). const noToolAhead = decision.route !== "escalate" && actionAsked && !displayBacks && !quickTools.some((t) => !displayTool(t.name)); let chat = noToolAhead ? null : await deps.createChatAdapter(fastKey, db); + let escalationKeyUsed = null; // the escalation model that really served this turn (primary or a fallback) result.route = "fast"; if (decision.route === "escalate") { // The filler runs alongside the (real-timer) readiness wait. A handler is attached NOW @@ -1037,13 +1060,19 @@ export function createVoiceTurnRunner(deps) { // gateway); the error is kept, logged below, and the turn carries on without the filler. let fillerErr = null; const filler = say.filler().catch((err) => { fillerErr = err; }); - const ready = await readyEscalation(decision.key, db, llmSignal); + const ready = await readyEscalationChain(keys, decision.key, db, llmSignal); await filler; if (fillerErr && !aborted() && !mute.signal.aborted) { console.warn(`[voice-turn] filler TTS failed: ${fillerErr.message}`); timings.filler_error = String(fillerErr.message || fillerErr); } - if (ready.adapter) { chat = ready.adapter; result.route = "escalate"; result.escalated = true; } + if (ready.adapter) { + chat = ready.adapter; + result.route = "escalate"; + result.escalated = true; + escalationKeyUsed = ready.key || decision.key; + noteEscalationFallback(timings, ready); + } // Qwen chat templates reject a system message anywhere but first (review C1): // the note joins the leading system message. else if (budgetHit) result.degraded = "budget"; // cut by the first-audio budget, not a barge-in @@ -1103,7 +1132,7 @@ export function createVoiceTurnRunner(deps) { // 4b. Prompt fit, against the model this turn really uses. A bound bot's skills are inlined // in the system message; a general assistant with many skills was ~41k tokens against the // 4B's 8,192 and every turn failed. Full prompt → the one without skill bodies → no call. - const modelKey = result.escalated ? decision.key : fastKey; + const modelKey = result.escalated ? (escalationKeyUsed || decision.key) : fastKey; // Decided on allTools (every extra counted, as the panel does), so an extra tool that is // hidden this turn never flips the level — and the system message — between turns. const fit = await promptFit({ db, bot, tools: allTools, promptSuffix: opts.promptSuffix, key: modelKey, full: withSuffix(system, opts.promptSuffix) }); @@ -1442,7 +1471,7 @@ export function createVoiceTurnRunner(deps) { steppedUp = true; let fillerErr = null; const filler = say.filler().catch((err) => { fillerErr = err; }); - const ready = await readyEscalation(keys.escalate ?? decision.key, db, llmSignal); + const ready = await readyEscalationChain(keys, keys.escalate ?? decision.key, db, llmSignal); await filler; if (fillerErr && !aborted() && !mute.signal.aborted) { log(`[voice-turn] ${device.id} filler TTS failed: ${fillerErr.message}`); @@ -1452,6 +1481,7 @@ export function createVoiceTurnRunner(deps) { chat = ready.adapter; result.route = "escalate"; result.escalated = true; + noteEscalationFallback(timings, ready); timings.claim_retry = "step_up"; } else forcing = rf; retryMode = mode; @@ -1843,7 +1873,7 @@ export function createVoiceTurnRunner(deps) { steppedUp = true; let fillerErr = null; const filler = say.filler().catch((err) => { fillerErr = err; }); - const ready = await readyEscalation(keys.escalate ?? decision.key, db, llmSignal); + const ready = await readyEscalationChain(keys, keys.escalate ?? decision.key, db, llmSignal); await filler; if (fillerErr && !aborted() && !mute.signal.aborted) { log(`[voice-turn] ${device.id} filler TTS failed: ${fillerErr.message}`); @@ -1854,6 +1884,7 @@ export function createVoiceTurnRunner(deps) { chat = ready.adapter; result.route = "escalate"; result.escalated = true; + noteEscalationFallback(timings, ready); timings.step_up = "own_tool"; tools = baseTools; if (fam) famAll = true; diff --git a/servers/shared/voice-keys.js b/servers/shared/voice-keys.js index b9f273ce3..8e2eeffed 100644 --- a/servers/shared/voice-keys.js +++ b/servers/shared/voice-keys.js @@ -52,11 +52,14 @@ export async function strictVoiceCloud(dbc, key, consumer = "voice") { /** * The voice route's keys for THIS request. Per side: the env var (read now), else the CHOSEN role binding * (role:voice for fast, role:everyday for escalate; a seeded or missing binding is not chosen), else the raw - * constant. A chosen role that resolves to nothing (its model not configured; voice has no fallback chain) falls - * to the raw constant, never to another role's model. Never throws. - * → { fast, escalate, fastRole, escalateRole, fastCloud, escalateCloud } + * constant. A chosen role that resolves to nothing (its model not configured) falls to the raw constant, never + * to another role's model. `escalateFallbacks` is the one explicit chain: the everyday binding's own `fallback` + * role ids, resolved in order, which voice may try when the everyday model cannot be readied; it is empty when + * the env var pins the escalation model, and no failure ever makes it anything but []. Never throws. + * → { fast, escalate, escalateFallbacks, fastRole, escalateRole, fastCloud, escalateCloud } * *Role the role id when the role gave the key, else null * *Cloud the key is not proven local (strictVoiceCloud), whatever gave it: env, role or raw constant + * escalateFallbacks raw keys, in order, de-duplicated, without the fast/escalate keys */ export async function resolveVoiceKeys(dbc, { env = process.env, resolveRoleFn = defaultResolveRole } = {}) { const side = async (envName, roleId, raw) => { @@ -75,6 +78,36 @@ export async function resolveVoiceKeys(dbc, { env = process.env, resolveRoleFn = side("COMPANION_FAST_MODEL", "voice", FAST_KEY), side("COMPANION_ESCALATION_MODEL", "everyday", ESC_KEY), ]); - const [fastCloud, escalateCloud] = await Promise.all([strictVoiceCloud(dbc, f.key), strictVoiceCloud(dbc, e.key)]); - return { fast: f.key, escalate: e.key, fastRole: f.role, escalateRole: e.role, fastCloud, escalateCloud }; + const [fastCloud, escalateCloud, escalateFallbacks] = await Promise.all([ + strictVoiceCloud(dbc, f.key), strictVoiceCloud(dbc, e.key), escalationFallbackKeys(dbc, env, resolveRoleFn, f.key, e.key), + ]); + return { fast: f.key, escalate: e.key, escalateFallbacks, fastRole: f.role, escalateRole: e.role, fastCloud, escalateCloud }; +} + +/** + * The escalation model's fallback chain: the everyday binding's `fallback` role ids, each resolved with + * resolveRoleFn (role:, need chat) in order. Excludes the escalate and fast keys, duplicates dropped. + * [] when the escalation env var is set (a pinned key has no chain) and on any failure: this never throws. + */ +async function escalationFallbackKeys(dbc, env, resolveRoleFn, fastKey, escalateKey) { + const pinned = env?.COMPANION_ESCALATION_MODEL; + if (typeof pinned === "string" && pinned) return []; + try { + const { readBindings } = await import("./model-roles-store.js"); + const bindings = await readBindings(dbc); + const ids = Array.isArray(bindings?.everyday?.fallback) ? bindings.everyday.fallback : []; + const out = []; + for (const id of ids) { + if (typeof id !== "string" || !id) continue; + try { + const r = await resolveRoleFn(dbc, `role:${id}`, { need: "chat" }); + const key = r && typeof r.key === "string" ? r.key : null; + if (!key || key === escalateKey || key === fastKey || out.includes(key)) continue; + out.push(key); + } catch { /* one bad fallback does not drop the others */ } + } + return out; + } catch { + return []; + } } diff --git a/tests/voice-keys-fallback.test.js b/tests/voice-keys-fallback.test.js new file mode 100644 index 000000000..b5e0b6c08 --- /dev/null +++ b/tests/voice-keys-fallback.test.js @@ -0,0 +1,84 @@ +/** + * The voice escalation chain (voice-keys.js): resolveVoiceKeys reads the + * everyday role binding's optional `fallback` role ids and returns their raw + * keys as `escalateFallbacks`, in order, de-duplicated, without the + * fast/escalate keys, empty when COMPANION_ESCALATION_MODEL pins the + * escalation model or on any failure. The role resolver is stubbed and the + * db is a plain object, so no model server or scratch database is needed. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { resolveVoiceKeys, VOICE_RAW_KEYS } from "../servers/shared/voice-keys.js"; +import { bindingKey } from "../servers/shared/model-roles.js"; + +const EVERYDAY = bindingKey("everyday"); + +/** A db stand-in: the dashboard_settings rows asked for, empty for everything else. */ +function rowsDb(rows, { fail = false } = {}) { + return { + async execute(q) { + if (fail) throw new Error("db down"); + const sql = typeof q === "string" ? q : q?.sql || ""; + return { rows: sql.includes("FROM dashboard_settings") ? rows : [] }; + }, + }; +} + +const everydayBinding = (fallback) => ({ key: EVERYDAY, value: JSON.stringify({ model: "crow-chat/m", fallback }) }); + +/** resolveRoleFn(dbc, "role:", opts) from a role id → key map. */ +const resolver = (map) => async (_dbc, key) => { + const id = String(key).replace(/^role:/, ""); + return Object.hasOwn(map, id) ? { key: map[id], role: id, reason: "bound" } : { key: null, role: id, reason: "not_configured" }; +}; + +test("escalateFallbacks: the everyday binding's fallback roles, resolved in order", async () => { + const db = rowsDb([everydayBinding(["coordinator"])]); + const k = await resolveVoiceKeys(db, { + env: {}, + resolveRoleFn: resolver({ + everyday: "crow-chat/m", + coordinator: "raven/qwen3.8-flash-next", + voice: VOICE_RAW_KEYS.fast, + }), + }); + assert.equal(k.escalate, "crow-chat/m"); + assert.deepEqual(k.escalateFallbacks, ["raven/qwen3.8-flash-next"]); +}); + +test("escalateFallbacks: duplicates and the fast/escalate keys themselves are excluded", async () => { + const db = rowsDb([everydayBinding(["voice", "coordinator", "coordinator", "everyday"])]); + const k = await resolveVoiceKeys(db, { + env: {}, + resolveRoleFn: resolver({ + everyday: "crow-chat/m", + voice: VOICE_RAW_KEYS.fast, + coordinator: "raven/flash", + }), + }); + assert.deepEqual(k.escalateFallbacks, ["raven/flash"]); +}); + +test("escalateFallbacks: [] when COMPANION_ESCALATION_MODEL pins the escalation model", async () => { + const db = rowsDb([everydayBinding(["coordinator"])]); + const k = await resolveVoiceKeys(db, { + env: { COMPANION_ESCALATION_MODEL: "raven/flash" }, + resolveRoleFn: resolver({ everyday: "crow-chat/m", coordinator: "raven/flash", voice: VOICE_RAW_KEYS.fast }), + }); + assert.equal(k.escalate, "raven/flash"); + assert.deepEqual(k.escalateFallbacks, []); +}); + +test("escalateFallbacks: [] when nothing is configured, and never throws when the store fails", async () => { + const empty = await resolveVoiceKeys(rowsDb([]), { + env: {}, + resolveRoleFn: resolver({ voice: VOICE_RAW_KEYS.fast }), + }); + assert.deepEqual(empty.escalateFallbacks, []); + + const broken = await resolveVoiceKeys(rowsDb([], { fail: true }), { + env: {}, + resolveRoleFn: resolver({ everyday: "crow-chat/m", coordinator: "raven/flash", voice: VOICE_RAW_KEYS.fast }), + }); + assert.deepEqual(broken.escalateFallbacks, []); +}); \ No newline at end of file diff --git a/tests/voice-turn.test.js b/tests/voice-turn.test.js index 798ff0b9b..cff04400d 100644 --- a/tests/voice-turn.test.js +++ b/tests/voice-turn.test.js @@ -88,6 +88,40 @@ test("box reserved → immediate fallback, no 8 s wait", async () => { assert.ok(h.calls.sleeps <= 1); }); +test("escalation fallback: the everyday role's fallback serves a box_reserved primary", async () => { + const h = harness({ route: "escalate", probe: () => true, + acquire: async (p) => { if (p === "crow-chat") throw Object.assign(new Error("reserved"), { code: "box_reserved" }); return null; } }); + h.deps.resolveVoiceKeys = async () => ({ fast: "crow-voice/qwen3.5-4b", escalate: "crow-chat/qwen3.6-35b-a3b", escalateFallbacks: ["raven/flash"] }); + const r = await h.runner.runVoiceTurn({ db: {}, device: h.device, transcript: "play a random album", sink: h.sink }); + assert.equal(r.route, "escalate"); + assert.equal(r.escalated, true); + assert.equal(r.degraded, null); + assert.deepEqual(h.calls.acquired, ["crow-chat", "raven"], "the primary first, then the fallback"); + assert.deepEqual(h.calls.chatKeys, ["crow-voice/qwen3.5-4b", "raven/flash"], "the fallback's adapter answers the turn"); + assert.equal(r.timings.escalation_fallback, "raven", "the metrics line names the provider that served it"); +}); + +test("escalation fallback: with no fallbacks a box_reserved primary degrades exactly as before", async () => { + const h = harness({ route: "escalate", acquire: async () => { throw Object.assign(new Error("reserved"), { code: "box_reserved" }); } }); + h.deps.resolveVoiceKeys = async () => ({ fast: "crow-voice/qwen3.5-4b", escalate: "crow-chat/qwen3.6-35b-a3b", escalateFallbacks: [] }); + const r = await h.runner.runVoiceTurn({ db: {}, device: h.device, transcript: "play a random album", sink: h.sink }); + assert.equal(r.degraded, "box_reserved"); + assert.equal(r.escalated, false); + assert.deepEqual(h.calls.chatKeys, ["crow-voice/qwen3.5-4b"]); + assert.equal(r.timings.escalation_fallback, undefined); +}); + +test("escalation fallback: a cold primary does not try the fallbacks", async () => { + const h = harness({ route: "escalate", probe: () => false }); + h.deps.resolveVoiceKeys = async () => ({ fast: "crow-voice/qwen3.5-4b", escalate: "crow-chat/qwen3.6-35b-a3b", escalateFallbacks: ["raven/flash"] }); + const r = await h.runner.runVoiceTurn({ db: {}, device: h.device, transcript: "play a random album", sink: h.sink }); + assert.equal(r.degraded, "cold_timeout"); + assert.equal(r.escalated, false); + assert.deepEqual(h.calls.acquired, ["crow-chat"], "only the primary was asked for"); + assert.deepEqual(h.calls.chatKeys, ["crow-voice/qwen3.5-4b"]); + assert.equal(r.timings.escalation_fallback, undefined); +}); + test("destructive tool: two-turn spoken confirmation", async () => { const del = { type: "tool_call", id: "d1", name: "crow_delete_post", arguments: { id: 7 } }; const h = harness({ chatTools: ["crow_blog"], rounds: [[del, { type: "done" }], [{ type: "content_delta", text: "Are you sure?" }, { type: "done" }], [del, { type: "done" }], [{ type: "content_delta", text: "Deleted." }, { type: "done" }]] }); From 308341744af21f619e89698de185003e9538f672 Mon Sep 17 00:00:00 2001 From: kh0pper <93635715+kh0pper@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:18:15 -0500 Subject: [PATCH 2/2] test(voice): resolveVoiceKeys' shape now includes escalateFallbacks Claude-Session: https://claude.ai/code/session_01N9fXzvNy8rCEArj1y5p6fp --- tests/llm-router-voice-roles.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/llm-router-voice-roles.test.js b/tests/llm-router-voice-roles.test.js index 05a71ded7..d981f0e2d 100644 --- a/tests/llm-router-voice-roles.test.js +++ b/tests/llm-router-voice-roles.test.js @@ -64,7 +64,7 @@ test("unedited: no binding gives the raw constants and no roles", async () => { try { await baseProviders(db); // The raw constants have no provider rows here, so neither is proven local (strict, Task 20). - assert.deepEqual(await resolveVoiceKeys(db, { env: NO_ENV }), { fast: FAST_KEY, escalate: ESC_KEY, fastRole: null, escalateRole: null, fastCloud: true, escalateCloud: true }); + assert.deepEqual(await resolveVoiceKeys(db, { env: NO_ENV }), { fast: FAST_KEY, escalate: ESC_KEY, fastRole: null, escalateRole: null, fastCloud: true, escalateCloud: true, escalateFallbacks: [] }); } finally { cleanup(); } }); @@ -97,7 +97,7 @@ test("env beats role", async () => { assert.equal(k.escalateRole, null); // Positive control: without the env the chosen bindings win. const c = await resolveVoiceKeys(db, { env: NO_ENV }); - assert.deepEqual(c, { fast: "local-voice/m2", escalate: "local-chat/m1", fastRole: "voice", escalateRole: "everyday", fastCloud: false, escalateCloud: false }); + assert.deepEqual(c, { fast: "local-voice/m2", escalate: "local-chat/m1", fastRole: "voice", escalateRole: "everyday", fastCloud: false, escalateCloud: false, escalateFallbacks: [] }); } finally { cleanup(); } }); @@ -126,7 +126,7 @@ test("resolveVoiceKeys never throws: a failing resolver gives the raw constants" try { k = await resolveVoiceKeys({}, { env: NO_ENV, resolveRoleFn: async () => { throw new Error("boom"); } }); } finally { console.warn = warn; } // No role context could be read: nothing is proven local. - assert.deepEqual(k, { fast: FAST_KEY, escalate: ESC_KEY, fastRole: null, escalateRole: null, fastCloud: true, escalateCloud: true }); + assert.deepEqual(k, { fast: FAST_KEY, escalate: ESC_KEY, fastRole: null, escalateRole: null, fastCloud: true, escalateCloud: true, escalateFallbacks: [] }); assert.deepEqual(warns.sort(), [ "[llm-router] role everyday: boom; using the consumer default", "[llm-router] role voice: boom; using the consumer default",