From 1152da1ebbe7cb48fa3dbb5e3d9ab56ea2174f24 Mon Sep 17 00:00:00 2001 From: Matt Miller Date: Tue, 27 Oct 2026 22:31:39 -0400 Subject: [PATCH] fix: keep agent-written option ids out of superseded-answer telemetry (#21327) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ELI5 When an answer to one of the agent's questions gets overridden by an answer given somewhere else, we log an error. That error used to include the agent's own option text, which could contain parts of the user's prompt and split identical errors into thousands of separate buckets. Now it logs a fixed message plus how many options each answer picked, whether the picked options, the typed text, or both changed, and whether the server said which options won at all. ## Summary `agent_ask_answer_superseded` built its `Error` message from the raw option ids of both answers. `ask_user` option ids are free-form text the agent writes, so the message could carry prompt-derived content, and each distinct id pair grouped as its own error. The message is now the fixed string `agent ask resolved with a different answer`, and the detail moves into bounded tags: `committed_selected_count` / `submitted_selected_count`, plus `selection_changed` / `other_text_changed` booleans from the same comparison that decides supersession, so a same-length selection change and a free-text-only change stay distinguishable. `selection_known` says whether the committed body carried a `selected` array; when it did not, `committed_selected_count` is omitted so "the server did not say" is not read as "the winner picked nothing". The user-facing "answered somewhere else" notice and the supersede detection are unchanged. Risk: telemetry only. A dashboard or alert that matches on the old `agent ask resolved as …` text needs the new message; grouping by `errorType` is unaffected. ## Verification - Rebased onto main after #21829 (free-text-only committed answers). Its "selection known" guard is kept: when the committed body carries no `selected` array, `selection_changed` is `false`, `selection_known` is `false` and `committed_selected_count` is omitted, and only `other_text_changed` can fire, so the report still matches main's supersede rule. - `useAgentSession.test.ts`: 275 pass (1 expected fail). `reports the answer the 202 committed when another one won` asserts the exact fixed message and all four tags; the run-approval superseded test asserts the tags; the table of superseding answers and the 409 free-text-only test (null and missing `selected`) asserts the full tag set, including `selection_known: false` and no committed count. - `pnpm typecheck`, oxfmt, oxlint, eslint: clean. - Doc-drift sweep: no doc, fixture or spec quotes the old message, the `agent_ask_answer_superseded` slug or the new tag names. The other matches were generic `agent`/`reportError` mentions. ## Provenance - **Authored by:** agent-work loop - **Verified:** - Rebased onto current main; the merge with main's free-text-only guard keeps both behaviours - `useAgentSession.test.ts` 275 pass at head, including the exact-message, change-tag and `selection_known` assertions - The updated superseded tests failed against the old message code - `pnpm typecheck`, oxfmt, oxlint and eslint are clean at head - Doc-drift sweep: nothing outside the changed code mentions the message, slug or tag names - **Deviations:** - The ticket gives "ask kind" as an example field. I left it out because the report site only has the ask id and the resolution, not the card type, so adding it would mean a new store lookup just for telemetry. I added the selected-id counts and the two change booleans (the booleans came from review). ## Residual - The ask kind (run approval vs `ask_user`) is not on the report. Adding it would need a lookup of the card type by ask id at the report site. - I could not see any dashboards or alerts that match on the old message text. Whoever owns agent error monitoring should check them. --- .../composables/agent/useAgentSession.test.ts | 48 ++++++++++++++++--- .../composables/agent/useAgentSession.ts | 42 +++++++++------- 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts index 3b4b08b553d..00472aa102e 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts @@ -2207,8 +2207,19 @@ describe('useAgentSession (v1 composition root)', () => { resolution: { status: 'answered', selected: ['flux'] } }) expect(reportError).toHaveBeenCalledWith( - expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + message: 'agent ask resolved with a different answer' + }), + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: true, + other_text_changed: false, + selection_known: true, + committed_selected_count: 1, + submitted_selected_count: 1 + } + }) ) }) @@ -2311,7 +2322,16 @@ describe('useAgentSession (v1 composition root)', () => { }) expect(reportError).toHaveBeenCalledWith( expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: false, + other_text_changed: true, + selection_known: false, + committed_selected_count: undefined, + submitted_selected_count: 1 + } + }) ) expect(session.notices.value).toEqual([ { @@ -2507,14 +2527,17 @@ describe('useAgentSession (v1 composition root)', () => { name: string committed: Pick expected: AskUserResolution + changes: { selection_changed: boolean; other_text_changed: boolean } }>([ { name: 'a different option', + changes: { selection_changed: true, other_text_changed: true }, committed: { selected: ['flux'] }, expected: { status: 'answered', selected: ['flux'] } }, { name: 'a superset of it', + changes: { selection_changed: true, other_text_changed: false }, committed: { selected: ['sdxl', 'flux'], other_text: 'a LoRA' }, expected: { status: 'answered', @@ -2524,11 +2547,13 @@ describe('useAgentSession (v1 composition root)', () => { }, { name: 'a subset of it', + changes: { selection_changed: true, other_text_changed: false }, committed: { selected: [], other_text: 'a LoRA' }, expected: { status: 'answered', selected: [], otherText: 'a LoRA' } }, { name: 'free text alone', + changes: { selection_changed: true, other_text_changed: true }, committed: { selected: [], other_text: 'someone else' }, expected: { status: 'answered', @@ -2538,12 +2563,13 @@ describe('useAgentSession (v1 composition root)', () => { }, { name: 'the same option with other free text', + changes: { selection_changed: false, other_text_changed: true }, committed: { selected: ['sdxl'], other_text: 'a VAE' }, expected: { status: 'answered', selected: ['sdxl'], otherText: 'a VAE' } } ])( 'shows the committed answer when $name won', - async ({ committed, expected }) => { + async ({ committed, expected, changes }) => { const { session, emit } = await parkedOnQuestion() await session.answerAsk('turn-1:call-1', { selected: ['sdxl'], @@ -2568,7 +2594,10 @@ describe('useAgentSession (v1 composition root)', () => { expect(shown.resolution).toEqual(expected) expect(reportError).toHaveBeenCalledWith( expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: expect.objectContaining(changes) + }) ) } ) @@ -2896,7 +2925,14 @@ describe('useAgentSession (v1 composition root)', () => { expect(reportError).toHaveBeenCalledWith(expect.any(Error), { surface: 'agent', - errorType: 'agent_ask_answer_superseded' + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: true, + other_text_changed: false, + selection_known: true, + committed_selected_count: 1, + submitted_selected_count: 1 + } }) expect(session.notices.value).toHaveLength(1) expect(cardOnScreen()).toBe(false) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts index 9aac95a13b7..581d8c6efa9 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts @@ -344,16 +344,17 @@ function namesCommittedAnswer({ return Array.isArray(selected) || Boolean(other_text) } -function isSupersededAnswer( +function diffAnswers( submitted: AgentAnswerRequest, committed: Extract, selectionKnown: boolean -): boolean { - return ( - (selectionKnown && - xor(submitted.selected, committed.selected).length > 0) || - (submitted.other_text || undefined) !== committed.otherText - ) +) { + return { + selection_changed: + selectionKnown && xor(submitted.selected, committed.selected).length > 0, + other_text_changed: + (submitted.other_text || undefined) !== committed.otherText + } } let sessionGeneration = 0 @@ -1549,17 +1550,22 @@ export function useAgentSession(deps: AgentSessionDeps) { const committed = toAskResolution(source) if (committed.status !== 'answered') return const submitted = conversationStore.takeSubmittedAskAnswer(askId) - if ( - submitted === undefined || - !isSupersededAnswer(submitted, committed, Array.isArray(source.selected)) - ) - return - reportError( - new Error( - `agent ask resolved as ${committed.selected.join(',')}, not ${submitted.selected.join(',')}` - ), - { surface: 'agent', errorType: 'agent_ask_answer_superseded' } - ) + if (submitted === undefined) return + const selectionKnown = Array.isArray(source.selected) + const changes = diffAnswers(submitted, committed, selectionKnown) + if (!changes.selection_changed && !changes.other_text_changed) return + reportError(new Error('agent ask resolved with a different answer'), { + surface: 'agent', + errorType: 'agent_ask_answer_superseded', + tags: { + ...changes, + selection_known: selectionKnown, + committed_selected_count: selectionKnown + ? committed.selected.length + : undefined, + submitted_selected_count: submitted.selected.length + } + }) pushError(i18n.global.t('agent.runApproval.answerSuperseded')) }