diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts index 3b4b08b553d..00472aa102e 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts @@ -2207,8 +2207,19 @@ describe('useAgentSession (v1 composition root)', () => { resolution: { status: 'answered', selected: ['flux'] } }) expect(reportError).toHaveBeenCalledWith( - expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + message: 'agent ask resolved with a different answer' + }), + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: true, + other_text_changed: false, + selection_known: true, + committed_selected_count: 1, + submitted_selected_count: 1 + } + }) ) }) @@ -2311,7 +2322,16 @@ describe('useAgentSession (v1 composition root)', () => { }) expect(reportError).toHaveBeenCalledWith( expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: false, + other_text_changed: true, + selection_known: false, + committed_selected_count: undefined, + submitted_selected_count: 1 + } + }) ) expect(session.notices.value).toEqual([ { @@ -2507,14 +2527,17 @@ describe('useAgentSession (v1 composition root)', () => { name: string committed: Pick expected: AskUserResolution + changes: { selection_changed: boolean; other_text_changed: boolean } }>([ { name: 'a different option', + changes: { selection_changed: true, other_text_changed: true }, committed: { selected: ['flux'] }, expected: { status: 'answered', selected: ['flux'] } }, { name: 'a superset of it', + changes: { selection_changed: true, other_text_changed: false }, committed: { selected: ['sdxl', 'flux'], other_text: 'a LoRA' }, expected: { status: 'answered', @@ -2524,11 +2547,13 @@ describe('useAgentSession (v1 composition root)', () => { }, { name: 'a subset of it', + changes: { selection_changed: true, other_text_changed: false }, committed: { selected: [], other_text: 'a LoRA' }, expected: { status: 'answered', selected: [], otherText: 'a LoRA' } }, { name: 'free text alone', + changes: { selection_changed: true, other_text_changed: true }, committed: { selected: [], other_text: 'someone else' }, expected: { status: 'answered', @@ -2538,12 +2563,13 @@ describe('useAgentSession (v1 composition root)', () => { }, { name: 'the same option with other free text', + changes: { selection_changed: false, other_text_changed: true }, committed: { selected: ['sdxl'], other_text: 'a VAE' }, expected: { status: 'answered', selected: ['sdxl'], otherText: 'a VAE' } } ])( 'shows the committed answer when $name won', - async ({ committed, expected }) => { + async ({ committed, expected, changes }) => { const { session, emit } = await parkedOnQuestion() await session.answerAsk('turn-1:call-1', { selected: ['sdxl'], @@ -2568,7 +2594,10 @@ describe('useAgentSession (v1 composition root)', () => { expect(shown.resolution).toEqual(expected) expect(reportError).toHaveBeenCalledWith( expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: expect.objectContaining(changes) + }) ) } ) @@ -2896,7 +2925,14 @@ describe('useAgentSession (v1 composition root)', () => { expect(reportError).toHaveBeenCalledWith(expect.any(Error), { surface: 'agent', - errorType: 'agent_ask_answer_superseded' + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: true, + other_text_changed: false, + selection_known: true, + committed_selected_count: 1, + submitted_selected_count: 1 + } }) expect(session.notices.value).toHaveLength(1) expect(cardOnScreen()).toBe(false) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts index 9aac95a13b7..581d8c6efa9 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts @@ -344,16 +344,17 @@ function namesCommittedAnswer({ return Array.isArray(selected) || Boolean(other_text) } -function isSupersededAnswer( +function diffAnswers( submitted: AgentAnswerRequest, committed: Extract, selectionKnown: boolean -): boolean { - return ( - (selectionKnown && - xor(submitted.selected, committed.selected).length > 0) || - (submitted.other_text || undefined) !== committed.otherText - ) +) { + return { + selection_changed: + selectionKnown && xor(submitted.selected, committed.selected).length > 0, + other_text_changed: + (submitted.other_text || undefined) !== committed.otherText + } } let sessionGeneration = 0 @@ -1549,17 +1550,22 @@ export function useAgentSession(deps: AgentSessionDeps) { const committed = toAskResolution(source) if (committed.status !== 'answered') return const submitted = conversationStore.takeSubmittedAskAnswer(askId) - if ( - submitted === undefined || - !isSupersededAnswer(submitted, committed, Array.isArray(source.selected)) - ) - return - reportError( - new Error( - `agent ask resolved as ${committed.selected.join(',')}, not ${submitted.selected.join(',')}` - ), - { surface: 'agent', errorType: 'agent_ask_answer_superseded' } - ) + if (submitted === undefined) return + const selectionKnown = Array.isArray(source.selected) + const changes = diffAnswers(submitted, committed, selectionKnown) + if (!changes.selection_changed && !changes.other_text_changed) return + reportError(new Error('agent ask resolved with a different answer'), { + surface: 'agent', + errorType: 'agent_ask_answer_superseded', + tags: { + ...changes, + selection_known: selectionKnown, + committed_selected_count: selectionKnown + ? committed.selected.length + : undefined, + submitted_selected_count: submitted.selected.length + } + }) pushError(i18n.global.t('agent.runApproval.answerSuperseded')) }