diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts index 4cacf0d27b4..c9d94ced6a1 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts @@ -2232,8 +2232,19 @@ describe('useAgentSession (v1 composition root)', () => { resolution: { status: 'answered', selected: ['flux'] } }) expect(reportError).toHaveBeenCalledWith( - expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + message: 'agent ask resolved with a different answer' + }), + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: true, + other_text_changed: false, + selection_known: true, + committed_selected_count: 1, + submitted_selected_count: 1 + } + }) ) }) @@ -2336,7 +2347,16 @@ describe('useAgentSession (v1 composition root)', () => { }) expect(reportError).toHaveBeenCalledWith( expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: false, + other_text_changed: true, + selection_known: false, + committed_selected_count: undefined, + submitted_selected_count: 1 + } + }) ) expect(session.notices.value).toEqual([ { @@ -2532,14 +2552,17 @@ describe('useAgentSession (v1 composition root)', () => { name: string committed: Pick expected: AskUserResolution + changes: { selection_changed: boolean; other_text_changed: boolean } }>([ { name: 'a different option', + changes: { selection_changed: true, other_text_changed: true }, committed: { selected: ['flux'] }, expected: { status: 'answered', selected: ['flux'] } }, { name: 'a superset of it', + changes: { selection_changed: true, other_text_changed: false }, committed: { selected: ['sdxl', 'flux'], other_text: 'a LoRA' }, expected: { status: 'answered', @@ -2549,11 +2572,13 @@ describe('useAgentSession (v1 composition root)', () => { }, { name: 'a subset of it', + changes: { selection_changed: true, other_text_changed: false }, committed: { selected: [], other_text: 'a LoRA' }, expected: { status: 'answered', selected: [], otherText: 'a LoRA' } }, { name: 'free text alone', + changes: { selection_changed: true, other_text_changed: true }, committed: { selected: [], other_text: 'someone else' }, expected: { status: 'answered', @@ -2563,12 +2588,13 @@ describe('useAgentSession (v1 composition root)', () => { }, { name: 'the same option with other free text', + changes: { selection_changed: false, other_text_changed: true }, committed: { selected: ['sdxl'], other_text: 'a VAE' }, expected: { status: 'answered', selected: ['sdxl'], otherText: 'a VAE' } } ])( 'shows the committed answer when $name won', - async ({ committed, expected }) => { + async ({ committed, expected, changes }) => { const { session, emit } = await parkedOnQuestion() await session.answerAsk('turn-1:call-1', { selected: ['sdxl'], @@ -2593,7 +2619,10 @@ describe('useAgentSession (v1 composition root)', () => { expect(shown.resolution).toEqual(expected) expect(reportError).toHaveBeenCalledWith( expect.any(Error), - expect.objectContaining({ errorType: 'agent_ask_answer_superseded' }) + expect.objectContaining({ + errorType: 'agent_ask_answer_superseded', + tags: expect.objectContaining(changes) + }) ) } ) @@ -2921,7 +2950,14 @@ describe('useAgentSession (v1 composition root)', () => { expect(reportError).toHaveBeenCalledWith(expect.any(Error), { surface: 'agent', - errorType: 'agent_ask_answer_superseded' + errorType: 'agent_ask_answer_superseded', + tags: { + selection_changed: true, + other_text_changed: false, + selection_known: true, + committed_selected_count: 1, + submitted_selected_count: 1 + } }) expect(session.notices.value).toHaveLength(1) expect(cardOnScreen()).toBe(false) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts index db7f4899e92..c5bca4f079d 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts @@ -345,16 +345,17 @@ function namesCommittedAnswer({ return Array.isArray(selected) || Boolean(other_text) } -function isSupersededAnswer( +function diffAnswers( submitted: AgentAnswerRequest, committed: Extract, selectionKnown: boolean -): boolean { - return ( - (selectionKnown && - xor(submitted.selected, committed.selected).length > 0) || - (submitted.other_text || undefined) !== committed.otherText - ) +) { + return { + selection_changed: + selectionKnown && xor(submitted.selected, committed.selected).length > 0, + other_text_changed: + (submitted.other_text || undefined) !== committed.otherText + } } let sessionGeneration = 0 @@ -1551,17 +1552,22 @@ export function useAgentSession(deps: AgentSessionDeps) { const committed = toAskResolution(source) if (committed.status !== 'answered') return const submitted = conversationStore.takeSubmittedAskAnswer(askId) - if ( - submitted === undefined || - !isSupersededAnswer(submitted, committed, Array.isArray(source.selected)) - ) - return - reportError( - new Error( - `agent ask resolved as ${committed.selected.join(',')}, not ${submitted.selected.join(',')}` - ), - { surface: 'agent', errorType: 'agent_ask_answer_superseded' } - ) + if (submitted === undefined) return + const selectionKnown = Array.isArray(source.selected) + const changes = diffAnswers(submitted, committed, selectionKnown) + if (!changes.selection_changed && !changes.other_text_changed) return + reportError(new Error('agent ask resolved with a different answer'), { + surface: 'agent', + errorType: 'agent_ask_answer_superseded', + tags: { + ...changes, + selection_known: selectionKnown, + committed_selected_count: selectionKnown + ? committed.selected.length + : undefined, + submitted_selected_count: submitted.selected.length + } + }) pushError(i18n.global.t('agent.runApproval.answerSuperseded')) }