Repository navigation
fix(agent): say when stored reasoning was shortened (BE-24672) #22989
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
9247674
dd0ce09
06251d3
ac35890
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| import { expect } from '@playwright/test' | ||
|
|
||
| import { promptHistoryTest as test } from '@e2e/fixtures/agentPromptHistoryFixture' | ||
| import enMessages from '@/locales/en/main.json' with { type: 'json' } | ||
| import type { PersistedThinkingEntry } from '@/workbench/extensions/agent/schemas/agentApiSchema' | ||
|
|
||
| test.describe.configure({ timeout: 120_000 }) | ||
| test.use({ connectWebSocketToServer: false }) | ||
|
|
||
| const cutRound: PersistedThinkingEntry = { | ||
| text: 'Picking an upscaler, then', | ||
| truncated: true | ||
| } | ||
| const wholeRound: PersistedThinkingEntry = { text: 'Reading the workflow.' } | ||
|
|
||
| test( | ||
| 'BE-24672: tells the reader a reloaded reasoning trace was shortened', | ||
| { tag: ['@cloud', '@ui'] }, | ||
| async ({ agentPanel, promptHistory, workflowSelection }) => { | ||
| await test.step('completes a turn the server stored with a cut round', async () => { | ||
| await agentPanel.open() | ||
| await agentPanel.chooseWorkflow() | ||
| await expect.poll(() => workflowSelection.savedPaths.length).toBe(1) | ||
| workflowSelection.finishSave(true) | ||
| await agentPanel.sendMessage('Upscale it') | ||
| await expect.poll(() => promptHistory.requests.length).toBe(1) | ||
|
|
||
| promptHistory.completeLatestTurn({ | ||
| text: 'Done.', | ||
| thinking: [wholeRound, cutRound] | ||
| }) | ||
| await agentPanel.stopButton.click() | ||
| }) | ||
|
|
||
| await test.step('reloads the persisted transcript', async () => { | ||
| await agentPanel.reload() | ||
| }) | ||
|
|
||
| await test.step('names the shortening once, on the round that was cut', async () => { | ||
| await agentPanel.openWorkSummary() | ||
| await expect(agentPanel.thinkingShortenedNotices).toHaveText([ | ||
| enMessages.agent.thinkingShortened | ||
| ]) | ||
| await expect(agentPanel.activityRows.last()).toContainText(cutRound.text) | ||
|
Comment on lines
+41
to
+44
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| }) | ||
| } | ||
| ) | ||
|
|
||
| test( | ||
| 'BE-24672: stays silent when the reloaded reasoning trace is whole', | ||
| { tag: ['@cloud', '@ui'] }, | ||
| async ({ agentPanel, promptHistory, workflowSelection }) => { | ||
| await test.step('completes a turn the server stored whole', async () => { | ||
| await agentPanel.open() | ||
| await agentPanel.chooseWorkflow() | ||
| await expect.poll(() => workflowSelection.savedPaths.length).toBe(1) | ||
| workflowSelection.finishSave(true) | ||
| await agentPanel.sendMessage('Upscale it') | ||
| await expect.poll(() => promptHistory.requests.length).toBe(1) | ||
|
|
||
| promptHistory.completeLatestTurn({ | ||
| text: 'Done.', | ||
| thinking: [wholeRound] | ||
| }) | ||
| await agentPanel.stopButton.click() | ||
| }) | ||
|
|
||
| await test.step('reloads the persisted transcript', async () => { | ||
| await agentPanel.reload() | ||
| }) | ||
|
|
||
| await test.step('shows the round with no shortening notice', async () => { | ||
| await agentPanel.openWorkSummary() | ||
| await expect(agentPanel.activityRows.last()).toContainText( | ||
| wholeRound.text | ||
| ) | ||
| await expect(agentPanel.thinkingShortenedNotices).toHaveCount(0) | ||
| }) | ||
| } | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| import { render, screen } from '@testing-library/vue' | ||
| import { describe, expect, it } from 'vitest' | ||
|
|
||
| import { i18n } from '@/i18n' | ||
| import type { ActivityPart } from '@/workbench/extensions/agent/services/agent/agentMessageParts' | ||
|
|
||
| import ActivityTrace from './ActivityTrace.vue' | ||
|
|
||
| function renderTrace(parts: ActivityPart[]) { | ||
| return render(ActivityTrace, { | ||
| props: { parts }, | ||
| global: { plugins: [i18n] } | ||
| }) | ||
| } | ||
|
|
||
| describe('ActivityTrace', () => { | ||
| it('BE-24672: says the reasoning was shortened where the stored thinking stops being whole', () => { | ||
| renderTrace([ | ||
| { type: 'thinking', text: 'Round one.', state: 'done' }, | ||
| { type: 'thinking', text: 'Round two…', state: 'done', truncated: true } | ||
| ]) | ||
|
|
||
| const notices = screen.getAllByTestId('agent-thinking-shortened') | ||
| expect(notices).toHaveLength(1) | ||
| expect(notices[0].textContent).toBe( | ||
| 'Reasoning shortened: the rest was too long to save.' | ||
| ) | ||
| }) | ||
|
|
||
| it('shows no notice for a whole trace', () => { | ||
| renderTrace([{ type: 'thinking', text: 'Round one.', state: 'done' }]) | ||
|
|
||
| expect(screen.queryByTestId('agent-thinking-shortened')).toBeNull() | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -149,11 +149,15 @@ export type PersistedToolCallSummary = z.infer<typeof zPersistedToolCallSummary> | |
|
|
||
| /** | ||
| * One persisted `content.thinking` entry: a model round's thinking text, | ||
| * anchored to that round's first tool call when it made one. | ||
| * anchored to that round's first tool call, `truncated` where it was cut. | ||
| */ | ||
| export const zPersistedThinkingEntry = zAgentThinkingEntry | ||
| .pick({ text: true }) | ||
| .extend({ tool_call_id: z.string().nullish() }) | ||
| .extend({ | ||
| tool_call_id: z.string().nullish(), | ||
| truncated: z.boolean().nullish() | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. issue:
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Taking the contract-first ordering rather than arguing for the handwritten field, so this PR should wait. Two things I measured that pin down what it is waiting on. cloud#18234 adds Once that merges and the regenerated One thing worth keeping either way: the behaviour had component and service coverage but nothing driving the actual panel, and the notice only ever renders on a chat reload. |
||
| }) | ||
| export type PersistedThinkingEntry = z.infer<typeof zPersistedThinkingEntry> | ||
|
|
||
| /** | ||
| * The message-parse boundary's `content.tool_calls` field is deliberately | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Move the round constants out of the spec.
cutRoundandwholeRoundare top-level constants in the spec. The browser test guide does not allow free-standing constants in a spec file. Move both constants to a typed fixture or data module underbrowser_tests/fixtures/, then import them into the spec. Keep thePersistedThinkingEntrytype annotation so the data stays typed.