Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions browser_tests/fixtures/agentFollowerHostSocket.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ export class AgentFollowerHostSocket {
private refusedSubscribes = 0

private socket: WebSocketRoute | null = null
private agentProcessDown = false
private refusedConnections = 0
private comfySocket: WebSocketRoute | null = null
private comfyConnections = 0
private connections = 0
Expand Down Expand Up @@ -229,12 +231,35 @@ export class AgentFollowerHostSocket {
)
})
await this.page.routeWebSocket(AGENT_SOCKET_URL, (socket) => {
if (this.agentProcessDown) {
this.refusedConnections += 1
void socket.close()
return
}
this.connections += 1
this.socket = socket
socket.onMessage((raw) => this.onClientFrame(raw))
})
}

/** Refuses reconnect attempts until the outage ends. */
async beginAgentOutage(): Promise<void> {
this.agentProcessDown = true
const socket = this.socket
this.socket = null
await socket?.close()
}

/** Allows the next reconnect attempt. */
endAgentOutage(): void {
this.agentProcessDown = false
}

/** Returns the number of refused reconnect attempts. */
refusedConnectionCount(): number {
return this.refusedConnections
}

send(frame: AgentWsEvent | AgentAskServerFrame | HostFrame): void {
if (frame.type.startsWith('doc_') || frame.type === 'awareness') {
if (parseServerDocFrame(frame) === null)
Expand Down
237 changes: 237 additions & 0 deletions browser_tests/fixtures/agentOfflineEditFixture.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
import type { APIRequestContext, Locator, Page } from '@playwright/test'

import type { WidgetCatalog } from '@comfyorg/comfy-multi-player'
import type { AgentPostMessageRequest } from '@comfyorg/ingest-types'
import { zAgentPostMessageRequest } from '@comfyorg/ingest-types/zod'

import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema'

import {
agentTest,
bootAgentApp,
mockAgentTurnApi
} from '@e2e/fixtures/agentPanelFixture'
import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc'
import { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket'
import { AgentPanel } from '@e2e/fixtures/components/AgentPanel'
import { Topbar } from '@e2e/fixtures/components/Topbar'
import { ComfyPage } from '@e2e/fixtures/ComfyPage'
import type { RecordedGraphOperation } from '@e2e/fixtures/data/agent/agentConversation'
import {
AGENT_NODE_IDS,
emptySeed
} from '@e2e/fixtures/data/agent/agentRemoteApply'
import { agentReplayNodeDefs } from '@e2e/fixtures/data/agentReplayNodeDefs'
import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute'
import { mockSavedWorkflowPersistence } from '@e2e/fixtures/utils/savedWorkflowPersistence'

const WORKFLOW_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10001'
const THREAD_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10002'
const FIRST_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10003'
const SECOND_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10004'
const NODE_IDS = [AGENT_NODE_IDS[0], AGENT_NODE_IDS[1]] as const
const AGENT_PROMPTS = ['agent prompt one', 'agent prompt two'] as const
const HUMAN_PROMPTS = ['human prompt one', 'human prompt two'] as const
const CLASS_TYPE = 'CLIPTextEncode'
const SAVED_NAME = 'pm-2392-offline-edit'

const catalog: WidgetCatalog = {
types: { [CLASS_TYPE]: { widget_order: ['text'] } }
}

type SavedWorkflow = Awaited<ReturnType<typeof mockSavedWorkflowPersistence>>

class AgentOfflineEditScenario {
public readonly agentPrompts = AGENT_PROMPTS
public readonly humanPrompts = HUMAN_PROMPTS

private readonly comfyPage: ComfyPage
private readonly panel: AgentPanel
private readonly topbar: Topbar

constructor(
private readonly page: Page,
request: APIRequestContext,
private readonly host: HostDoc,
private readonly hostSocket: AgentFollowerHostSocket,
private readonly saved: SavedWorkflow,
private readonly posts: AgentPostMessageRequest[]
) {
this.comfyPage = new ComfyPage(page, request)
this.panel = new AgentPanel(page)
this.topbar = new Topbar(page)
}

addPromptNode(nodeId: number, index: number): RecordedGraphOperation {
const pos = [120 + index * 320, 140]
return {
op: 'add_node',
pos,
node: {
id: nodeId,
pos,
mode: 0,
size: [300, 200],
type: CLASS_TYPE,
flags: {},
order: index,
inputs: [],
outputs: [],
properties: {},
widgets_values: [AGENT_PROMPTS[index]]
},
node_id: nodeId,
class_type: CLASS_TYPE
}
}

done(messageId: string): AgentWsEvent {
return {
type: 'agent_message_done',
data: { thread_id: THREAD_ID, message_id: messageId, usage: null }
}
}

promptField(index: number): Locator {
return this.comfyPage.vueNodes
.getNodeLocator(String(NODE_IDS[index]))
.getByLabel('text', { exact: true })
}

async canvasPromptValues(): Promise<string[]> {
return await Promise.all(
NODE_IDS.map((_, index) => this.promptField(index).inputValue())
)
}

documentPromptValues(): unknown[] {
return NODE_IDS.map((nodeId) => {
const widgets = this.host.graph().nodes[String(nodeId)]?.widgets as
| Record<string, unknown>
| undefined
return widgets?.text
})
}

subscribeCount(): number {
return this.hostSocket.subscribeCount()
}

refusedConnectionCount(): number {
return this.hostSocket.refusedConnectionCount()
}

postCount(): number {
return this.posts.length
}

draftAt(index: number): string {
return JSON.stringify(this.posts[index]?.draft ?? {})
}

savedContent(): string {
return this.saved.savedContent() ?? ''
}

async buildAgentPromptNodes(): Promise<void> {
await this.panel.open()
await this.panel.selectWorkflow()
await this.panel.sendMessage('Add two prompt nodes.')
await this.hostSocket.waitForSubscribe()
this.hostSocket.send(
this.host.apply([
this.addPromptNode(NODE_IDS[0], 0),
this.addPromptNode(NODE_IDS[1], 1)
])
)
this.hostSocket.send(this.done(FIRST_MESSAGE_ID))
}

async beginOutageAndRetype(): Promise<void> {
await this.hostSocket.beginAgentOutage()
for (const [index, value] of HUMAN_PROMPTS.entries()) {
await this.promptField(index).fill(value)
await this.promptField(index).press('Tab')
}
}

restoreAgent(): void {
this.hostSocket.answerSubscribesWith(() => this.host.initialSync())
this.hostSocket.endAgentOutage()
}

async saveWorkflow(): Promise<void> {
await this.topbar.saveWorkflowAs(SAVED_NAME)
}

async reloadSavedWorkflow(): Promise<boolean> {
const restored = this.page.waitForResponse(
(response) =>
response.request().method() === 'GET' &&
decodeURIComponent(new URL(response.url()).pathname) ===
`/api/userdata/workflows/${SAVED_NAME}.json`,
{ timeout: 90_000 }
)
await this.page.reload({ waitUntil: 'domcontentloaded' })
await this.panel.open()
await this.panel.selectWorkflow(SAVED_NAME)
return (await restored).ok()
}

async askNextTurn(): Promise<void> {
await this.panel.sendMessage('Which prompts am I using?')
}
}

type AgentOfflineEditFixtures = {
offlineEdit: AgentOfflineEditScenario
}

export const agentOfflineEditTest = agentTest.extend<AgentOfflineEditFixtures>({
offlineEdit: async ({ page, request, agentFlagEnabled }, use) => {
const host = new HostDoc(WORKFLOW_ID, emptySeed, catalog)
const hostSocket = new AgentFollowerHostSocket(
page,
WORKFLOW_ID,
host,
'apply'
)
await page.route('**/api/object_info', (route) =>
route.fulfill(jsonRoute(agentReplayNodeDefs))
)
await hostSocket.install()
await mockAgentTurnApi(page, {
thread_id: THREAD_ID,
message_id: FIRST_MESSAGE_ID,
workflow_id: WORKFLOW_ID
})

const posts: AgentPostMessageRequest[] = []
await page.route('**/api/agent/threads/*/messages', (route) => {
if (route.request().method() !== 'POST') return route.fallback()
const parsed: unknown = JSON.parse(route.request().postData() ?? '{}')
posts.push(zAgentPostMessageRequest.parse(parsed))
return route.fulfill(
jsonRoute({
thread_id: THREAD_ID,
message_id: posts.length === 1 ? FIRST_MESSAGE_ID : SECOND_MESSAGE_ID,
workflow_id: WORKFLOW_ID
})
)
})
await bootAgentApp(page, agentFlagEnabled, {
objectInfo: 'server',
settings: { 'Comfy.Graph.CanvasInfo': false }
})
const saved = await mockSavedWorkflowPersistence(page, WORKFLOW_ID)
const scenario = new AgentOfflineEditScenario(
page,
request,
host,
hostSocket,
saved,
posts
)
await use(scenario)
}
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import { expect } from '@playwright/test'

import { agentOfflineEditTest as test } from '@e2e/fixtures/agentOfflineEditFixture'

test.describe(
'Human edits made while the agent process is down',
{ tag: ['@cloud', '@agent', '@vue-nodes', '@widget'] },
() => {
test.describe.configure({ timeout: 180_000 })

test(
'survive the reconnect, the save, a reload and the next agent turn',
{
annotation: [
{
type: 'issue',
description:
'https://linear.app/comfyorg/issue/PM-2392/agent-reconnect-overwrites-load-image-edits-saved-during-the-outage'
}
]
},
async ({ offlineEdit }) => {
await test.step('the agent builds two prompt nodes', async () => {
await offlineEdit.buildAgentPromptNodes()
await expect
.poll(() => offlineEdit.canvasPromptValues())
.toEqual(offlineEdit.agentPrompts)
})

const subscribesBeforeOutage = offlineEdit.subscribeCount()

await test.step('the agent process goes down and the user retypes both prompts', async () => {
await offlineEdit.beginOutageAndRetype()
await expect
.poll(() => offlineEdit.refusedConnectionCount(), {
timeout: 90_000,
message: 'the page keeps trying while the agent is down'
})
.toBeGreaterThanOrEqual(4)
await expect
.poll(() => offlineEdit.canvasPromptValues())
.toEqual(offlineEdit.humanPrompts)
})

await test.step('the agent comes back and catches the canvas up', async () => {
offlineEdit.restoreAgent()
await expect
.poll(() => offlineEdit.subscribeCount(), { timeout: 90_000 })
.toBeGreaterThan(subscribesBeforeOutage)
await expect
.poll(() => offlineEdit.canvasPromptValues(), {
timeout: 60_000,
message: 'the reconnect must not overwrite the offline edit'
})
.toEqual(offlineEdit.humanPrompts)
await expect
.poll(() => offlineEdit.documentPromptValues(), {
timeout: 60_000,
message: 'the held edit reaches the document on reconnect'
})
.toEqual(offlineEdit.humanPrompts)
})

await test.step('the save after the reconnect still holds them', async () => {
await offlineEdit.saveWorkflow()
await expect
.poll(() => offlineEdit.savedContent())
.toContain(offlineEdit.humanPrompts[0])
const content = offlineEdit.savedContent()
expect(content).toContain(offlineEdit.humanPrompts[1])
expect(content).not.toContain(offlineEdit.agentPrompts[0])
expect(content).not.toContain(offlineEdit.agentPrompts[1])
})

await test.step('a reload reopens them from the server', async () => {
expect(await offlineEdit.reloadSavedWorkflow()).toBe(true)
await expect
.poll(() => offlineEdit.canvasPromptValues(), { timeout: 60_000 })
.toEqual(offlineEdit.humanPrompts)
})

await test.step('the next agent turn reads them too', async () => {
const postsBefore = offlineEdit.postCount()
await offlineEdit.askNextTurn()
await expect.poll(() => offlineEdit.postCount()).toBe(postsBefore + 1)
const draft = offlineEdit.draftAt(postsBefore)
expect(draft).toContain(offlineEdit.humanPrompts[0])
expect(draft).toContain(offlineEdit.humanPrompts[1])
expect(draft).not.toContain(offlineEdit.agentPrompts[0])
expect(draft).not.toContain(offlineEdit.agentPrompts[1])
})
}
)
}
)
Loading
Loading