diff --git a/src/core/graph/widgets/dynamicGroupDiagnostics.test.ts b/src/core/graph/widgets/dynamicGroupDiagnostics.test.ts new file mode 100644 index 00000000000..409210a6bb4 --- /dev/null +++ b/src/core/graph/widgets/dynamicGroupDiagnostics.test.ts @@ -0,0 +1,311 @@ +import { nodesMap } from '@comfyorg/comfy-multi-player' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +import { LGraph, LGraphNode, LiteGraph } from '@/lib/litegraph/src/litegraph' +import { remoteConfig } from '@/platform/remoteConfig/remoteConfig' +import { reportError } from '@/platform/telemetry/reportError' +import { useLitegraphService } from '@/services/litegraphService' +import { useLinkStore } from '@/stores/linkStore' +import { applyRemoteDynamicGroupRowCount } from './dynamicGroupDiagnostics' +import { toNodeId } from '@/types/nodeId' +import { followedDoc } from '@/workbench/extensions/agent/crdt/__fixtures__/followedDoc' +import { LiveGraphApplier } from '@/workbench/extensions/agent/crdt/liveGraphApplier' + +const distribution = vi.hoisted(() => ({ isCloud: false })) + +vi.mock(import('@/platform/distribution/types'), () => ({ + get isCloud() { + return distribution.isCloud + } +})) + +vi.mock(import('@/platform/telemetry/reportError')) + +function addGroup(node: LGraphNode, min = 0) { + useLitegraphService().addNodeInput(node, { + name: 'rows', + type: 'COMFY_DYNAMICGROUP_V3', + isOptional: false, + min, + max: 3, + template: { required: { strength: ['FLOAT', { default: 1 }] } } + }) +} + +function group(type: string, { min = 0, count = 2 } = {}) { + const graph = new LGraph() + const node = new LGraphNode('Group', type) + node.serialize_widgets = true + graph.add(node) + addGroup(node, min) + function widget(name: string) { + const found = node.widgets?.find((widget) => widget.name === name) + if (!found) throw new Error(`Missing widget ${name}`) + return found + } + widget('rows').value = count + return { node, widget } +} + +function refuseInputChanges() { + vi.spyOn(useLinkStore(), 'updateEndpoints').mockReturnValue({ + ok: false, + error: { code: 'unowned-topology', message: 'Rejected' } + }) +} + +beforeEach(() => { + remoteConfig.value = { enable_telemetry: true } +}) + +afterEach(() => { + distribution.isCloud = false + remoteConfig.value = {} +}) + +describe('DynamicGroup row refusal reports', () => { + function rowRefusal( + type: string, + operation: string, + requested: number, + actual: number + ) { + return [ + expect.objectContaining({ message: 'Refused a DynamicGroup row change' }), + { + errorType: 'failure_resizing_dynamic_group_rows', + surface: 'graph', + level: 'warning', + tags: { node_type: type, operation, in_subgraph: false }, + context: { requested, actual } + } + ] + } + + type Widget = ReturnType['widget'] + + it.for([ + { + name: 'an added row', + operation: 'add_row', + requested: 3, + change: (widget: Widget) => (widget('rows').value = 3) + }, + { + name: 'a count reduction', + operation: 'remove_row', + requested: 0, + change: (widget: Widget) => (widget('rows').value = 0) + }, + { + name: "a row's remove button", + operation: 'remove_row', + requested: 1, + change: (widget: Widget) => widget('rows.0').callback?.(undefined) + } + ])( + 'reports a refused change from $name with counts only', + ({ name, operation, requested, change }) => { + const { widget } = group(name) + refuseInputChanges() + + change(widget) + + expect(widget('rows').value).toBe(2) + expect(vi.mocked(reportError).mock.calls).toEqual([ + rowRefusal(name, operation, requested, 2) + ]) + } + ) + + it('does not report successful changes or a count clamped up to min', () => { + const { widget } = group('Accepted changes', { min: 1, count: 3 }) + widget('rows').value = -1 + widget('rows.$add').callback?.(undefined) + widget('rows.1').callback?.(undefined) + + expect(widget('rows').value).toBe(1) + expect(reportError).not.toHaveBeenCalled() + }) + + it('reports each node type and operation once per session', () => { + const first = group('Deduped refusal') + const second = group('Deduped refusal') + const other = group('Other refusal') + refuseInputChanges() + + first.widget('rows').value = 3 + first.widget('rows').value = 0 + first.widget('rows').value = 4 + second.widget('rows').value = 3 + other.widget('rows').value = 3 + + expect(vi.mocked(reportError).mock.calls).toEqual([ + rowRefusal('Deduped refusal', 'add_row', 3, 2), + rowRefusal('Deduped refusal', 'remove_row', 0, 2), + rowRefusal('Other refusal', 'add_row', 3, 2) + ]) + }) + + const resizeRefusal = ['failure_resizing_dynamic_group_rows'] + it.for([ + { isCloud: false, enableTelemetry: false, reported: [] }, + { isCloud: false, enableTelemetry: true, reported: resizeRefusal }, + { isCloud: true, enableTelemetry: false, reported: resizeRefusal } + ])( + 'sends with cloud=$isCloud and host telemetry=$enableTelemetry: $reported', + ({ isCloud, enableTelemetry, reported }) => { + distribution.isCloud = isCloud + remoteConfig.value = { enable_telemetry: enableTelemetry } + const { widget } = group(`Gated ${isCloud} ${enableTelemetry}`) + refuseInputChanges() + + widget('rows').value = 3 + + expect( + vi + .mocked(reportError) + .mock.calls.map(([, options]) => options.errorType) + ).toEqual(reported) + } + ) +}) + +describe('remote DynamicGroup row count writes', () => { + it('suppresses only the target node and restores an outer target', () => { + const outer = group('OuterRemoteTarget') + const inner = group('InnerRemoteTarget') + const other = group('LocalDuringRemote') + refuseInputChanges() + + applyRemoteDynamicGroupRowCount(outer.node, () => { + applyRemoteDynamicGroupRowCount(inner.node, () => {}) + outer.widget('rows').value = 3 + other.widget('rows').value = 3 + }) + + expect( + vi.mocked(reportError).mock.calls.map(([, options]) => options.tags) + ).toEqual([expect.objectContaining({ node_type: 'LocalDuringRemote' })]) + }) +}) + +describe('remote DynamicGroup row count reports', () => { + function remoteNodeType(min: number) { + return class extends LGraphNode { + constructor() { + super('Remote Dynamic Group') + this.serialize_widgets = true + addGroup(this, min) + } + } + } + + beforeEach(() => { + LiteGraph.registerNodeType('RemoteDynamicGroup', remoteNodeType(0)) + LiteGraph.registerNodeType('MinOneDynamicGroup', remoteNodeType(1)) + LiteGraph.registerNodeType('ConsentDynamicGroup', remoteNodeType(0)) + }) + + function remoteGroup(type = 'RemoteDynamicGroup') { + const graph = new LGraph() + const { doc, collector } = followedDoc( + { + nodes: [ + { + id: 1, + type, + pos: [0, 0], + size: [210, 100], + widgets_values: [1, 0.5] + } + ], + links: [] + }, + { + types: { + [type]: { + widget_order: ['rows', 'rows.0.strength', 'rows.1.strength'] + } + } + } + ) + const applier = new LiveGraphApplier({ getGraph: () => graph }) + const context = { actor: 'agent:test', opIds: ['op-1'] } + applier.applyChanges(doc, collector.take(), context) + const rows = graph + .getNodeById(toNodeId(1)) + ?.widgets?.find((widget) => widget.name === 'rows') + if (!rows) throw new Error('DynamicGroup node 1 was not created') + function setRows(count: unknown) { + doc.transact(() => { + const widgets = nodesMap(doc).get('1')?.get('widgets') + if (!(widgets instanceof Y.Map)) throw new Error('named storage') + widgets.set('rows', count) + }) + return applier.applyChanges(doc, collector.take(), context).failureCount + } + return { rows, setRows } + } + + it('applies a remote row count without reporting', () => { + const { rows, setRows } = remoteGroup() + + expect([setRows(2), setRows(0)]).toEqual([0, 0]) + expect(rows.value).toBe(0) + expect(reportError).not.toHaveBeenCalled() + }) + + it.for([ + { type: 'RemoteDynamicGroup', value: 2.5, expected: 2 }, + { type: 'RemoteDynamicGroup', value: -0, expected: 0 }, + { type: 'MinOneDynamicGroup', value: 0, expected: 1 } + ])( + 'does not report a remote $value the $type setter normalizes', + ({ type, value, expected }) => { + const { rows, setRows } = remoteGroup(type) + + expect(setRows(value)).toBe(0) + expect(rows.value).toBe(expected) + expect(reportError).not.toHaveBeenCalled() + } + ) + + it('reports a remote row count the live group did not reach, once per node type', () => { + const { rows, setRows } = remoteGroup() + refuseInputChanges() + + expect([setRows(3), setRows(2)]).toEqual([1, 1]) + expect(rows.value).toBe(1) + expect(vi.mocked(reportError).mock.calls).toEqual([ + [ + expect.objectContaining({ + message: 'Remote DynamicGroup row count did not apply' + }), + { + errorType: 'agent_graph_dynamic_group_rows_mismatch', + surface: 'agent', + tags: { + subsystem: 'agent-crdt', + layer: 'graph-api', + outcome: 'degraded', + node_type: 'RemoteDynamicGroup', + operation: 'remote_apply', + in_subgraph: false + }, + context: { requested: 3, actual: 1 } + } + ] + ]) + }) + + it('counts but does not send a mismatch without telemetry consent', () => { + remoteConfig.value = {} + const { setRows } = remoteGroup('ConsentDynamicGroup') + refuseInputChanges() + + expect(setRows(2)).toBe(1) + expect(reportError).not.toHaveBeenCalled() + }) +}) diff --git a/src/core/graph/widgets/dynamicGroupDiagnostics.ts b/src/core/graph/widgets/dynamicGroupDiagnostics.ts new file mode 100644 index 00000000000..d837b9188ab --- /dev/null +++ b/src/core/graph/widgets/dynamicGroupDiagnostics.ts @@ -0,0 +1,117 @@ +import type { LGraphNode } from '@/lib/litegraph/src/LGraphNode' +import type { IBaseWidget } from '@/lib/litegraph/src/types/widgets' +import { isCloud } from '@/platform/distribution/types' +import { isHostTelemetryEnabled } from '@/platform/telemetry/hostTelemetryEnabled' +import { reportError } from '@/platform/telemetry/reportError' +import { isSubgraph } from '@/utils/typeGuardUtil' + +type RowOperation = 'add_row' | 'remove_row' | 'remote_apply' + +const controllers = new WeakSet() +const reported = new Set() +let remoteRowCountNode: LGraphNode | undefined + +export function markDynamicGroupController(widget: IBaseWidget): void { + controllers.add(widget) +} + +/** True for the hidden widget whose value is a DynamicGroup's row count. */ +export function isDynamicGroupController(widget: IBaseWidget): boolean { + return controllers.has(widget) +} + +/** + * Runs a remote row count write to `node`. A refusal on that node inside it is + * reported once, as the remote mismatch, not also as a local row change. + */ +export function applyRemoteDynamicGroupRowCount( + node: LGraphNode, + write: () => void +): void { + const previous = remoteRowCountNode + remoteRowCountNode = node + try { + write() + } finally { + remoteRowCountNode = previous + } +} + +/** Whether a controller holds the row count its setter derives from `value`. */ +export function reachesDynamicGroupRowCount( + widget: IBaseWidget, + value: unknown +): boolean { + if (typeof value !== 'number' || !Number.isFinite(value)) return false + return widget.value === Math.max(widget.options.min ?? 0, Math.trunc(value)) +} + +function reportRowCount( + node: LGraphNode, + operation: RowOperation, + counts: { requested: number | null; actual: number }, + report: { + message: string + errorType: string + surface: 'graph' | 'agent' + level?: 'warning' + tags?: Record + } +): void { + const key = `${report.errorType}:${node.type}:${operation}` + if (!(isCloud || isHostTelemetryEnabled()) || reported.has(key)) return + reported.add(key) + reportError(new Error(report.message), { + errorType: report.errorType, + surface: report.surface, + level: report.level, + tags: { + ...report.tags, + node_type: node.type, + operation, + in_subgraph: isSubgraph(node.graph) + }, + context: counts + }) +} + +/** Reports a local DynamicGroup row change that left a different row count. */ +export function reportDynamicGroupRowRefusal( + node: LGraphNode, + operation: 'add_row' | 'remove_row', + requested: number, + actual: number +): void { + if (remoteRowCountNode === node) return + reportRowCount( + node, + operation, + { requested, actual }, + { + message: 'Refused a DynamicGroup row change', + errorType: 'failure_resizing_dynamic_group_rows', + surface: 'graph', + level: 'warning' + } + ) +} + +/** Reports a remote DynamicGroup row count the live group did not reach. */ +export function reportRemoteDynamicGroupRowMismatch( + node: LGraphNode, + requested: unknown, + actual: number, + tags: Record +): void { + reportRowCount( + node, + 'remote_apply', + { requested: typeof requested === 'number' ? requested : null, actual }, + { + message: 'Remote DynamicGroup row count did not apply', + errorType: 'agent_graph_dynamic_group_rows_mismatch', + surface: 'agent', + tags + } + ) +} diff --git a/src/core/graph/widgets/dynamicGroupWidget.ts b/src/core/graph/widgets/dynamicGroupWidget.ts index f1a643aa781..ad087e21ca4 100644 --- a/src/core/graph/widgets/dynamicGroupWidget.ts +++ b/src/core/graph/widgets/dynamicGroupWidget.ts @@ -24,6 +24,11 @@ import { deriveWidgetSurfaces } from '@/types/widgetVisibility' import { mapUniqueNodes } from '@/utils/graphTraversalUtil' import { isSubgraph } from '@/utils/typeGuardUtil' +import { + markDynamicGroupController, + reportDynamicGroupRowRefusal +} from './dynamicGroupDiagnostics' + export function dynamicGroupWidget( node: LGraphNode, inputName: string, @@ -46,6 +51,7 @@ export function dynamicGroupWidget( hidden: true, options: { min, max, socketless: true, serialize: false } }) + markDynamicGroupController(controller) node.addCustomWidget( new DynamicGroupNoticeWidget( { @@ -249,7 +255,14 @@ export function dynamicGroupWidget( callback: () => { if (rows().length <= min) return changeRows(() => { - removeRow(Number(header.name.slice(inputName.length + 1))) + const requested = rows().length - 1 + if (!removeRow(Number(header.name.slice(inputName.length + 1)))) + reportDynamicGroupRowRefusal( + node, + 'remove_row', + requested, + rows().length + ) publish() }) } @@ -381,6 +394,13 @@ export function dynamicGroupWidget( } publish() announceAddedValues(existing) + if (rowCount !== count) + reportDynamicGroupRowRefusal( + node, + rowCount > count ? 'remove_row' : 'add_row', + count, + rowCount + ) } }) controller.value = initialCount diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts index e53803fd765..b80e679501b 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts @@ -8,6 +8,12 @@ import { isEqual } from 'es-toolkit' import * as Y from 'yjs' import { z } from 'zod' +import { + applyRemoteDynamicGroupRowCount, + isDynamicGroupController, + reachesDynamicGroupRowCount, + reportRemoteDynamicGroupRowMismatch +} from '@/core/graph/widgets/dynamicGroupDiagnostics' import { growAutogrowInput } from '@/core/graph/widgets/dynamicWidgets' import type { INodeFlags } from '@/lib/litegraph/src/interfaces' import type { INodeInputSlot } from '@/lib/litegraph/src/types/slots' @@ -918,7 +924,9 @@ export class LiveGraphApplier { this.holdsLocalWrite(node, widget.name, value) ) continue - this.setWidgetValue(node, widget, value) + if (isDynamicGroupController(widget)) + this.applyDynamicGroupRowCount(node, widget, value) + else this.setWidgetValue(node, widget, value) resolved = true } return { resolved, unresolved } @@ -1046,6 +1054,22 @@ export class LiveGraphApplier { ) } + private applyDynamicGroupRowCount( + node: LGraphNode, + widget: IBaseWidget, + value: WidgetValue + ) { + applyRemoteDynamicGroupRowCount(node, () => + this.setWidgetValue(node, widget, value) + ) + if (reachesDynamicGroupRowCount(widget, value)) return + this.incrementFailureCount() + reportRemoteDynamicGroupRowMismatch(node, value, Number(widget.value), { + ...AGENT_APPLY_TAGS, + outcome: 'degraded' + }) + } + private setWidgetValue( node: LGraphNode, widget: IBaseWidget,