From 490f958345c1e3e7f21882e21391166b0cd6c6ec Mon Sep 17 00:00:00 2001 From: jaeone lee <89377375+jaeone94@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:02:12 +0000 Subject: [PATCH] feat: report DynamicGroup row refusals and remote count mismatches (#20993) ## Summary When DynamicGroup refuses a row change, or a remote frame sets a row count that the live group does not reach, nothing is reported in production ([DynamicGroup architecture and safety review](https://app.notion.com/p/comfy-org/DynamicGroup-architecture-and-safety-review-3f06d73d365081679352c7032c88fd1c), "Operational monitoring"). This PR reports both cases through `reportError()`: - `failure_resizing_dynamic_group_rows` (`surface: 'graph'`, `level: 'warning'`): sent when the controller setter or a row's remove button ends with a different row count than requested. Today these failures are swallowed by `break`. A count clamped up to `min` is not reported. - `agent_graph_dynamic_group_rows_mismatch` (`surface: 'agent'`, existing `agent-crdt` / `graph-api` / `degraded` tags): sent when `LiveGraphApplier` writes a DynamicGroup count and the live value read back afterwards differs from the document's value. It counts toward `apply_failure_count`. Per AGENT-OBSERVABILITY-0034, only degradation is reported. There is no positive "effect observed" event. Tags are `node_type`, `operation` (`add_row` | `remove_row` | `remote_apply`) and `in_subgraph`. Context is `{ requested, actual }`, numbers only. Reports never include widget values, labels, input names or workflow content, and the message strings are constant. Each (error type, node type, operation) is reported at most once per session. Consent: Sentry also runs in local builds, and the repo has no shared gate for this, so both reports are sent only when `isCloud || isHostTelemetryEnabled()`. The reporters live in a new `dynamicGroupDiagnostics.ts`. Keeping them out of the node-widgets and agent domains avoids new cross-domain imports, which the Fallow boundary audit would reject. Follow-ups, not in this PR: signal 3, orphaned promotions after row removal; signal 5, trailing widgets displaced on positional restore (the named-vs-positional shadow diff from #14375 partly covers this). Stacked on #20992. Part of CORE-186 ## Verification - `dynamicGroupDiagnostics.test.ts` (11 tests) covers: refused add, refused count reduction and refused remove button; no report on success or on a `min` clamp; dedupe; the three consent cases (local without host telemetry, Desktop with telemetry, Cloud); remote mismatch reported once and counted; remote success not reported; remote mismatch without consent counted but not sent. - Red: with the two call sites reverted, 8 of 11 fail (the 3 that still pass assert no report). Mutations that open the gate, drop the Cloud branch, remove the dedupe, or compare against the unclamped count each turn the matching test red. - `vitest run src/core/graph/widgets src/workbench/extensions/agent/crdt` passes (1177 tests). Typecheck, oxlint, eslint, oxfmt, knip and the Fallow audit pass through the commit and push hooks. --- .../widgets/dynamicGroupDiagnostics.test.ts | 311 ++++++++++++++++++ .../graph/widgets/dynamicGroupDiagnostics.ts | 117 +++++++ src/core/graph/widgets/dynamicGroupWidget.ts | 22 +- .../extensions/agent/crdt/liveGraphApplier.ts | 26 +- 4 files changed, 474 insertions(+), 2 deletions(-) create mode 100644 src/core/graph/widgets/dynamicGroupDiagnostics.test.ts create mode 100644 src/core/graph/widgets/dynamicGroupDiagnostics.ts diff --git a/src/core/graph/widgets/dynamicGroupDiagnostics.test.ts b/src/core/graph/widgets/dynamicGroupDiagnostics.test.ts new file mode 100644 index 00000000000..409210a6bb4 --- /dev/null +++ b/src/core/graph/widgets/dynamicGroupDiagnostics.test.ts @@ -0,0 +1,311 @@ +import { nodesMap } from '@comfyorg/comfy-multi-player' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +import { LGraph, LGraphNode, LiteGraph } from '@/lib/litegraph/src/litegraph' +import { remoteConfig } from '@/platform/remoteConfig/remoteConfig' +import { reportError } from '@/platform/telemetry/reportError' +import { useLitegraphService } from '@/services/litegraphService' +import { useLinkStore } from '@/stores/linkStore' +import { applyRemoteDynamicGroupRowCount } from './dynamicGroupDiagnostics' +import { toNodeId } from '@/types/nodeId' +import { followedDoc } from '@/workbench/extensions/agent/crdt/__fixtures__/followedDoc' +import { LiveGraphApplier } from '@/workbench/extensions/agent/crdt/liveGraphApplier' + +const distribution = vi.hoisted(() => ({ isCloud: false })) + +vi.mock(import('@/platform/distribution/types'), () => ({ + get isCloud() { + return distribution.isCloud + } +})) + +vi.mock(import('@/platform/telemetry/reportError')) + +function addGroup(node: LGraphNode, min = 0) { + useLitegraphService().addNodeInput(node, { + name: 'rows', + type: 'COMFY_DYNAMICGROUP_V3', + isOptional: false, + min, + max: 3, + template: { required: { strength: ['FLOAT', { default: 1 }] } } + }) +} + +function group(type: string, { min = 0, count = 2 } = {}) { + const graph = new LGraph() + const node = new LGraphNode('Group', type) + node.serialize_widgets = true + graph.add(node) + addGroup(node, min) + function widget(name: string) { + const found = node.widgets?.find((widget) => widget.name === name) + if (!found) throw new Error(`Missing widget ${name}`) + return found + } + widget('rows').value = count + return { node, widget } +} + +function refuseInputChanges() { + vi.spyOn(useLinkStore(), 'updateEndpoints').mockReturnValue({ + ok: false, + error: { code: 'unowned-topology', message: 'Rejected' } + }) +} + +beforeEach(() => { + remoteConfig.value = { enable_telemetry: true } +}) + +afterEach(() => { + distribution.isCloud = false + remoteConfig.value = {} +}) + +describe('DynamicGroup row refusal reports', () => { + function rowRefusal( + type: string, + operation: string, + requested: number, + actual: number + ) { + return [ + expect.objectContaining({ message: 'Refused a DynamicGroup row change' }), + { + errorType: 'failure_resizing_dynamic_group_rows', + surface: 'graph', + level: 'warning', + tags: { node_type: type, operation, in_subgraph: false }, + context: { requested, actual } + } + ] + } + + type Widget = ReturnType['widget'] + + it.for([ + { + name: 'an added row', + operation: 'add_row', + requested: 3, + change: (widget: Widget) => (widget('rows').value = 3) + }, + { + name: 'a count reduction', + operation: 'remove_row', + requested: 0, + change: (widget: Widget) => (widget('rows').value = 0) + }, + { + name: "a row's remove button", + operation: 'remove_row', + requested: 1, + change: (widget: Widget) => widget('rows.0').callback?.(undefined) + } + ])( + 'reports a refused change from $name with counts only', + ({ name, operation, requested, change }) => { + const { widget } = group(name) + refuseInputChanges() + + change(widget) + + expect(widget('rows').value).toBe(2) + expect(vi.mocked(reportError).mock.calls).toEqual([ + rowRefusal(name, operation, requested, 2) + ]) + } + ) + + it('does not report successful changes or a count clamped up to min', () => { + const { widget } = group('Accepted changes', { min: 1, count: 3 }) + widget('rows').value = -1 + widget('rows.$add').callback?.(undefined) + widget('rows.1').callback?.(undefined) + + expect(widget('rows').value).toBe(1) + expect(reportError).not.toHaveBeenCalled() + }) + + it('reports each node type and operation once per session', () => { + const first = group('Deduped refusal') + const second = group('Deduped refusal') + const other = group('Other refusal') + refuseInputChanges() + + first.widget('rows').value = 3 + first.widget('rows').value = 0 + first.widget('rows').value = 4 + second.widget('rows').value = 3 + other.widget('rows').value = 3 + + expect(vi.mocked(reportError).mock.calls).toEqual([ + rowRefusal('Deduped refusal', 'add_row', 3, 2), + rowRefusal('Deduped refusal', 'remove_row', 0, 2), + rowRefusal('Other refusal', 'add_row', 3, 2) + ]) + }) + + const resizeRefusal = ['failure_resizing_dynamic_group_rows'] + it.for([ + { isCloud: false, enableTelemetry: false, reported: [] }, + { isCloud: false, enableTelemetry: true, reported: resizeRefusal }, + { isCloud: true, enableTelemetry: false, reported: resizeRefusal } + ])( + 'sends with cloud=$isCloud and host telemetry=$enableTelemetry: $reported', + ({ isCloud, enableTelemetry, reported }) => { + distribution.isCloud = isCloud + remoteConfig.value = { enable_telemetry: enableTelemetry } + const { widget } = group(`Gated ${isCloud} ${enableTelemetry}`) + refuseInputChanges() + + widget('rows').value = 3 + + expect( + vi + .mocked(reportError) + .mock.calls.map(([, options]) => options.errorType) + ).toEqual(reported) + } + ) +}) + +describe('remote DynamicGroup row count writes', () => { + it('suppresses only the target node and restores an outer target', () => { + const outer = group('OuterRemoteTarget') + const inner = group('InnerRemoteTarget') + const other = group('LocalDuringRemote') + refuseInputChanges() + + applyRemoteDynamicGroupRowCount(outer.node, () => { + applyRemoteDynamicGroupRowCount(inner.node, () => {}) + outer.widget('rows').value = 3 + other.widget('rows').value = 3 + }) + + expect( + vi.mocked(reportError).mock.calls.map(([, options]) => options.tags) + ).toEqual([expect.objectContaining({ node_type: 'LocalDuringRemote' })]) + }) +}) + +describe('remote DynamicGroup row count reports', () => { + function remoteNodeType(min: number) { + return class extends LGraphNode { + constructor() { + super('Remote Dynamic Group') + this.serialize_widgets = true + addGroup(this, min) + } + } + } + + beforeEach(() => { + LiteGraph.registerNodeType('RemoteDynamicGroup', remoteNodeType(0)) + LiteGraph.registerNodeType('MinOneDynamicGroup', remoteNodeType(1)) + LiteGraph.registerNodeType('ConsentDynamicGroup', remoteNodeType(0)) + }) + + function remoteGroup(type = 'RemoteDynamicGroup') { + const graph = new LGraph() + const { doc, collector } = followedDoc( + { + nodes: [ + { + id: 1, + type, + pos: [0, 0], + size: [210, 100], + widgets_values: [1, 0.5] + } + ], + links: [] + }, + { + types: { + [type]: { + widget_order: ['rows', 'rows.0.strength', 'rows.1.strength'] + } + } + } + ) + const applier = new LiveGraphApplier({ getGraph: () => graph }) + const context = { actor: 'agent:test', opIds: ['op-1'] } + applier.applyChanges(doc, collector.take(), context) + const rows = graph + .getNodeById(toNodeId(1)) + ?.widgets?.find((widget) => widget.name === 'rows') + if (!rows) throw new Error('DynamicGroup node 1 was not created') + function setRows(count: unknown) { + doc.transact(() => { + const widgets = nodesMap(doc).get('1')?.get('widgets') + if (!(widgets instanceof Y.Map)) throw new Error('named storage') + widgets.set('rows', count) + }) + return applier.applyChanges(doc, collector.take(), context).failureCount + } + return { rows, setRows } + } + + it('applies a remote row count without reporting', () => { + const { rows, setRows } = remoteGroup() + + expect([setRows(2), setRows(0)]).toEqual([0, 0]) + expect(rows.value).toBe(0) + expect(reportError).not.toHaveBeenCalled() + }) + + it.for([ + { type: 'RemoteDynamicGroup', value: 2.5, expected: 2 }, + { type: 'RemoteDynamicGroup', value: -0, expected: 0 }, + { type: 'MinOneDynamicGroup', value: 0, expected: 1 } + ])( + 'does not report a remote $value the $type setter normalizes', + ({ type, value, expected }) => { + const { rows, setRows } = remoteGroup(type) + + expect(setRows(value)).toBe(0) + expect(rows.value).toBe(expected) + expect(reportError).not.toHaveBeenCalled() + } + ) + + it('reports a remote row count the live group did not reach, once per node type', () => { + const { rows, setRows } = remoteGroup() + refuseInputChanges() + + expect([setRows(3), setRows(2)]).toEqual([1, 1]) + expect(rows.value).toBe(1) + expect(vi.mocked(reportError).mock.calls).toEqual([ + [ + expect.objectContaining({ + message: 'Remote DynamicGroup row count did not apply' + }), + { + errorType: 'agent_graph_dynamic_group_rows_mismatch', + surface: 'agent', + tags: { + subsystem: 'agent-crdt', + layer: 'graph-api', + outcome: 'degraded', + node_type: 'RemoteDynamicGroup', + operation: 'remote_apply', + in_subgraph: false + }, + context: { requested: 3, actual: 1 } + } + ] + ]) + }) + + it('counts but does not send a mismatch without telemetry consent', () => { + remoteConfig.value = {} + const { setRows } = remoteGroup('ConsentDynamicGroup') + refuseInputChanges() + + expect(setRows(2)).toBe(1) + expect(reportError).not.toHaveBeenCalled() + }) +}) diff --git a/src/core/graph/widgets/dynamicGroupDiagnostics.ts b/src/core/graph/widgets/dynamicGroupDiagnostics.ts new file mode 100644 index 00000000000..d837b9188ab --- /dev/null +++ b/src/core/graph/widgets/dynamicGroupDiagnostics.ts @@ -0,0 +1,117 @@ +import type { LGraphNode } from '@/lib/litegraph/src/LGraphNode' +import type { IBaseWidget } from '@/lib/litegraph/src/types/widgets' +import { isCloud } from '@/platform/distribution/types' +import { isHostTelemetryEnabled } from '@/platform/telemetry/hostTelemetryEnabled' +import { reportError } from '@/platform/telemetry/reportError' +import { isSubgraph } from '@/utils/typeGuardUtil' + +type RowOperation = 'add_row' | 'remove_row' | 'remote_apply' + +const controllers = new WeakSet() +const reported = new Set() +let remoteRowCountNode: LGraphNode | undefined + +export function markDynamicGroupController(widget: IBaseWidget): void { + controllers.add(widget) +} + +/** True for the hidden widget whose value is a DynamicGroup's row count. */ +export function isDynamicGroupController(widget: IBaseWidget): boolean { + return controllers.has(widget) +} + +/** + * Runs a remote row count write to `node`. A refusal on that node inside it is + * reported once, as the remote mismatch, not also as a local row change. + */ +export function applyRemoteDynamicGroupRowCount( + node: LGraphNode, + write: () => void +): void { + const previous = remoteRowCountNode + remoteRowCountNode = node + try { + write() + } finally { + remoteRowCountNode = previous + } +} + +/** Whether a controller holds the row count its setter derives from `value`. */ +export function reachesDynamicGroupRowCount( + widget: IBaseWidget, + value: unknown +): boolean { + if (typeof value !== 'number' || !Number.isFinite(value)) return false + return widget.value === Math.max(widget.options.min ?? 0, Math.trunc(value)) +} + +function reportRowCount( + node: LGraphNode, + operation: RowOperation, + counts: { requested: number | null; actual: number }, + report: { + message: string + errorType: string + surface: 'graph' | 'agent' + level?: 'warning' + tags?: Record + } +): void { + const key = `${report.errorType}:${node.type}:${operation}` + if (!(isCloud || isHostTelemetryEnabled()) || reported.has(key)) return + reported.add(key) + reportError(new Error(report.message), { + errorType: report.errorType, + surface: report.surface, + level: report.level, + tags: { + ...report.tags, + node_type: node.type, + operation, + in_subgraph: isSubgraph(node.graph) + }, + context: counts + }) +} + +/** Reports a local DynamicGroup row change that left a different row count. */ +export function reportDynamicGroupRowRefusal( + node: LGraphNode, + operation: 'add_row' | 'remove_row', + requested: number, + actual: number +): void { + if (remoteRowCountNode === node) return + reportRowCount( + node, + operation, + { requested, actual }, + { + message: 'Refused a DynamicGroup row change', + errorType: 'failure_resizing_dynamic_group_rows', + surface: 'graph', + level: 'warning' + } + ) +} + +/** Reports a remote DynamicGroup row count the live group did not reach. */ +export function reportRemoteDynamicGroupRowMismatch( + node: LGraphNode, + requested: unknown, + actual: number, + tags: Record +): void { + reportRowCount( + node, + 'remote_apply', + { requested: typeof requested === 'number' ? requested : null, actual }, + { + message: 'Remote DynamicGroup row count did not apply', + errorType: 'agent_graph_dynamic_group_rows_mismatch', + surface: 'agent', + tags + } + ) +} diff --git a/src/core/graph/widgets/dynamicGroupWidget.ts b/src/core/graph/widgets/dynamicGroupWidget.ts index 0db92e7e809..358eabf1698 100644 --- a/src/core/graph/widgets/dynamicGroupWidget.ts +++ b/src/core/graph/widgets/dynamicGroupWidget.ts @@ -24,6 +24,11 @@ import { deriveWidgetSurfaces } from '@/types/widgetVisibility' import { mapUniqueNodes } from '@/utils/graphTraversalUtil' import { isSubgraph } from '@/utils/typeGuardUtil' +import { + markDynamicGroupController, + reportDynamicGroupRowRefusal +} from './dynamicGroupDiagnostics' + export function dynamicGroupWidget( node: LGraphNode, inputName: string, @@ -46,6 +51,7 @@ export function dynamicGroupWidget( hidden: true, options: { min, max, socketless: true, serialize: false } }) + markDynamicGroupController(controller) node.addCustomWidget( new DynamicGroupNoticeWidget( { @@ -268,7 +274,14 @@ export function dynamicGroupWidget( callback: () => { if (rows().length <= min) return changeRows(() => { - removeRow(Number(header.name.slice(inputName.length + 1))) + const requested = rows().length - 1 + if (!removeRow(Number(header.name.slice(inputName.length + 1)))) + reportDynamicGroupRowRefusal( + node, + 'remove_row', + requested, + rows().length + ) publish() }) } @@ -400,6 +413,13 @@ export function dynamicGroupWidget( } publish() announceAddedValues(existing) + if (rowCount !== count) + reportDynamicGroupRowRefusal( + node, + rowCount > count ? 'remove_row' : 'add_row', + count, + rowCount + ) } }) controller.value = initialCount diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts index e53803fd765..b80e679501b 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts @@ -8,6 +8,12 @@ import { isEqual } from 'es-toolkit' import * as Y from 'yjs' import { z } from 'zod' +import { + applyRemoteDynamicGroupRowCount, + isDynamicGroupController, + reachesDynamicGroupRowCount, + reportRemoteDynamicGroupRowMismatch +} from '@/core/graph/widgets/dynamicGroupDiagnostics' import { growAutogrowInput } from '@/core/graph/widgets/dynamicWidgets' import type { INodeFlags } from '@/lib/litegraph/src/interfaces' import type { INodeInputSlot } from '@/lib/litegraph/src/types/slots' @@ -918,7 +924,9 @@ export class LiveGraphApplier { this.holdsLocalWrite(node, widget.name, value) ) continue - this.setWidgetValue(node, widget, value) + if (isDynamicGroupController(widget)) + this.applyDynamicGroupRowCount(node, widget, value) + else this.setWidgetValue(node, widget, value) resolved = true } return { resolved, unresolved } @@ -1046,6 +1054,22 @@ export class LiveGraphApplier { ) } + private applyDynamicGroupRowCount( + node: LGraphNode, + widget: IBaseWidget, + value: WidgetValue + ) { + applyRemoteDynamicGroupRowCount(node, () => + this.setWidgetValue(node, widget, value) + ) + if (reachesDynamicGroupRowCount(widget, value)) return + this.incrementFailureCount() + reportRemoteDynamicGroupRowMismatch(node, value, Number(widget.value), { + ...AGENT_APPLY_TAGS, + outcome: 'degraded' + }) + } + private setWidgetValue( node: LGraphNode, widget: IBaseWidget,