From ac0cf05c3db9baf1a948eae36e39babe2c87708f Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Sat, 14 Nov 2026 22:35:18 +0000 Subject: [PATCH 1/8] feat: prototype native widget augmentations Keep core in charge of textarea rendering and edits while extensions contribute targeted completions, hints and text actions. Demonstrate the contract with an ordinary devtools node and imported prompt catalogs. Describe the ownership decision and partial Custom Scripts and Impact Pack migration stories in a proposed ADR. Amp-Thread-ID: https://ampcode.com/threads/T-bca37262-aa37-4d85-ad1d-a8a1d96e4d47 --- .gitignore | 1 + ...57-core-renders-extension-contributions.md | 71 ++++ .../editing.svg | 39 ++ .../registration.svg | 39 ++ knip.config.ts | 2 + .../graph/widgets/DomWidget.test.ts | 34 +- src/components/graph/widgets/DomWidget.vue | 28 +- src/locales/en/main.json | 6 + .../extensions/useWidgetAugmentations.ts | 72 ++++ src/platform/extensions/widgetAugmentation.ts | 23 ++ .../components/TextEditorDialogContent.vue | 76 +++- .../components/TextareaAugmentations.vue | 117 ++++++ .../components/TextareaAutocomplete.vue | 100 ++++++ .../textEditor/promptTagCatalogStore.test.ts | 57 +++ .../textEditor/promptTagCatalogStore.ts | 67 ++++ .../textEditor/textEditorDialog.test.ts | 335 +++++++++++++++++- .../extensions/textEditor/textEditorDialog.ts | 4 + .../textEditor/useTextareaAutocomplete.ts | 205 +++++++++++ .../extensions/textEditor/useTextareaEdit.ts | 61 ++++ .../widgets/components/WidgetTextarea.test.ts | 29 ++ .../widgets/components/WidgetTextarea.vue | 22 ++ src/types/comfy.ts | 2 + tools/devtools/README.md | 41 +++ tools/devtools/build.ts | 18 + tools/devtools/nodes/inputs.py | 20 ++ tools/devtools/widgetAugmentations.ts | 52 +++ 26 files changed, 1515 insertions(+), 6 deletions(-) create mode 100644 docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions.md create mode 100644 docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/editing.svg create mode 100644 docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/registration.svg create mode 100644 src/platform/extensions/useWidgetAugmentations.ts create mode 100644 src/platform/extensions/widgetAugmentation.ts create mode 100644 src/renderer/extensions/textEditor/components/TextareaAugmentations.vue create mode 100644 src/renderer/extensions/textEditor/components/TextareaAutocomplete.vue create mode 100644 src/renderer/extensions/textEditor/promptTagCatalogStore.test.ts create mode 100644 src/renderer/extensions/textEditor/promptTagCatalogStore.ts create mode 100644 src/renderer/extensions/textEditor/useTextareaAutocomplete.ts create mode 100644 src/renderer/extensions/textEditor/useTextareaEdit.ts create mode 100644 tools/devtools/build.ts create mode 100644 tools/devtools/widgetAugmentations.ts diff --git a/.gitignore b/.gitignore index 48e07a8fc9a..1df000ce846 100644 --- a/.gitignore +++ b/.gitignore @@ -100,6 +100,7 @@ storybook-static # MCP Servers .playwright-mcp/* +/tools/devtools/web/widgetAugmentations.js vite.config.*.timestamp* vitest.config.*.timestamp* diff --git a/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions.md b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions.md new file mode 100644 index 00000000000..9577d9ca81d --- /dev/null +++ b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions.md @@ -0,0 +1,71 @@ +# ADR-WIDGET-AUGMENTATION-0057: Core renders extension contributions + +Date: 2026-11-14 + +## Status + +Proposed + +## Context + +A custom-node author wants to help someone write a prompt, not maintain another +editor. Today that help can require replacing widget factories, attaching DOM +listeners, or assigning widget values. Those integrations inherit each +renderer's selection, focus, undo, and teardown problems. Expanding the same +prompt into a dialog can leave the extension's behavior behind. + +## Decision + +Let extensions register declarative contributions against named node widgets. +Core resolves active contributions and renders their controls beside the +existing input. Start with completion suggestions, live text hints, and +synchronous text transformations. The devtools example tells this story with +lighting suggestions, a tag count, and a Normalize tags action on one ordinary +multiline widget, leaving its neighboring notes field alone. + +Registration supplies behavior without handing the extension an input element. + +![Core resolves and renders targeted contributions](./WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/registration.svg) + +An accepted suggestion or transformation goes through core's guarded native +editing operation, then the input's existing value pipeline. There is no second +widget-value writer. The node editor and expanded editor share contributions, +not selection or browser undo history. Refusing an edit leaves the value intact. + +![Guarded insertion rejoins the existing widget update path](./WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/editing.svg) + +## Migration stories + +**Custom Scripts could keep its dictionaries and stop owning the input.** Its +[autocomplete integration](https://github.com/pythongosssss/ComfyUI-Custom-Scripts/blob/609f3afaa74b2f88ef9ce8d939626065e3247469/web/js/autocompleter.js) +replaces the string-widget factory and attaches an autocomplete engine to each +eligible textarea. Already-loaded words could supply completion providers for +explicit widget targets. This migrates cached lookup, not its global widget +discovery, rich dropdown, or custom escaping and insertion rules. + +**Impact Pack could offer prompt help without mutating prompt widgets.** Its +[wildcard integration](https://github.com/ltdrdata/ComfyUI-Impact-Pack/blob/9e2d41da75c185bf7638f860d5d004035f378e9b/js/impact-pack.js) +appends wildcard and LoRA tokens through combo callbacks and writes guidance +directly to textareas. Cached wildcard tokens could supply completions for +`ImpactWildcardProcessor.wildcard_text` and +`ImpactWildcardEncode.wildcard_text`; static guidance could become hints. +Completion would be a different discovery UI, not a drop-in combo replacement. +Population, serialization overrides, and mode-dependent editability stay +outside this API. These are source-based migration candidates, not tested +migrations or promises of full compatibility. + +## Tradeoffs and limits + +Keep native textareas instead of replacing prompt editing with a rich editor. +Reuse existing positioning and design-system controls instead of adding an +autocomplete library. A general mount callback would admit arbitrary UI, but +would also require a public focus, keyboard, accessibility, and edit-lifecycle +contract. Defer that until a concrete contribution cannot fit this model. + +The prototype covers multiline native textareas, not every widget or the +agent's structured ProseMirror editor. Exact targets and synchronous callbacks +deliberately limit discovery and remote suggestions. Native insertion still +depends on deprecated `execCommand`. Core skips unsupported insertion to +preserve undo. Cross-browser undo and automatic loading of the +built devtools extension remain release gates. Existing extension behavior is +not removed or intercepted by this proposal. diff --git a/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/editing.svg b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/editing.svg new file mode 100644 index 00000000000..f72bf63b3ac --- /dev/null +++ b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/editing.svg @@ -0,0 +1,39 @@ + + An augmentation edits through the existing input pipeline + A selected completion or text action requests replacement text. Core checks editability, composition, current text and selection, and length. Accepted native insertion emits input into the existing widget update path. Refused edits leave the value unchanged. + + + + + + + The extension proposes text. Core decides whether to edit. + + Choose a completion + or run a text action + + Guard the replacement + useTextareaEdit.replace + + Native text insertion + document.execCommand + + Existing widget update + input / update:model-value + + + + + + text + valid + input + + refused + + Keep the current value + Read-only, disabled, composing, + stale text or selection, or maxlength. + Native undo stays with the focused editor. No second widget-value writer or direct-value fallback. + + diff --git a/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/registration.svg b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/registration.svg new file mode 100644 index 00000000000..0493599733a --- /dev/null +++ b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/registration.svg @@ -0,0 +1,39 @@ + + Extensions supply behavior and core renders it + An extension registers targeted contributions. Core resolves enabled contributions for a widget and renders hints, text actions, and completions without giving the extension its input element. + + + + + + Extensions supply behavior. Core owns the widget. + + Register contributions + app.registerExtension + + Match enabled targets + useWidgetAugmentations + + Render core controls + TextareaAugmentations + + Live hint + text(value) + + Text action button + transform(value) + + Autocomplete list + suggest(query) + + + + + + + + target + resolve + Disable or unregister an extension: its contributions disappear, while the existing widget remains. + + diff --git a/knip.config.ts b/knip.config.ts index a94760a5bc5..c8a21288e78 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -12,6 +12,8 @@ const config: KnipConfig = { 'src/scripts/ui/menu/index.ts', 'src/types/index.ts', 'src/storybook/mocks/**/*.ts', + 'tools/devtools/build.ts', + 'tools/devtools/widgetAugmentations.ts', 'tools/oxlint-plugins/comfyIngestTypes.ts', 'tools/oxlint-plugins/serverFacts.ts', 'tools/oxlint-plugins/vitestCleanup.ts' diff --git a/src/components/graph/widgets/DomWidget.test.ts b/src/components/graph/widgets/DomWidget.test.ts index 7aa815865a2..171164ec5a6 100644 --- a/src/components/graph/widgets/DomWidget.test.ts +++ b/src/components/graph/widgets/DomWidget.test.ts @@ -1,11 +1,14 @@ -import { render } from '@testing-library/vue' +import { render, screen } from '@testing-library/vue' +import userEvent from '@testing-library/user-event' import { fromPartial } from '@total-typescript/shoehorn' import { beforeEach, afterEach, describe, expect, it, vi } from 'vitest' import { nextTick, reactive, ref } from 'vue' +import { i18n } from '@/i18n' import { LGraphNode } from '@/lib/litegraph/src/litegraph' import { useSettingStore } from '@/platform/settings/settingStore' import { useCanvasStore } from '@/renderer/core/canvas/canvasStore' +import { usePromptTagCatalogStore } from '@/renderer/extensions/textEditor/promptTagCatalogStore' import type { BaseDOMWidget } from '@/types/domWidget' import type { DomWidgetState } from '@/stores/domWidgetStore' import { useDomWidgetStore } from '@/stores/domWidgetStore' @@ -273,6 +276,35 @@ describe('native DOM widget interaction lifecycle', () => { useDomWidgetStore().clear() }) + it('offers shared catalog suggestions for a native multiline widget and restores its attributes on removal', async () => { + const user = userEvent.setup() + usePromptTagCatalogStore().importCatalog('blue_eyes') + const widgetState = createWidgetState(false) + const textarea = document.createElement('textarea') + textarea.setAttribute('aria-label', 'Prompt') + textarea.setAttribute('aria-autocomplete', 'none') + Object.assign(widgetState.widget, { element: textarea, type: 'customtext' }) + const view = render(DomWidget, { + props: { widgetState }, + global: { plugins: [i18n] } + }) + await nextTick() + const editor = await screen.findByRole('textbox', { name: 'Prompt' }) + + await user.type(editor, 'blue') + + expect( + await screen.findByRole('option', { name: 'blue eyes' }) + ).toBeVisible() + expect(editor).toHaveFocus() + expect(editor).toHaveAttribute('aria-autocomplete', 'list') + + view.unmount() + + expect(screen.queryByRole('listbox')).toBeNull() + expect(textarea).toHaveAttribute('aria-autocomplete', 'none') + }) + function renderWithUnmountableElement(widgetState: DomWidgetState) { const input = document.createElement('input') Object.assign(widgetState.widget, { element: input }) diff --git a/src/components/graph/widgets/DomWidget.vue b/src/components/graph/widgets/DomWidget.vue index 2d0e92ea14d..e97b05f2c1e 100644 --- a/src/components/graph/widgets/DomWidget.vue +++ b/src/components/graph/widgets/DomWidget.vue @@ -3,6 +3,7 @@ v-show="widgetState.visible" ref="widgetElement" class="dom-widget size-full" + :class="cn(widget.type === 'customtext' && 'flex flex-col')" :title="tooltip" :style="style" > @@ -15,6 +16,16 @@ v-bind="widget.props" @update:model-value="emit('update:widgetValue', $event)" /> + @@ -23,11 +34,13 @@ import { useElementBounding, useEventListener, whenever } from '@vueuse/core' import type { CSSProperties } from 'vue' import { computed, nextTick, onMounted, ref, watch } from 'vue' +import { cn } from '@comfyorg/tailwind-utils' import { useAbsolutePosition } from '@/composables/element/useAbsolutePosition' import { useDomClipping } from '@/composables/element/useDomClipping' import { findFirstNode } from '@/lib/litegraph/src/utils/collections' import { useSettingStore } from '@/platform/settings/settingStore' import { useCanvasStore } from '@/renderer/core/canvas/canvasStore' +import TextareaAugmentations from '@/renderer/extensions/textEditor/components/TextareaAugmentations.vue' import { isComponentWidget, isDOMWidget } from '@/scripts/domWidget' import type { DomWidgetState } from '@/stores/domWidgetStore' @@ -38,6 +51,13 @@ const { widgetState } = defineProps<{ }>() const widget = widgetState.widget +function textareaElement(): HTMLTextAreaElement | undefined { + if (!isDOMWidget(widget)) return undefined + return widget.element instanceof HTMLTextAreaElement + ? widget.element + : undefined +} + const emit = defineEmits<{ 'update:widgetValue': [value: string | object] }>() @@ -169,18 +189,20 @@ onMounted(() => { const inputSpec = widget.node.constructor.nodeData const tooltip = inputSpec?.inputs?.[widget.name]?.tooltip -// Mount DOM element when widget is or becomes visible const mountElementIfVisible = () => { if (!(widgetState.visible && isDOMWidget(widget) && widgetElement.value)) { return } - // Only append if not already a child if (widgetElement.value.contains(widget.element)) { return } try { - widget.element.classList.add('h-full', 'w-full') + widget.element.classList.add( + 'h-full', + 'w-full', + ...(widget.type === 'customtext' ? ['min-h-0', 'flex-1'] : []) + ) widgetElement.value.appendChild(widget.element) } catch (error) { reportDomWidgetMountFailure(error, { diff --git a/src/locales/en/main.json b/src/locales/en/main.json index 19316d247a5..1d5fb7442c2 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -4416,6 +4416,12 @@ "textEditor": { "characterCount": "{count} char | {count} chars", "expand": "Expand", + "importTags": "Import prompt tags", + "importTagsError": "Could not load tags. Choose a TXT list or a tag CSV (name, category, count, aliases), up to 10 MB. Your previous catalog is unchanged.", + "importTagsHint": "CSV or TXT · local to this session", + "suggestionCount": "{count} suggestion available | {count} suggestions available", + "suggestions": "Prompt suggestions", + "tagsLoaded": "{count} tag loaded for this session | {count} tags loaded for this session", "title": "{widget} · {node}" }, "widgetFileUpload": { diff --git a/src/platform/extensions/useWidgetAugmentations.ts b/src/platform/extensions/useWidgetAugmentations.ts new file mode 100644 index 00000000000..a5ecabb3ccf --- /dev/null +++ b/src/platform/extensions/useWidgetAugmentations.ts @@ -0,0 +1,72 @@ +import { computed, shallowRef } from 'vue' + +import { reportError } from '@comfyorg/telemetry/reportError' +import { useExtensionStore } from '@/platform/extensions/stores/extensionStore' +import type { + WidgetAugmentation, + WidgetAugmentationTarget +} from '@/platform/extensions/widgetAugmentation' + +interface Contribution { + key: string + extension: string + augmentation: WidgetAugmentation +} + +export function useWidgetAugmentations( + target: () => WidgetAugmentationTarget | undefined +) { + const extensions = useExtensionStore() + const failed = shallowRef(new Set()) + const nodeType = computed(() => target()?.nodeType) + const widgetName = computed(() => target()?.widgetName) + const registrations = computed(() => + extensions.extensions + .filter((extension) => extensions.contributes(extension.name)) + .flatMap((extension) => + (extension.widgetAugmentations ?? []) + .filter( + (augmentation) => + augmentation.target.nodeType === nodeType.value && + augmentation.target.widgetName === widgetName.value + ) + .map((augmentation) => ({ + key: `${extension.name}:${augmentation.id}`, + extension: extension.name, + augmentation + })) + ) + ) + const contributions = computed(() => + registrations.value.filter((item) => !failed.value.has(item.augmentation)) + ) + + function evaluate( + contribution: Contribution, + read: () => T + ): T | undefined { + if ( + !contributions.value.some( + (item) => + item.extension === contribution.extension && + item.augmentation === contribution.augmentation + ) + ) + return undefined + try { + return read() + } catch (error) { + failed.value = new Set([...failed.value, contribution.augmentation]) + reportError(error, { + errorType: 'error_evaluating_widget_augmentation', + surface: 'platform', + context: { + extension: contribution.extension, + augmentation: contribution.augmentation.id + } + }) + } + } + + return { registrations, contributions, evaluate } +} diff --git a/src/platform/extensions/widgetAugmentation.ts b/src/platform/extensions/widgetAugmentation.ts new file mode 100644 index 00000000000..cf74752b630 --- /dev/null +++ b/src/platform/extensions/widgetAugmentation.ts @@ -0,0 +1,23 @@ +export interface WidgetAugmentationTarget { + nodeType: string + widgetName: string +} + +export type WidgetAugmentation = { + id: string + target: WidgetAugmentationTarget +} & ( + | { + kind: 'text-completion' + suggest: (query: string) => readonly string[] + } + | { + kind: 'hint' + text: (value: string) => string + } + | { + kind: 'text-action' + label: string + transform: (value: string) => string + } +) diff --git a/src/renderer/extensions/textEditor/components/TextEditorDialogContent.vue b/src/renderer/extensions/textEditor/components/TextEditorDialogContent.vue index 66214fb6cf5..c696aa7835f 100644 --- a/src/renderer/extensions/textEditor/components/TextEditorDialogContent.vue +++ b/src/renderer/extensions/textEditor/components/TextEditorDialogContent.vue @@ -1,4 +1,39 @@