diff --git a/.gitignore b/.gitignore index 48e07a8fc9a..1df000ce846 100644 --- a/.gitignore +++ b/.gitignore @@ -100,6 +100,7 @@ storybook-static # MCP Servers .playwright-mcp/* +/tools/devtools/web/widgetAugmentations.js vite.config.*.timestamp* vitest.config.*.timestamp* diff --git a/browser_tests/tests/vueNodes/widgets/text/textEditorDialog.spec.ts b/browser_tests/tests/vueNodes/widgets/text/textEditorDialog.spec.ts index 7e368b4b719..aad818034a5 100644 --- a/browser_tests/tests/vueNodes/widgets/text/textEditorDialog.spec.ts +++ b/browser_tests/tests/vueNodes/widgets/text/textEditorDialog.spec.ts @@ -73,6 +73,59 @@ test.describe( } ) + test('accepts an imported prompt tag with native undo and redo', async ({ + comfyPage + }) => { + const node = comfyPage.vueNodes + .getNodeByTitle('CLIP Text Encode (Prompt)') + .first() + const widget = node.getByRole('textbox', { name: 'text' }) + const dialog = new TextEditorDialog(comfyPage.page) + + await test.step('import a catalog in the expanded prompt editor', async () => { + await widget.fill('') + await widget.hover() + await node + .getByRole('group', { name: 'Actions' }) + .getByRole('button', { name: 'Expand' }) + .click() + await dialog.root.getByLabel('Import prompt tags').setInputFiles({ + name: 'tags.txt', + mimeType: 'text/plain', + buffer: Buffer.from('blue_eyes,20\nblue_hair,10') + }) + await expect( + dialog.root.getByText('2 tags loaded for this session', { + exact: true + }) + ).toBeVisible() + }) + + await test.step('navigate and accept without leaving the textarea', async () => { + await dialog.editor.pressSequentially('blue') + await expect( + dialog.root.getByRole('option', { selected: true }) + ).toHaveText('blue eyes') + await dialog.editor.press('ArrowDown') + await expect( + dialog.root.getByRole('option', { selected: true }) + ).toHaveText('blue hair') + await dialog.editor.press('Enter') + await expect(dialog.editor).toHaveValue('blue hair') + await expect(dialog.editor).toBeFocused() + await expect(dialog.root.getByRole('listbox')).toBeHidden() + }) + + await test.step('undo and redo the insertion through the native editor', async () => { + await dialog.editor.press('ControlOrMeta+z') + await expect(dialog.editor).toHaveValue('blue') + await dialog.editor.press('ControlOrMeta+Shift+z') + await expect(dialog.editor).toHaveValue('blue hair') + await dialog.close() + await expect(widget).toHaveValue('blue hair') + }) + }) + test('opens long text scrolled to the top', async ({ comfyPage }) => { const node = comfyPage.vueNodes .getNodeByTitle('CLIP Text Encode (Prompt)') diff --git a/browser_tests/tests/vueNodes/widgets/text/textEditorDialog.spec.ts-snapshots/text-editor-dialog-chromium-linux.png b/browser_tests/tests/vueNodes/widgets/text/textEditorDialog.spec.ts-snapshots/text-editor-dialog-chromium-linux.png index 4561ad307d6..5ace0275366 100644 Binary files a/browser_tests/tests/vueNodes/widgets/text/textEditorDialog.spec.ts-snapshots/text-editor-dialog-chromium-linux.png and b/browser_tests/tests/vueNodes/widgets/text/textEditorDialog.spec.ts-snapshots/text-editor-dialog-chromium-linux.png differ diff --git a/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions.md b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions.md new file mode 100644 index 00000000000..9577d9ca81d --- /dev/null +++ b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions.md @@ -0,0 +1,71 @@ +# ADR-WIDGET-AUGMENTATION-0057: Core renders extension contributions + +Date: 2026-11-14 + +## Status + +Proposed + +## Context + +A custom-node author wants to help someone write a prompt, not maintain another +editor. Today that help can require replacing widget factories, attaching DOM +listeners, or assigning widget values. Those integrations inherit each +renderer's selection, focus, undo, and teardown problems. Expanding the same +prompt into a dialog can leave the extension's behavior behind. + +## Decision + +Let extensions register declarative contributions against named node widgets. +Core resolves active contributions and renders their controls beside the +existing input. Start with completion suggestions, live text hints, and +synchronous text transformations. The devtools example tells this story with +lighting suggestions, a tag count, and a Normalize tags action on one ordinary +multiline widget, leaving its neighboring notes field alone. + +Registration supplies behavior without handing the extension an input element. + + + +An accepted suggestion or transformation goes through core's guarded native +editing operation, then the input's existing value pipeline. There is no second +widget-value writer. The node editor and expanded editor share contributions, +not selection or browser undo history. Refusing an edit leaves the value intact. + + + +## Migration stories + +**Custom Scripts could keep its dictionaries and stop owning the input.** Its +[autocomplete integration](https://github.com/pythongosssss/ComfyUI-Custom-Scripts/blob/609f3afaa74b2f88ef9ce8d939626065e3247469/web/js/autocompleter.js) +replaces the string-widget factory and attaches an autocomplete engine to each +eligible textarea. Already-loaded words could supply completion providers for +explicit widget targets. This migrates cached lookup, not its global widget +discovery, rich dropdown, or custom escaping and insertion rules. + +**Impact Pack could offer prompt help without mutating prompt widgets.** Its +[wildcard integration](https://github.com/ltdrdata/ComfyUI-Impact-Pack/blob/9e2d41da75c185bf7638f860d5d004035f378e9b/js/impact-pack.js) +appends wildcard and LoRA tokens through combo callbacks and writes guidance +directly to textareas. Cached wildcard tokens could supply completions for +`ImpactWildcardProcessor.wildcard_text` and +`ImpactWildcardEncode.wildcard_text`; static guidance could become hints. +Completion would be a different discovery UI, not a drop-in combo replacement. +Population, serialization overrides, and mode-dependent editability stay +outside this API. These are source-based migration candidates, not tested +migrations or promises of full compatibility. + +## Tradeoffs and limits + +Keep native textareas instead of replacing prompt editing with a rich editor. +Reuse existing positioning and design-system controls instead of adding an +autocomplete library. A general mount callback would admit arbitrary UI, but +would also require a public focus, keyboard, accessibility, and edit-lifecycle +contract. Defer that until a concrete contribution cannot fit this model. + +The prototype covers multiline native textareas, not every widget or the +agent's structured ProseMirror editor. Exact targets and synchronous callbacks +deliberately limit discovery and remote suggestions. Native insertion still +depends on deprecated `execCommand`. Core skips unsupported insertion to +preserve undo. Cross-browser undo and automatic loading of the +built devtools extension remain release gates. Existing extension behavior is +not removed or intercepted by this proposal. diff --git a/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/editing.svg b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/editing.svg new file mode 100644 index 00000000000..f72bf63b3ac --- /dev/null +++ b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/editing.svg @@ -0,0 +1,39 @@ + + An augmentation edits through the existing input pipeline + A selected completion or text action requests replacement text. Core checks editability, composition, current text and selection, and length. Accepted native insertion emits input into the existing widget update path. Refused edits leave the value unchanged. + + + + + + + The extension proposes text. Core decides whether to edit. + + Choose a completion + or run a text action + + Guard the replacement + useTextareaEdit.replace + + Native text insertion + document.execCommand + + Existing widget update + input / update:model-value + + + + + + text + valid + input + + refused + + Keep the current value + Read-only, disabled, composing, + stale text or selection, or maxlength. + Native undo stays with the focused editor. No second widget-value writer or direct-value fallback. + + diff --git a/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/registration.svg b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/registration.svg new file mode 100644 index 00000000000..0493599733a --- /dev/null +++ b/docs/adr/WIDGET-AUGMENTATION-0057-core-renders-extension-contributions/registration.svg @@ -0,0 +1,39 @@ + + Extensions supply behavior and core renders it + An extension registers targeted contributions. Core resolves enabled contributions for a widget and renders hints, text actions, and completions without giving the extension its input element. + + + + + + Extensions supply behavior. Core owns the widget. + + Register contributions + app.registerExtension + + Match enabled targets + useWidgetAugmentations + + Render core controls + TextareaAugmentations + + Live hint + text(value) + + Text action button + transform(value) + + Autocomplete list + suggest(query) + + + + + + + + target + resolve + Disable or unregister an extension: its contributions disappear, while the existing widget remains. + + diff --git a/knip.config.ts b/knip.config.ts index a94760a5bc5..c8a21288e78 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -12,6 +12,8 @@ const config: KnipConfig = { 'src/scripts/ui/menu/index.ts', 'src/types/index.ts', 'src/storybook/mocks/**/*.ts', + 'tools/devtools/build.ts', + 'tools/devtools/widgetAugmentations.ts', 'tools/oxlint-plugins/comfyIngestTypes.ts', 'tools/oxlint-plugins/serverFacts.ts', 'tools/oxlint-plugins/vitestCleanup.ts' diff --git a/src/components/graph/widgets/DomWidget.test.ts b/src/components/graph/widgets/DomWidget.test.ts index 7aa815865a2..171164ec5a6 100644 --- a/src/components/graph/widgets/DomWidget.test.ts +++ b/src/components/graph/widgets/DomWidget.test.ts @@ -1,11 +1,14 @@ -import { render } from '@testing-library/vue' +import { render, screen } from '@testing-library/vue' +import userEvent from '@testing-library/user-event' import { fromPartial } from '@total-typescript/shoehorn' import { beforeEach, afterEach, describe, expect, it, vi } from 'vitest' import { nextTick, reactive, ref } from 'vue' +import { i18n } from '@/i18n' import { LGraphNode } from '@/lib/litegraph/src/litegraph' import { useSettingStore } from '@/platform/settings/settingStore' import { useCanvasStore } from '@/renderer/core/canvas/canvasStore' +import { usePromptTagCatalogStore } from '@/renderer/extensions/textEditor/promptTagCatalogStore' import type { BaseDOMWidget } from '@/types/domWidget' import type { DomWidgetState } from '@/stores/domWidgetStore' import { useDomWidgetStore } from '@/stores/domWidgetStore' @@ -273,6 +276,35 @@ describe('native DOM widget interaction lifecycle', () => { useDomWidgetStore().clear() }) + it('offers shared catalog suggestions for a native multiline widget and restores its attributes on removal', async () => { + const user = userEvent.setup() + usePromptTagCatalogStore().importCatalog('blue_eyes') + const widgetState = createWidgetState(false) + const textarea = document.createElement('textarea') + textarea.setAttribute('aria-label', 'Prompt') + textarea.setAttribute('aria-autocomplete', 'none') + Object.assign(widgetState.widget, { element: textarea, type: 'customtext' }) + const view = render(DomWidget, { + props: { widgetState }, + global: { plugins: [i18n] } + }) + await nextTick() + const editor = await screen.findByRole('textbox', { name: 'Prompt' }) + + await user.type(editor, 'blue') + + expect( + await screen.findByRole('option', { name: 'blue eyes' }) + ).toBeVisible() + expect(editor).toHaveFocus() + expect(editor).toHaveAttribute('aria-autocomplete', 'list') + + view.unmount() + + expect(screen.queryByRole('listbox')).toBeNull() + expect(textarea).toHaveAttribute('aria-autocomplete', 'none') + }) + function renderWithUnmountableElement(widgetState: DomWidgetState) { const input = document.createElement('input') Object.assign(widgetState.widget, { element: input }) diff --git a/src/components/graph/widgets/DomWidget.vue b/src/components/graph/widgets/DomWidget.vue index 2d0e92ea14d..e97b05f2c1e 100644 --- a/src/components/graph/widgets/DomWidget.vue +++ b/src/components/graph/widgets/DomWidget.vue @@ -3,6 +3,7 @@ v-show="widgetState.visible" ref="widgetElement" class="dom-widget size-full" + :class="cn(widget.type === 'customtext' && 'flex flex-col')" :title="tooltip" :style="style" > @@ -15,6 +16,16 @@ v-bind="widget.props" @update:model-value="emit('update:widgetValue', $event)" /> + @@ -23,11 +34,13 @@ import { useElementBounding, useEventListener, whenever } from '@vueuse/core' import type { CSSProperties } from 'vue' import { computed, nextTick, onMounted, ref, watch } from 'vue' +import { cn } from '@comfyorg/tailwind-utils' import { useAbsolutePosition } from '@/composables/element/useAbsolutePosition' import { useDomClipping } from '@/composables/element/useDomClipping' import { findFirstNode } from '@/lib/litegraph/src/utils/collections' import { useSettingStore } from '@/platform/settings/settingStore' import { useCanvasStore } from '@/renderer/core/canvas/canvasStore' +import TextareaAugmentations from '@/renderer/extensions/textEditor/components/TextareaAugmentations.vue' import { isComponentWidget, isDOMWidget } from '@/scripts/domWidget' import type { DomWidgetState } from '@/stores/domWidgetStore' @@ -38,6 +51,13 @@ const { widgetState } = defineProps<{ }>() const widget = widgetState.widget +function textareaElement(): HTMLTextAreaElement | undefined { + if (!isDOMWidget(widget)) return undefined + return widget.element instanceof HTMLTextAreaElement + ? widget.element + : undefined +} + const emit = defineEmits<{ 'update:widgetValue': [value: string | object] }>() @@ -169,18 +189,20 @@ onMounted(() => { const inputSpec = widget.node.constructor.nodeData const tooltip = inputSpec?.inputs?.[widget.name]?.tooltip -// Mount DOM element when widget is or becomes visible const mountElementIfVisible = () => { if (!(widgetState.visible && isDOMWidget(widget) && widgetElement.value)) { return } - // Only append if not already a child if (widgetElement.value.contains(widget.element)) { return } try { - widget.element.classList.add('h-full', 'w-full') + widget.element.classList.add( + 'h-full', + 'w-full', + ...(widget.type === 'customtext' ? ['min-h-0', 'flex-1'] : []) + ) widgetElement.value.appendChild(widget.element) } catch (error) { reportDomWidgetMountFailure(error, { diff --git a/src/locales/en/main.json b/src/locales/en/main.json index 19316d247a5..6a17cbc6413 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -4414,8 +4414,16 @@ } }, "textEditor": { + "actionTooLong": "This action exceeds the text limit. Shorten the text and try again.", + "actionUnavailable": "This action could not edit the text. Your text is unchanged. Edit it manually instead.", "characterCount": "{count} char | {count} chars", "expand": "Expand", + "importTags": "Import prompt tags", + "importTagsError": "Could not load tags. Choose a TXT list or a tag CSV (name, category, count, aliases), up to 10 MB. Your previous catalog is unchanged.", + "importTagsHint": "CSV or TXT · local to this session", + "suggestionCount": "{count} suggestion available | {count} suggestions available", + "suggestions": "Prompt suggestions", + "tagsLoaded": "{count} tag loaded for this session | {count} tags loaded for this session", "title": "{widget} · {node}" }, "widgetFileUpload": { diff --git a/src/platform/extensions/stores/extensionStore.test.ts b/src/platform/extensions/stores/extensionStore.test.ts index 34f81a8ddcf..27b9e9c557a 100644 --- a/src/platform/extensions/stores/extensionStore.test.ts +++ b/src/platform/extensions/stores/extensionStore.test.ts @@ -3,6 +3,13 @@ import { describe, expect, it, vi } from 'vitest' import { useSettingStore } from '@/platform/settings/settingStore' import { api } from '@/scripts/api' import { useExtensionStore } from '@/platform/extensions/stores/extensionStore' +import { useWidgetAugmentations } from '@/platform/extensions/useWidgetAugmentations' +import { reportError } from '@comfyorg/telemetry/reportError' +import type { ComfyExtension } from '@/types/comfy' + +vi.mock(import('@comfyorg/telemetry/reportError'), () => ({ + reportError: vi.fn() +})) describe('extensionStore', () => { describe('registerExtension', () => { @@ -12,6 +19,68 @@ describe('extensionStore', () => { expect(store.isExtensionInstalled('test.ext')).toBe(true) }) + it('registers immutable extensions with valid augmentations', () => { + const store = useExtensionStore() + const extension = Object.freeze({ + name: 'test.immutable', + widgetAugmentations: [ + { + id: 'hint', + kind: 'hint', + target: { nodeType: 'TestNode', widgetName: 'prompt' }, + text: () => 'Prompt hint' + } + ] + } satisfies ComfyExtension) + + store.registerExtension(extension) + + expect(store.extensions[0]).toBe(extension) + }) + + it('does not restore removed augmentations when registering the same object again', () => { + const store = useExtensionStore() + const target = { nodeType: 'TestNode', widgetName: 'prompt' } + const extension: ComfyExtension = { + name: 'test.reregister', + widgetAugmentations: [ + { id: 'hint', kind: 'hint', target, text: () => 'Prompt hint' } + ] + } + const { contributions } = useWidgetAugmentations(() => target) + store.registerExtension(extension) + expect(contributions.value.map((item) => item.key)).toEqual([ + 'test.reregister:hint' + ]) + + store.unregisterExtension(extension.name) + delete extension.widgetAugmentations + store.registerExtension(extension) + + expect(contributions.value).toEqual([]) + }) + + it('retains prototype hooks when excluding malformed augmentations', () => { + class Extension implements ComfyExtension { + [key: string]: unknown + name = 'test.prototype' + widgetAugmentations = [] + initialized = false + + setup() { + this.initialized = true + } + } + const store = useExtensionStore() + const extension = new Extension() + Reflect.set(extension, 'widgetAugmentations', [null]) + + store.registerExtension(extension) + + expect(store.extensions[0]).toBe(extension) + expect(store.extensions[0].setup).toBe(extension.setup) + }) + it('throws for extension without name', () => { const store = useExtensionStore() expect(() => store.registerExtension({ name: '' })).toThrow( @@ -27,6 +96,41 @@ describe('extensionStore', () => { ) }) + it.for([ + null, + { id: 'missing-target', kind: 'hint', text: () => 'broken' }, + { id: 'missing-callback', kind: 'hint', target: {} } + ])('isolates malformed JavaScript augmentations: %j', (invalid) => { + const store = useExtensionStore() + const setup = vi.fn() + const extension: ComfyExtension = { name: 'test.invalid', setup } + const valid = { + id: 'valid', + kind: 'hint', + target: { nodeType: 'TestNode', widgetName: 'prompt' }, + text: (value: string) => `Length: ${value.length}` + } + Reflect.set(extension, 'widgetAugmentations', [invalid, valid]) + Object.freeze(extension) + + store.registerExtension(extension) + store.registerExtension({ name: 'test.neighbor' }) + + expect( + store.getWidgetAugmentations(store.extensions[0]).map((item) => item.id) + ).toEqual(['valid']) + expect(extension.widgetAugmentations).toEqual([invalid, valid]) + expect(store.extensions[0].setup).toBe(setup) + expect(store.isExtensionInstalled('test.neighbor')).toBe(true) + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'error_registering_widget_augmentation', + context: { extension: 'test.invalid' } + }) + ) + }) + it('warns when registering a disabled extension but still installs it', () => { const store = useExtensionStore() try { diff --git a/src/platform/extensions/stores/extensionStore.ts b/src/platform/extensions/stores/extensionStore.ts index 28f63e706c5..5ca04593e45 100644 --- a/src/platform/extensions/stores/extensionStore.ts +++ b/src/platform/extensions/stores/extensionStore.ts @@ -1,6 +1,8 @@ import { defineStore } from 'pinia' import { computed, markRaw, ref, shallowRef } from 'vue' +import { isWidgetAugmentation } from '@/platform/extensions/widgetAugmentation' +import type { WidgetAugmentation } from '@/platform/extensions/widgetAugmentation' import { useSettingStore } from '@/platform/settings/settingStore' import { reportError } from '@comfyorg/telemetry/reportError' import type { ComfyExtension } from '@/types/comfy' @@ -38,6 +40,10 @@ interface ExtensionPathState { export const useExtensionStore = defineStore('extension', () => { // For legacy reasons, the name uniquely identifies an extension const extensionByName = ref>({}) + const widgetAugmentations = new WeakMap< + ComfyExtension, + readonly WidgetAugmentation[] + >() const extensions = computed(() => Object.values(extensionByName.value)) // Not using computed because disable extension requires reloading of the page. // Dynamically update this list won't affect extensions that are already loaded. @@ -91,9 +97,27 @@ export const useExtensionStore = defineStore('extension', () => { console.warn(`Extension ${extension.name} is disabled.`) } + if (extension.widgetAugmentations !== undefined) { + const supplied: unknown = extension.widgetAugmentations + const valid = Array.isArray(supplied) + ? supplied.filter(isWidgetAugmentation) + : [] + if (!Array.isArray(supplied) || valid.length !== supplied.length) { + reportError(new TypeError('Invalid widget augmentation registration'), { + errorType: 'error_registering_widget_augmentation', + surface: 'platform', + context: { extension: extension.name } + }) + } + widgetAugmentations.set(extension, valid) + } extensionByName.value[extension.name] = markRaw(extension) } + function getWidgetAugmentations(extension: ComfyExtension) { + return widgetAugmentations.get(extension) ?? [] + } + const registration = shallowRef({ mode: 'immediate' }) const isHoldingCustomNodes = computed( () => registration.value.mode === 'holding' @@ -166,6 +190,8 @@ export const useExtensionStore = defineStore('extension', () => { } function unregisterExtension(name: string) { + if (isExtensionInstalled(name)) + widgetAugmentations.delete(extensionByName.value[name]) delete extensionByName.value[name] } @@ -281,6 +307,7 @@ export const useExtensionStore = defineStore('extension', () => { isExtensionEnabled, isExtensionReadOnly, registerExtension, + getWidgetAugmentations, unregisterExtension, isHoldingCustomNodes, areLateCustomNodesPending, diff --git a/src/platform/extensions/useWidgetAugmentations.ts b/src/platform/extensions/useWidgetAugmentations.ts new file mode 100644 index 00000000000..66b997a19f5 --- /dev/null +++ b/src/platform/extensions/useWidgetAugmentations.ts @@ -0,0 +1,73 @@ +import { computed, shallowRef } from 'vue' + +import { reportError } from '@comfyorg/telemetry/reportError' +import { useExtensionStore } from '@/platform/extensions/stores/extensionStore' +import type { + WidgetAugmentation, + WidgetAugmentationTarget +} from '@/platform/extensions/widgetAugmentation' + +interface Contribution { + key: string + extension: string + augmentation: WidgetAugmentation +} + +export function useWidgetAugmentations( + target: () => WidgetAugmentationTarget | undefined +) { + const extensions = useExtensionStore() + const failed = shallowRef(new Set()) + const nodeType = computed(() => target()?.nodeType) + const widgetName = computed(() => target()?.widgetName) + const registrations = computed(() => + extensions.extensions + .filter((extension) => extensions.contributes(extension.name)) + .flatMap((extension) => + extensions + .getWidgetAugmentations(extension) + .filter( + (augmentation) => + augmentation.target.nodeType === nodeType.value && + augmentation.target.widgetName === widgetName.value + ) + .map((augmentation) => ({ + key: `${extension.name}:${augmentation.id}`, + extension: extension.name, + augmentation + })) + ) + ) + const contributions = computed(() => + registrations.value.filter((item) => !failed.value.has(item.augmentation)) + ) + + function evaluate( + contribution: Contribution, + read: () => T + ): T | undefined { + if ( + !contributions.value.some( + (item) => + item.extension === contribution.extension && + item.augmentation === contribution.augmentation + ) + ) + return undefined + try { + return read() + } catch (error) { + failed.value = new Set([...failed.value, contribution.augmentation]) + reportError(error, { + errorType: 'error_evaluating_widget_augmentation', + surface: 'platform', + context: { + extension: contribution.extension, + augmentation: contribution.augmentation.id + } + }) + } + } + + return { registrations, contributions, evaluate } +} diff --git a/src/platform/extensions/widgetAugmentation.ts b/src/platform/extensions/widgetAugmentation.ts new file mode 100644 index 00000000000..de914063e97 --- /dev/null +++ b/src/platform/extensions/widgetAugmentation.ts @@ -0,0 +1,66 @@ +export interface WidgetAugmentationTarget { + nodeType: string + widgetName: string +} + +export type WidgetAugmentation = { + id: string + target: WidgetAugmentationTarget +} & ( + | { + kind: 'text-completion' + suggest: (query: string) => readonly string[] + } + | { + kind: 'hint' + text: (value: string) => string + } + | { + kind: 'text-action' + label: string + transform: (value: string) => string + } +) + +function isWidgetAugmentationTarget( + target: unknown +): target is WidgetAugmentationTarget { + return ( + typeof target === 'object' && + target !== null && + 'nodeType' in target && + typeof target.nodeType === 'string' && + 'widgetName' in target && + typeof target.widgetName === 'string' + ) +} + +export function isWidgetAugmentation( + value: unknown +): value is WidgetAugmentation { + if ( + typeof value !== 'object' || + value === null || + !('id' in value) || + typeof value.id !== 'string' || + !('target' in value) || + !isWidgetAugmentationTarget(value.target) || + !('kind' in value) + ) + return false + switch (value.kind) { + case 'text-completion': + return 'suggest' in value && typeof value.suggest === 'function' + case 'hint': + return 'text' in value && typeof value.text === 'function' + case 'text-action': + return ( + 'label' in value && + typeof value.label === 'string' && + 'transform' in value && + typeof value.transform === 'function' + ) + default: + return false + } +} diff --git a/src/renderer/extensions/textEditor/components/TextEditorDialogContent.vue b/src/renderer/extensions/textEditor/components/TextEditorDialogContent.vue index 66214fb6cf5..15049e5b522 100644 --- a/src/renderer/extensions/textEditor/components/TextEditorDialogContent.vue +++ b/src/renderer/extensions/textEditor/components/TextEditorDialogContent.vue @@ -1,4 +1,39 @@ + + + {{ $t('textEditor.importTags') }} + + + + {{ + catalog.tags.length + ? $t('textEditor.tagsLoaded', catalog.tags.length) + : $t('textEditor.importTagsHint') + }} + + + {{ $t('textEditor.importTagsError') }} + + onEdit(String(value))" /> + diff --git a/src/renderer/extensions/textEditor/components/TextareaAutocomplete.vue b/src/renderer/extensions/textEditor/components/TextareaAutocomplete.vue new file mode 100644 index 00000000000..5d6a229286e --- /dev/null +++ b/src/renderer/extensions/textEditor/components/TextareaAutocomplete.vue @@ -0,0 +1,100 @@ + + + + + {{ $t('textEditor.suggestions') }} + + + + {{ match }} + + + + + + {{ matches.length ? $t('textEditor.suggestionCount', matches.length) : '' }} + + + + diff --git a/src/renderer/extensions/textEditor/promptTagCatalogStore.test.ts b/src/renderer/extensions/textEditor/promptTagCatalogStore.test.ts new file mode 100644 index 00000000000..1c9cfe6905b --- /dev/null +++ b/src/renderer/extensions/textEditor/promptTagCatalogStore.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest' + +import { usePromptTagCatalogStore } from './promptTagCatalogStore' + +describe('prompt tag catalog', () => { + it.for([ + { + text: '\uFEFFblue_eyes\r\nblue_hair\r\n', + expected: ['blue eyes', 'blue hair'] + }, + { text: 'blue_eyes,4\nblue_hair,9', expected: ['blue hair', 'blue eyes'] }, + { + text: 'name,"count"\nblue_eyes,"4"\nblue_hair,"19"', + expected: ['blue hair', 'blue eyes'] + }, + { + text: '"name","category","post_count","aliases"\n"blue_eyes","0","14",""\n"blue_hair","0","9",""', + expected: ['blue eyes', 'blue hair'] + }, + { + text: 'name,category,post_count,aliases\nblue_eyes,0,4,"azure_eyes,cyan_eyes"\nblue_hair,0,9,""\nblue_eyes,0,12,""', + expected: ['blue eyes', 'blue hair'] + }, + { text: '"blue, ""azure""",0,9,""', expected: ['blue, "azure"'] } + ])('imports supported catalog data: $text', ({ text, expected }) => { + const catalog = usePromptTagCatalogStore() + + expect(catalog.importCatalog(text)).toBe(true) + expect(catalog.tags).toEqual(expected) + }) + + it('ranks prefixes before substrings, then frequency, with normalized queries and a bounded list', () => { + const catalog = usePromptTagCatalogStore() + catalog.importCatalog( + 'dark_blue_hair,100\nblue_eyes,4\nblue_hair,9\nblue_sky,20\nblue_flower,18\nblue_dress,17\nblue_shirt,16\nblue_ribbon,15\nblue_bow,14\nblue_background,13' + ) + + expect(catalog.suggest('BLUE_')).toEqual([ + 'blue sky', + 'blue flower', + 'blue dress', + 'blue shirt', + 'blue ribbon', + 'blue bow', + 'blue background', + 'blue hair' + ]) + expect(catalog.suggest('blue hair')).toEqual(['dark blue hair']) + expect(catalog.suggest('nothing matching')).toEqual([]) + }) + + it.for([ + '', + 'blue_eyes,NaN', + 'blue_eyes,-1', + '"unterminated', + 'blue_eyes\n,0,1' + ])('keeps the previous catalog after rejecting %j', (text) => { + const catalog = usePromptTagCatalogStore() + catalog.importCatalog('blue_eyes') + + expect(catalog.importCatalog(text)).toBe(false) + expect(catalog.suggest('blue')).toEqual(['blue eyes']) + }) +}) diff --git a/src/renderer/extensions/textEditor/promptTagCatalogStore.ts b/src/renderer/extensions/textEditor/promptTagCatalogStore.ts new file mode 100644 index 00000000000..2013dbbfc02 --- /dev/null +++ b/src/renderer/extensions/textEditor/promptTagCatalogStore.ts @@ -0,0 +1,68 @@ +import { defineStore } from 'pinia' +import { shallowRef } from 'vue' + +function readCatalogFields(line: string) { + const fields = line.match(/^(?:"((?:[^"]|"")*)"|([^",]+))(?:,(.*))?$/) + if (!fields) return undefined + const name = (fields.at(1) ?? fields.at(2) ?? '') + .replaceAll('""', '"') + .replaceAll('_', ' ') + .trim() + const columns = fields.at(3)?.split(',') ?? [] + const frequency = columns.at(columns.length > 1 ? 1 : 0) + return { name, frequency: frequency?.replace(/^"(.*)"$/, '$1') } +} + +function parseCatalogLine( + line: string +): { name: string; count: number } | 'header' | undefined { + const fields = readCatalogFields(line) + if (!fields) return undefined + const { name, frequency } = fields + if ( + ['name', 'tag'].includes(name) && + ['count', 'post_count'].includes(frequency ?? '') + ) + return 'header' + const count = frequency === undefined ? 0 : Number(frequency) + if (!name || !Number.isFinite(count) || count < 0) return undefined + return { name, count } +} + +export const usePromptTagCatalogStore = defineStore('promptTagCatalog', () => { + const tags = shallowRef([]) + + function importCatalog(text: string): boolean { + const entries = new Map() + for (const line of text.replace(/^\uFEFF/, '').split(/\r?\n/)) { + if (!line.trim()) continue + const tag = parseCatalogLine(line) + if (!tag) return false + if (tag === 'header') { + if (entries.size > 0) return false + continue + } + entries.set(tag.name, Math.max(tag.count, entries.get(tag.name) ?? 0)) + } + if (!entries.size) return false + tags.value = [...entries].sort((a, b) => b[1] - a[1]).map(([name]) => name) + return true + } + + function suggest(query: string): string[] { + const search = query.toLowerCase().replaceAll('_', ' ') + const prefix: string[] = [] + const substring: string[] = [] + for (const tag of tags.value) { + const normalized = tag.toLowerCase() + if (normalized === search) continue + if (normalized.startsWith(search)) prefix.push(tag) + else if (substring.length < 8 && normalized.includes(search)) + substring.push(tag) + if (prefix.length === 8) break + } + return [...prefix, ...substring].slice(0, 8) + } + + return { tags, importCatalog, suggest } +}) diff --git a/src/renderer/extensions/textEditor/textEditorDialog.test.ts b/src/renderer/extensions/textEditor/textEditorDialog.test.ts index 3ada991b878..8aa86a8bcb6 100644 --- a/src/renderer/extensions/textEditor/textEditorDialog.test.ts +++ b/src/renderer/extensions/textEditor/textEditorDialog.test.ts @@ -1,18 +1,53 @@ import { fromPartial } from '@total-typescript/shoehorn' import { createEvent, render, screen, waitFor } from '@testing-library/vue' import userEvent from '@testing-library/user-event' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + assert, + beforeEach, + describe, + expect, + it, + onTestFinished, + vi +} from 'vitest' import { nextTick, ref } from 'vue' import GlobalDialog from '@/components/dialog/GlobalDialog.vue' +import { useToast } from '@/components/ui/toast/toastStore' import { i18n } from '@/i18n' +import { reportError } from '@comfyorg/telemetry/reportError' +import { useExtensionStore } from '@/platform/extensions/stores/extensionStore' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import { showTextEditor } from '@/renderer/extensions/textEditor/textEditorDialog' +import { usePromptTagCatalogStore } from '@/renderer/extensions/textEditor/promptTagCatalogStore' +import type { ComfyExtension } from '@/types/comfy' + +const widgetAugmentations = { + name: 'test.WidgetAugmentations', + widgetAugmentations: [ + { + id: 'studio-tags', + kind: 'text-completion', + target: { + nodeType: 'DevToolsNodeWithAugmentedTextarea', + widgetName: 'prompt' + }, + suggest: (query: string) => + ['studio lighting', 'studio portrait'].filter((tag) => + tag.includes(query) + ) + } + ] +} satisfies ComfyExtension const captureCanvasState = vi.fn() const markModified = vi.fn() const updateModified = vi.fn() +vi.mock(import('@comfyorg/telemetry/reportError'), () => ({ + reportError: vi.fn() +})) + function renderDialogHost() { return render(GlobalDialog, { global: { plugins: [i18n] } }) } @@ -133,6 +168,392 @@ describe('showTextEditor', () => { await waitFor(() => expect(editor).toHaveAttribute('readonly')) }) + it('imports a catalog and navigates suggestions without moving focus before dismissing them', async () => { + const user = userEvent.setup() + const source = ref('') + renderDialogHost() + openEditor({ + readText: () => source.value, + onEdit: (value) => (source.value = value) + }) + const editor = await screen.findByRole('textbox') + + expect(screen.queryByRole('listbox')).toBeNull() + await user.upload( + screen.getByLabelText('Import prompt tags'), + new File(['cinematic_lighting,20\nsoft_lighting,10'], 'tags.csv', { + type: 'text/csv' + }) + ) + expect( + await screen.findByText('2 tags loaded for this session') + ).toBeVisible() + await user.type(editor, 'a portrait, light') + + expect( + await screen.findByRole('listbox', { name: 'Prompt suggestions' }) + ).toBeVisible() + expect(screen.getByRole('option', { selected: true })).toHaveTextContent( + 'cinematic lighting' + ) + await user.keyboard('{ArrowDown}') + expect(screen.getByRole('option', { selected: true })).toHaveTextContent( + 'soft lighting' + ) + expect(editor).toHaveFocus() + expect(editor).toHaveAttribute( + 'aria-activedescendant', + screen.getByRole('option', { selected: true }).id + ) + + await user.keyboard('{Escape}') + + expect(screen.queryByRole('listbox')).toBeNull() + expect(screen.getByRole('dialog')).toBeVisible() + expect(editor).toHaveValue('a portrait, light') + await user.keyboard('{Escape}') + await waitFor(() => expect(screen.queryByRole('dialog')).toBeNull()) + }) + + it.for([ + { + nodeType: 'DevToolsNodeWithAugmentedTextarea', + widgetName: 'prompt', + expected: ['studio lighting', 'studio portrait'] + }, + { + nodeType: 'DevToolsNodeWithAugmentedTextarea', + widgetName: 'notes', + expected: [] + }, + { nodeType: 'OtherNode', widgetName: 'prompt', expected: [] } + ])( + 'scopes extension completion to $nodeType.$widgetName', + async ({ nodeType, widgetName, expected }) => { + const user = userEvent.setup() + const source = ref('') + useExtensionStore().registerExtension(widgetAugmentations) + renderDialogHost() + openEditor({ + readText: () => source.value, + onEdit: (value) => (source.value = value), + augmentationTarget: { nodeType, widgetName } + }) + const editor = await screen.findByRole('textbox') + await user.type(editor, 'studio') + + expect( + screen.queryAllByRole('option').map((item) => item.textContent.trim()) + ).toEqual(expected) + expect(editor).toHaveValue('studio') + expect(editor).toHaveFocus() + } + ) + + it('removes extension suggestions when the extension unregisters without disabling the imported catalog', async () => { + const user = userEvent.setup() + const source = ref('') + const extensions = useExtensionStore() + extensions.registerExtension(widgetAugmentations) + usePromptTagCatalogStore().importCatalog('studio_backdrop') + renderDialogHost() + openEditor({ + readText: () => source.value, + onEdit: (value) => (source.value = value), + augmentationTarget: { + nodeType: 'DevToolsNodeWithAugmentedTextarea', + widgetName: 'prompt' + } + }) + const editor = await screen.findByRole('textbox') + await user.type(editor, 'studio') + expect( + await screen.findByRole('option', { name: 'studio lighting' }) + ).toBeVisible() + expect( + screen.getByRole('option', { name: 'studio backdrop' }) + ).toBeVisible() + + extensions.unregisterExtension(widgetAugmentations.name) + await waitFor(() => expect(screen.queryByRole('listbox')).toBeNull()) + await user.type(editor, ' ') + + expect( + await screen.findByRole('option', { name: 'studio backdrop' }) + ).toBeVisible() + expect(screen.queryByRole('option', { name: 'studio lighting' })).toBeNull() + }) + + it('isolates a failed extension provider while retaining catalog suggestions', async () => { + const user = userEvent.setup() + const source = ref('') + const failure = new Error('Provider unavailable') + useExtensionStore().registerExtension({ + name: 'test.failingCompletion', + widgetAugmentations: [ + { + id: 'failing-tags', + kind: 'text-completion', + target: { nodeType: 'TestNode', widgetName: 'prompt' }, + suggest() { + throw failure + } + } + ] + }) + usePromptTagCatalogStore().importCatalog('studio_backdrop') + renderDialogHost() + openEditor({ + readText: () => source.value, + onEdit: (value) => (source.value = value), + augmentationTarget: { nodeType: 'TestNode', widgetName: 'prompt' } + }) + const editor = await screen.findByRole('textbox') + await user.type(editor, 'studio') + + expect( + await screen.findByRole('option', { name: 'studio backdrop' }) + ).toBeVisible() + expect(editor).toHaveValue('studio') + expect(reportError).toHaveBeenCalledExactlyOnceWith(failure, { + errorType: 'error_evaluating_widget_augmentation', + surface: 'platform', + context: { + extension: 'test.failingCompletion', + augmentation: 'failing-tags' + } + }) + }) + + it('updates a widget hint from live text and removes it when its extension is disabled', async () => { + const user = userEvent.setup() + const source = ref('blue') + const extensions = useExtensionStore() + extensions.captureCoreExtensions() + extensions.registerExtension({ + name: 'test.hint', + widgetAugmentations: [ + { + id: 'length', + kind: 'hint', + target: { nodeType: 'TestNode', widgetName: 'prompt' }, + text: (value) => `Length: ${value.length}` + } + ] + }) + renderDialogHost() + openEditor({ + readText: () => source.value, + onEdit: (value) => (source.value = value), + augmentationTarget: { nodeType: 'TestNode', widgetName: 'prompt' } + }) + const editor = await screen.findByRole('textbox') + expect(await screen.findByText('Length: 4')).toBeVisible() + await user.clear(editor) + await user.type(editor, 'blue sky') + expect(await screen.findByText('Length: 8')).toBeVisible() + + extensions.loadDisabledExtensionNames(['test.hint']) + + await waitFor(() => expect(screen.queryByText('Length: 8')).toBeNull()) + expect(editor).toHaveValue('blue sky') + }) + + it('applies a text action through live editing and locks it when read-only', async () => { + const insertText: Document['execCommand'] = (_command, _showUI, text) => { + editor.setRangeText(text ?? '') + editor.dispatchEvent(new Event('input', { bubbles: true })) + return true + } + Object.defineProperty(document, 'execCommand', { + configurable: true, + value: insertText + }) + onTestFinished(() => { + Reflect.deleteProperty(document, 'execCommand') + }) + const user = userEvent.setup() + const source = ref('blue_eyes, soft_focus') + const readOnly = ref(false) + useExtensionStore().registerExtension({ + name: 'test.action', + widgetAugmentations: [ + { + id: 'normalize', + kind: 'text-action', + target: { nodeType: 'TestNode', widgetName: 'prompt' }, + label: 'Normalize tags', + transform: (value) => value.replaceAll('_', ' ') + } + ] + }) + renderDialogHost() + openEditor({ + readText: () => source.value, + readOnly: () => readOnly.value, + onEdit: (value) => (source.value = value), + augmentationTarget: { nodeType: 'TestNode', widgetName: 'prompt' } + }) + const editor = await screen.findByRole('textbox') + await user.click(screen.getByRole('button', { name: 'Normalize tags' })) + + expect(source.value).toBe('blue eyes, soft focus') + expect(editor).toHaveValue('blue eyes, soft focus') + expect(editor).toHaveFocus() + expect(markModified).toHaveBeenCalledOnce() + readOnly.value = true + await waitFor(() => + expect( + screen.getByRole('button', { name: 'Normalize tags' }) + ).toBeDisabled() + ) + }) + + it.for([ + { + maxlength: 5, + message: + 'This action exceeds the text limit. Shorten the text and try again.' + }, + { + maxlength: 100, + message: + 'This action could not edit the text. Your text is unchanged. Edit it manually instead.' + } + ])( + 'explains a refused text action with maxlength $maxlength', + async ({ maxlength, message }) => { + const user = userEvent.setup() + useExtensionStore().registerExtension({ + name: 'test.expandingAction', + widgetAugmentations: [ + { + id: 'expand', + kind: 'text-action', + target: { nodeType: 'TestNode', widgetName: 'prompt' }, + label: 'Expand tag', + transform: () => 'blue eyes' + } + ] + }) + renderDialogHost() + const { onEdit } = openEditor({ + readText: () => 'blue', + textareaAttributes: () => ({ maxlength }), + augmentationTarget: { nodeType: 'TestNode', widgetName: 'prompt' } + }) + const editor = await screen.findByRole('textbox') + + await user.click(screen.getByRole('button', { name: 'Expand tag' })) + + expect(editor).toHaveValue('blue') + expect(onEdit).not.toHaveBeenCalled() + expect(useToast().toasts).toContainEqual( + expect.objectContaining({ + kind: 'warning', + title: message + }) + ) + } + ) + + it('discards an unfinished catalog import when another editor opens', async () => { + const user = userEvent.setup() + let finish: ((text: string) => void) | undefined + const pending = new Promise((resolve) => (finish = resolve)) + const file = new File(['blue_eyes'], 'old.txt', { type: 'text/plain' }) + vi.spyOn(file, 'text').mockReturnValue(pending) + renderDialogHost() + const { close } = openEditor() + await screen.findByRole('textbox') + await user.upload(screen.getByLabelText('Import prompt tags'), file) + close() + await waitFor(() => expect(screen.queryByRole('dialog')).toBeNull()) + openEditor() + const editor = await screen.findByRole('textbox') + await user.upload( + screen.getByLabelText('Import prompt tags'), + new File(['red_hair'], 'new.txt', { type: 'text/plain' }) + ) + expect( + await screen.findByText('1 tag loaded for this session') + ).toBeVisible() + + assert.exists(finish) + finish('blue_eyes') + await pending + await nextTick() + + expect(usePromptTagCatalogStore().tags).toEqual(['red hair']) + expect(editor).toHaveFocus() + }) + + it('reports an invalid import without losing the loaded catalog', async () => { + const user = userEvent.setup() + usePromptTagCatalogStore().importCatalog('blue_eyes') + renderDialogHost() + openEditor() + await screen.findByRole('textbox') + + await user.upload( + screen.getByLabelText('Import prompt tags'), + new File(['blue_hair,not-a-count'], 'invalid.csv', { type: 'text/csv' }) + ) + + expect(await screen.findByRole('alert')).toHaveTextContent( + 'Your previous catalog is unchanged.' + ) + expect(screen.getByText('1 tag loaded for this session')).toBeVisible() + }) + + it('removes suggestions when the editor becomes read-only', async () => { + const user = userEvent.setup() + const source = ref('') + const readOnly = ref(false) + usePromptTagCatalogStore().importCatalog('cinematic_lighting') + renderDialogHost() + openEditor({ + readText: () => source.value, + onEdit: (value) => (source.value = value), + readOnly: () => readOnly.value + }) + const editor = await screen.findByRole('textbox') + await user.type(editor, 'cin') + expect(await screen.findByRole('listbox')).toBeVisible() + + readOnly.value = true + + await waitFor(() => expect(screen.queryByRole('listbox')).toBeNull()) + expect(editor).toHaveValue('cin') + }) + + it('suspends suggestions during composition and resumes after committed input', async () => { + const user = userEvent.setup() + const source = ref('') + usePromptTagCatalogStore().importCatalog('cinematic_lighting') + renderDialogHost() + openEditor({ + readText: () => source.value, + onEdit: (value) => (source.value = value) + }) + const editor = await screen.findByRole('textbox') + await user.type(editor, 'cin') + expect(await screen.findByRole('listbox')).toBeVisible() + + editor.dispatchEvent(createEvent.compositionStart(editor)) + await nextTick() + expect(screen.queryByRole('listbox')).toBeNull() + editor.dispatchEvent( + createEvent.keyDown(editor, { key: 'Enter', isComposing: true }) + ) + expect(editor).toHaveValue('cin') + editor.dispatchEvent(createEvent.compositionEnd(editor)) + + expect( + await screen.findByRole('option', { name: 'cinematic lighting' }) + ).toBeVisible() + }) + it('keeps the selection and its direction when the text changes elsewhere', async () => { const source = ref('a portrait at dusk') renderDialogHost() diff --git a/src/renderer/extensions/textEditor/textEditorDialog.ts b/src/renderer/extensions/textEditor/textEditorDialog.ts index 5e915f686c8..5dc01bc9fb6 100644 --- a/src/renderer/extensions/textEditor/textEditorDialog.ts +++ b/src/renderer/extensions/textEditor/textEditorDialog.ts @@ -1,6 +1,7 @@ import type { TextareaHTMLAttributes } from 'vue' import { t } from '@/i18n' +import type { WidgetAugmentationTarget } from '@/platform/extensions/widgetAugmentation' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import TextEditorDialogContent from '@/renderer/extensions/textEditor/components/TextEditorDialogContent.vue' import TextEditorDialogHeader from '@/renderer/extensions/textEditor/components/TextEditorDialogHeader.vue' @@ -11,6 +12,7 @@ interface TextEditorOptions { readText: () => string widgetLabel: string nodeTitle?: string + augmentationTarget?: WidgetAugmentationTarget readOnly: () => boolean onEdit: (text: string) => void textareaAttributes?: () => TextareaHTMLAttributes @@ -20,6 +22,7 @@ export function showTextEditor({ readText, widgetLabel, nodeTitle, + augmentationTarget, readOnly, onEdit, textareaAttributes @@ -42,6 +45,7 @@ export function showTextEditor({ props: { readText, label: widgetLabel, + augmentationTarget, readOnly, attributes: textareaAttributes, onEdit: (value: string) => { diff --git a/src/renderer/extensions/textEditor/useTextareaAutocomplete.ts b/src/renderer/extensions/textEditor/useTextareaAutocomplete.ts new file mode 100644 index 00000000000..7437f1f5486 --- /dev/null +++ b/src/renderer/extensions/textEditor/useTextareaAutocomplete.ts @@ -0,0 +1,205 @@ +import { flip, offset, shift, useFloating } from '@floating-ui/vue' +import { useEventListener, useResizeObserver } from '@vueuse/core' +import type { Ref } from 'vue' +import { computed, ref, shallowRef, watch } from 'vue' + +import { useTextareaEdit } from '@/renderer/extensions/textEditor/useTextareaEdit' + +function caretRectangle(element: HTMLTextAreaElement): DOMRect { + const mirror = document.createElement('div') + const style = getComputedStyle(element) + for (const property of [ + 'box-sizing', + 'width', + 'border-width', + 'border-style', + 'padding', + 'font', + 'line-height', + 'letter-spacing', + 'word-spacing', + 'text-indent', + 'text-align', + 'direction', + 'tab-size', + 'word-break', + 'overflow-wrap', + 'scrollbar-gutter' + ]) { + mirror.style.setProperty(property, style.getPropertyValue(property)) + } + Object.assign(mirror.style, { + position: 'fixed', + visibility: 'hidden', + whiteSpace: 'pre-wrap', + overflow: 'hidden', + top: '0', + left: '0' + }) + mirror.textContent = element.value.slice(0, element.selectionStart) + const marker = document.createElement('span') + marker.textContent = element.value.slice(element.selectionStart) || '\u200b' + mirror.append(marker) + document.body.append(mirror) + try { + const origin = element.getBoundingClientRect() + const caret = + marker.getClientRects().item(0) ?? marker.getBoundingClientRect() + const scaleX = element.offsetWidth ? origin.width / element.offsetWidth : 1 + const scaleY = element.offsetHeight + ? origin.height / element.offsetHeight + : 1 + return new DOMRect( + origin.left + (caret.left - element.scrollLeft) * scaleX, + origin.top + (caret.top - element.scrollTop) * scaleY, + 0, + (caret.height || parseFloat(style.lineHeight)) * scaleY + ) + } finally { + mirror.remove() + } +} + +export function useTextareaAutocomplete({ + element, + floating, + enabled, + suggest +}: { + element: () => HTMLTextAreaElement | undefined + floating: Readonly> + enabled: () => boolean + suggest: (query: string) => string[] +}) { + const anchor = shallowRef<{ getBoundingClientRect: () => DOMRect }>() + const query = shallowRef<{ + value: string + from: number + to: number + matches: string[] + }>() + const active = ref(0) + let composing = false + const edit = useTextareaEdit(element, enabled) + const matches = computed(() => query.value?.matches ?? []) + const { floatingStyles, update } = useFloating(anchor, floating, { + strategy: 'fixed', + placement: 'bottom-start', + middleware: [offset(6), flip(), shift({ padding: 12 })] + }) + + function close() { + query.value = undefined + } + + function sync() { + const input = element() + const current = edit.snapshot() + if ( + !input || + !current || + document.activeElement !== input || + current.start !== current.end + ) { + close() + return + } + const to = input.selectionStart + const token = input.value.slice(0, to).match(/[^,;\n]+$/)?.[0] ?? '' + const search = token.trimStart() + const found = search.length < 2 ? [] : suggest(search) + if (!found.length) { + close() + return + } + const rectangle = caretRectangle(input) + anchor.value = { getBoundingClientRect: () => rectangle } + query.value = { + value: input.value, + from: to - search.length, + to, + matches: found + } + active.value = 0 + update() + } + + function accept(index: number) { + const current = query.value + const text = current?.matches[index] + if (!current || !text) return + edit.replace( + { value: current.value, start: current.to, end: current.to }, + { from: current.from, to: current.to }, + text + ) + close() + } + + function navigate(event: KeyboardEvent) { + if (event.key === 'Tab') { + close() + return + } + if (!['ArrowDown', 'ArrowUp', 'Enter', 'Escape'].includes(event.key)) return + event.preventDefault() + event.stopPropagation() + if (event.key === 'Escape') close() + else if (event.key === 'Enter') accept(active.value) + else { + active.value = + (active.value + + (event.key === 'ArrowDown' ? 1 : -1) + + matches.value.length) % + matches.value.length + } + } + + useEventListener(element, 'input', sync) + useEventListener(element, 'click', sync) + useEventListener(element, 'select', () => { + const input = element() + if ( + query.value && + (input?.selectionStart !== query.value.to || + input.selectionEnd !== query.value.to) + ) + close() + }) + useEventListener(element, 'keyup', (event) => { + if (['ArrowLeft', 'ArrowRight', 'Home', 'End'].includes(event.key)) sync() + }) + useEventListener(element, 'scroll', close) + useEventListener(element, 'blur', close) + useEventListener(document, 'wheel', close, { capture: true, passive: true }) + useResizeObserver(element, close) + useEventListener(window, 'resize', close) + useEventListener(element, 'compositionstart', () => { + composing = true + close() + }) + useEventListener(element, 'compositionend', () => { + composing = false + sync() + }) + useEventListener( + element, + 'keydown', + (event) => { + if (event.isComposing || event.keyCode === 229 || composing) return + if ( + !query.value || + event.ctrlKey || + event.metaKey || + event.altKey || + event.shiftKey + ) + return + navigate(event) + }, + { capture: true } + ) + watch(enabled, close) + + return { matches, active, floatingStyles, accept, close } +} diff --git a/src/renderer/extensions/textEditor/useTextareaEdit.ts b/src/renderer/extensions/textEditor/useTextareaEdit.ts new file mode 100644 index 00000000000..7be86689d09 --- /dev/null +++ b/src/renderer/extensions/textEditor/useTextareaEdit.ts @@ -0,0 +1,65 @@ +import { useEventListener } from '@vueuse/core' + +interface TextSnapshot { + value: string + start: number + end: number +} + +type TextEditResult = + | { ok: true } + | { ok: false; reason: 'maxlength' | 'unavailable' } + +function sameSnapshot(current: TextSnapshot, previous: TextSnapshot): boolean { + return ( + current.value === previous.value && + current.start === previous.start && + current.end === previous.end + ) +} + +export function useTextareaEdit( + element: () => HTMLTextAreaElement | undefined, + enabled: () => boolean +) { + let composing = false + useEventListener(element, 'compositionstart', () => (composing = true)) + useEventListener(element, 'compositionend', () => (composing = false)) + + function snapshot(): TextSnapshot | undefined { + const input = element() + if (!input || !enabled() || input.readOnly || input.disabled || composing) + return undefined + return { + value: input.value, + start: input.selectionStart, + end: input.selectionEnd + } + } + + function replace( + previous: TextSnapshot, + range: { from: number; to: number }, + text: string + ): TextEditResult { + const input = element() + const current = snapshot() + if (!input || !current || !sameSnapshot(current, previous)) + return { ok: false, reason: 'unavailable' } + if ( + input.maxLength >= 0 && + current.value.length - (range.to - range.from) + text.length > + input.maxLength + ) + return { ok: false, reason: 'maxlength' } + if (typeof document.execCommand !== 'function') + return { ok: false, reason: 'unavailable' } + input.focus() + input.setSelectionRange(range.from, range.to) + const applied = document.execCommand('insertText', false, text) + if (!applied) input.setSelectionRange(previous.start, previous.end) + return applied ? { ok: true } : { ok: false, reason: 'unavailable' } + } + + return { snapshot, replace } +} diff --git a/src/renderer/extensions/vueNodes/widgets/components/WidgetTextarea.test.ts b/src/renderer/extensions/vueNodes/widgets/components/WidgetTextarea.test.ts index d687843d86b..25f6d971669 100644 --- a/src/renderer/extensions/vueNodes/widgets/components/WidgetTextarea.test.ts +++ b/src/renderer/extensions/vueNodes/widgets/components/WidgetTextarea.test.ts @@ -11,6 +11,7 @@ import { describe, expect, it, vi } from 'vitest' import GlobalDialog from '@/components/dialog/GlobalDialog.vue' import { i18n } from '@/i18n' +import { usePromptTagCatalogStore } from '@/renderer/extensions/textEditor/promptTagCatalogStore' import type { SimplifiedWidget } from '@/types/simplifiedWidget' import WidgetTextarea from './WidgetTextarea.vue' @@ -408,6 +409,34 @@ describe('WidgetTextarea contextmenu', () => { }) }) +describe('WidgetTextarea autocomplete', () => { + it('uses the shared catalog and closes suggestions when the node becomes read-only', async () => { + const user = userEvent.setup() + usePromptTagCatalogStore().importCatalog('blue_eyes,20\nblue_hair,10') + const widget = createTextareaWidget('') + const view = renderComponent(widget, '') + const textarea = screen.getByRole('textbox') + + await user.type(textarea, 'blue') + + expect(await screen.findByRole('listbox')).toBeVisible() + expect(screen.getByRole('option', { selected: true })).toHaveTextContent( + 'blue eyes' + ) + expect(textarea).toHaveFocus() + await user.keyboard('{ArrowDown}') + expect(textarea).toHaveAttribute( + 'aria-activedescendant', + screen.getByRole('option', { name: 'blue hair' }).id + ) + + await view.rerender({ widget: { ...widget, options: { read_only: true } } }) + + expect(screen.queryByRole('listbox')).toBeNull() + expect(textarea).toHaveAttribute('readonly') + }) +}) + describe('WidgetTextarea expand to editor', () => { function renderWithDialogHost( widgetOverrides: Partial> = {} diff --git a/src/renderer/extensions/vueNodes/widgets/components/WidgetTextarea.vue b/src/renderer/extensions/vueNodes/widgets/components/WidgetTextarea.vue index 7952c346bf7..84105848df0 100644 --- a/src/renderer/extensions/vueNodes/widgets/components/WidgetTextarea.vue +++ b/src/renderer/extensions/vueNodes/widgets/components/WidgetTextarea.vue @@ -83,6 +83,16 @@ @contextmenu.capture="handleContextMenu" @update:model-value="(value) => (modelValue = String(value))" /> + @@ -97,6 +107,7 @@ import Button from '@comfyorg/design-system/button/Button' import Textarea from '@comfyorg/design-system/textarea/Textarea' import { useCopyToClipboard } from '@/composables/useCopyToClipboard' import { isNodeOptionsOpen } from '@/composables/graph/useMoreOptionsMenu' +import TextareaAugmentations from '@/renderer/extensions/textEditor/components/TextareaAugmentations.vue' import { showTextEditor } from '@/renderer/extensions/textEditor/textEditorDialog' import type { SimplifiedWidget } from '@/types/simplifiedWidget' import { useHideLayoutField } from '@/types/widgetTypes' @@ -111,16 +122,23 @@ const actionButtonClass = const { widget, + nodeType = '', nodeTitle, placeholder = '' } = defineProps<{ widget: SimplifiedWidget + nodeType?: string nodeTitle?: string placeholder?: string }>() const textAreaRef = useTemplateRef('textAreaRef') +function textareaElement(): HTMLTextAreaElement | undefined { + const element: unknown = textAreaRef.value?.$el + return element instanceof HTMLTextAreaElement ? element : undefined +} + const modelValue = defineModel({ default: '' }) const text = computed(() => String(modelValue.value ?? '')) @@ -173,6 +191,10 @@ function openTextEditor() { readText: () => text.value, widgetLabel: displayName.value, nodeTitle, + augmentationTarget: { + nodeType: widget.nodeType ?? nodeType, + widgetName: widget.name + }, readOnly: () => isReadOnly.value, onEdit: (text) => (modelValue.value = text), textareaAttributes: () => ({ diff --git a/src/types/comfy.ts b/src/types/comfy.ts index 417e0a350b8..d6c5c0c93d8 100644 --- a/src/types/comfy.ts +++ b/src/types/comfy.ts @@ -1,4 +1,5 @@ import type { ShallowRef } from 'vue' +import type { WidgetAugmentation } from '@/platform/extensions/widgetAugmentation' import type { Positionable } from '@comfyorg/graph-model/interfaces' import type { IContextMenuValue } from '@/lib/litegraph/src/types/contextMenu' @@ -294,6 +295,7 @@ export interface ActionBarButton { } export interface ComfyExtension { + widgetAugmentations?: readonly WidgetAugmentation[] /** * The name of the extension */ diff --git a/tools/devtools/README.md b/tools/devtools/README.md index 6d4ceae428a..669e8dd00cc 100644 --- a/tools/devtools/README.md +++ b/tools/devtools/README.md @@ -24,6 +24,47 @@ During CI/CD, these files are automatically copied to the ComfyUI `custom_nodes` cp -r tools/devtools/* /path/to/your/ComfyUI/custom_nodes/ComfyUI_devtools/ ``` +## Try widget augmentation + +Before copying devtools into ComfyUI, build the TypeScript example: + +```bash +pnpm exec tsx tools/devtools/build.ts +``` + +Restart ComfyUI and reload the frontend. Add **Node With Augmented Textarea**. +Type `studio` in **prompt** to open suggestions, then press Enter to insert one. +Press Undo to restore the query. The **notes** field has no extension suggestions. +With Nodes 2.0 enabled, expand **prompt** and try the same query in the dialog. +The live hint counts prompt tags. Enter `blue_eyes ; soft_focus`, then click +**Normalize tags** to produce `blue eyes, soft focus`. Undo restores the original. + +The example declares `widgetAugmentations` through `app.registerExtension`. +Each contribution has an extension-scoped `id`, a `target` identifying the node +type and widget name, and a `kind`: + +- `text-completion` supplies synchronous suggestions through `suggest(query)`. +- `hint` returns plain text from the current widget value through `text(value)`. +- `text-action` supplies a button `label` and a synchronous `transform(value)` + returning the replacement text. + +Core renders contributions, isolates callback failures, honors extension +enablement, and removes contributions when an extension unregisters. Text edits +use the normal input pipeline and native undo. Read-only, disabled, composing, +stale, and over-length edits are refused. There is no direct-value fallback +when native insertion is unsupported. + +The prototype supports multiline native textareas in both node renderers and +the expanded editor. It does not expose widget elements, callbacks, arbitrary +mounting, or a generic value setter. It does not augment canvas-painted controls +or the ProseMirror agent editor. Register immutable contributions; replace the +extension registration rather than mutating an installed contribution. + +The Vite development server does not load backend extension scripts. To try the +example there, register the exported extension from the TypeScript example +through `app.registerExtension` after startup. A built frontend loads the +generated script through the usual ComfyUI extension loader. + ## DynamicGroup contract test `Node With Dynamic Group` repeats a LoRA-shaped widget template and returns its diff --git a/tools/devtools/build.ts b/tools/devtools/build.ts new file mode 100644 index 00000000000..3633fc50997 --- /dev/null +++ b/tools/devtools/build.ts @@ -0,0 +1,18 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import ts from 'typescript' + +const source = await readFile( + new URL('./widgetAugmentations.ts', import.meta.url), + 'utf8' +) +const { outputText } = ts.transpileModule(source, { + compilerOptions: { + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext + } +}) +await writeFile( + fileURLToPath(new URL('./web/widgetAugmentations.js', import.meta.url)), + `${outputText}\nimport { app } from '../../scripts/app.js'\napp.registerExtension(extension)\n` +) diff --git a/tools/devtools/nodes/inputs.py b/tools/devtools/nodes/inputs.py index 1c9e9e0b65c..d230a02a63c 100644 --- a/tools/devtools/nodes/inputs.py +++ b/tools/devtools/nodes/inputs.py @@ -173,6 +173,24 @@ def node_with_string_input(self, string_input: str): print(f"string_input: {string_input}") +class NodeWithAugmentedTextarea: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt": ("STRING", {"multiline": True}), + "notes": ("STRING", {"multiline": True}), + } + } + + RETURN_TYPES = ("STRING",) + FUNCTION = "execute" + CATEGORY = "DevTools" + + def execute(self, prompt: str, notes: str): + return (prompt,) + + class NodeWithUnionInput: @classmethod def INPUT_TYPES(cls): @@ -571,6 +589,7 @@ async def execute(cls): "DevToolsNodeWithForceInput": NodeWithForceInput, "DevToolsNodeWithDefaultInput": NodeWithDefaultInput, "DevToolsNodeWithStringInput": NodeWithStringInput, + "DevToolsNodeWithAugmentedTextarea": NodeWithAugmentedTextarea, "DevToolsNodeWithUnionInput": NodeWithUnionInput, "DevToolsNodeWithBooleanInput": NodeWithBooleanInput, "DevToolsNodeWithColorInput": NodeWithColorInput, @@ -604,6 +623,7 @@ async def execute(cls): "DevToolsNodeWithForceInput": "Node With Force Input", "DevToolsNodeWithDefaultInput": "Node With Default Input", "DevToolsNodeWithStringInput": "Node With String Input", + "DevToolsNodeWithAugmentedTextarea": "Node With Augmented Textarea", "DevToolsNodeWithUnionInput": "Node With Union Input", "DevToolsNodeWithBooleanInput": "Node With Boolean Input", "DevToolsNodeWithColorInput": "Node With Color Input", diff --git a/tools/devtools/widgetAugmentations.ts b/tools/devtools/widgetAugmentations.ts new file mode 100644 index 00000000000..f7d52ea0dcb --- /dev/null +++ b/tools/devtools/widgetAugmentations.ts @@ -0,0 +1,52 @@ +import type { WidgetAugmentation } from '../../src/platform/extensions/widgetAugmentation' + +const tags = [ + 'studio lighting', + 'studio portrait', + 'soft lighting', + 'soft focus', + 'cinematic lighting' +] + +const target = { + nodeType: 'DevToolsNodeWithAugmentedTextarea', + widgetName: 'prompt' +} + +const extension = { + name: 'DevTools.WidgetAugmentations', + widgetAugmentations: [ + { + id: 'lighting-tags', + kind: 'text-completion', + target, + suggest(query: string) { + return tags.filter((tag) => tag.includes(query.toLowerCase())) + } + }, + { + id: 'tag-count', + kind: 'hint', + target, + text(value: string) { + const count = value.split(/[,;\n]/).filter((tag) => tag.trim()).length + return `${count} prompt tags` + } + }, + { + id: 'normalize-tags', + kind: 'text-action', + target, + label: 'Normalize tags', + transform(value: string) { + return value + .split(/[,;\n]/) + .map((tag) => tag.trim().replaceAll('_', ' ')) + .filter(Boolean) + .join(', ') + } + } + ] satisfies readonly WidgetAugmentation[] +} + +export default extension
+ {{ $t('textEditor.importTagsError') }} +