Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 43 additions & 9 deletions src/extensions/core/simpleTouchSupport.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { fromPartial } from '@total-typescript/shoehorn'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest'
import { shallowRef } from 'vue'

import type { DragAndScale } from '@/lib/litegraph/src/litegraph'
import { LGraphCanvas } from '@/lib/litegraph/src/litegraph'
Expand All @@ -13,11 +14,11 @@ type Viewport = Pick<
'scale' | 'offset' | 'min_scale' | 'max_scale'
>

const { canvasEl, container } = vi.hoisted(() => {
const { canvasEl, container, canvasAvailability } = vi.hoisted(() => {
const container = document.createElement('div')
const canvasEl = document.createElement('canvas')
container.append(canvasEl)
return { container, canvasEl }
return { container, canvasEl, canvasAvailability: { missing: false } }
})

const processMouseDown = vi.fn()
Expand All @@ -30,14 +31,18 @@ vi.mock(import('@/scripts/app'), () => {
min_scale: 0.1,
max_scale: 10
}
const canvas = fromPartial<LGraphCanvas>({
ds,
pointer: { isDown: false },
setDirty: vi.fn()
})
return {
app: fromPartial<typeof comfyApp>({
canvasEl,
canvas: fromPartial({
ds,
pointer: { isDown: false },
setDirty: vi.fn()
})
canvasElRef: shallowRef(canvasEl),
get canvasOrUndefined() {
return canvasAvailability.missing ? undefined : canvas
}
})
}
})
Expand Down Expand Up @@ -82,10 +87,19 @@ function dispatchTouch(
}

function pressPointer(init: PointerEventInit) {
const canvas = fromPartial<LGraphCanvas>({ pointer: { isDown: true } })
LGraphCanvas.prototype.processMouseDown.call(
fromPartial<LGraphCanvas>({}),
canvas,
new PointerEvent('pointerdown', { pointerType: 'touch', ...init })
)
return canvas
}

function withoutCanvas() {
canvasAvailability.missing = true
onTestFinished(() => {
canvasAvailability.missing = false
})
}

/**
Expand Down Expand Up @@ -138,6 +152,26 @@ describe('Comfy.SimpleTouchSupport', () => {
expect(processMouseDown).not.toHaveBeenCalled()
}
)

it('presses through to the original when no canvas exists', () => {
withoutCanvas()

const canvas = pressPointer({ isPrimary: true })

expect(processMouseDown).toHaveBeenCalledOnce()
expect(canvas.pointer.isDown).toBe(false)
})

it('ignores a pinch when no canvas exists', () => {
withoutCanvas()
const twoFingers = [touchAt(canvasEl), touchAt(canvasEl, 100)]

dispatchTouch('touchstart', twoFingers)
dispatchTouch('touchmove', twoFingers)
pressPointer({ isPrimary: true })

expect(processMouseDown).toHaveBeenCalledOnce()
})
})

describe('touch pointer guards', () => {
Expand Down
94 changes: 56 additions & 38 deletions src/extensions/core/simpleTouchSupport.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,56 @@
import { LiteGraph } from '@/lib/litegraph/src/litegraph'
import type { LGraphCanvas } from '@/lib/litegraph/src/litegraph'

import { app } from '@/scripts/app'

import { canvasOrReport } from './touchCanvasAccess'
import { touchGesture } from './touchPointerGuards'
import type { ComfyExtension } from '@/types/comfy'

type TouchPoint = { clientX: number; clientY: number }

function zoomCanvasAroundPinch(
canvas: LGraphCanvas,
requestedScale: number,
center: TouchPoint,
lastTouch: TouchPoint
) {
let scale = requestedScale
const newX = (center.clientX - lastTouch.clientX) / requestedScale
const newY = (center.clientY - lastTouch.clientY) / requestedScale

if (scale < canvas.ds.min_scale) {
scale = canvas.ds.min_scale
} else if (scale > canvas.ds.max_scale) {
scale = canvas.ds.max_scale
}

const oldScale = canvas.ds.scale

canvas.ds.scale = scale

if (Math.abs(canvas.ds.scale - 1) < 0.01) {
canvas.ds.scale = 1
}

const newScale = canvas.ds.scale

const convertScaleToOffset = (scale: number) => [
center.clientX / scale - canvas.ds.offset[0],
center.clientY / scale - canvas.ds.offset[1]
]
const oldCenter = convertScaleToOffset(oldScale)
const newCenter = convertScaleToOffset(newScale)

canvas.ds.offset[0] += newX + newCenter[0] - oldCenter[0]
canvas.ds.offset[1] += newY + newCenter[1] - oldCenter[1]

lastTouch.clientX = center.clientX
lastTouch.clientY = center.clientY

canvas.setDirty(true, true)
}

export const simpleTouchSupportExtension: ComfyExtension = {
name: 'Comfy.SimpleTouchSupport',
setup() {
Expand Down Expand Up @@ -39,11 +85,13 @@ export const simpleTouchSupportExtension: ComfyExtension = {
} else {
touchTime = null
if (e.touches.length === 2) {
lastScale = app.canvas.ds.scale
const canvas = canvasOrReport('touchstart', e)
if (!canvas) return
lastScale = canvas.ds.scale
lastTouch = getMultiTouchCenter(e)

touchDist = getMultiTouchPos(e)
app.canvas.pointer.isDown = false
canvas.pointer.isDown = false
}
}
},
Expand Down Expand Up @@ -107,52 +155,22 @@ export const simpleTouchSupportExtension: ComfyExtension = {
}
}
if (e.touches.length === 2 && lastTouch && !e.ctrlKey && !e.shiftKey) {
const canvas = canvasOrReport('touchmove', e)
if (!canvas) return
e.preventDefault()
app.canvas.pointer.isDown = false
canvas.pointer.isDown = false
touchGesture.zooming = true

LiteGraph.closeAllContextMenus(window)
app.canvas.search_box?.close()
canvas.search_box?.close()
const newTouchDist = getMultiTouchPos(e)

const center = getMultiTouchCenter(e)

if (lastScale === null || touchDist === null) return
let scale = (lastScale * newTouchDist) / touchDist

const newX = (center.clientX - lastTouch.clientX) / scale
const newY = (center.clientY - lastTouch.clientY) / scale

if (scale < app.canvas.ds.min_scale) {
scale = app.canvas.ds.min_scale
} else if (scale > app.canvas.ds.max_scale) {
scale = app.canvas.ds.max_scale
}

const oldScale = app.canvas.ds.scale

app.canvas.ds.scale = scale

if (Math.abs(app.canvas.ds.scale - 1) < 0.01) {
app.canvas.ds.scale = 1
}

const newScale = app.canvas.ds.scale

const convertScaleToOffset = (scale: number) => [
center.clientX / scale - app.canvas.ds.offset[0],
center.clientY / scale - app.canvas.ds.offset[1]
]
const oldCenter = convertScaleToOffset(oldScale)
const newCenter = convertScaleToOffset(newScale)

app.canvas.ds.offset[0] += newX + newCenter[0] - oldCenter[0]
app.canvas.ds.offset[1] += newY + newCenter[1] - oldCenter[1]

lastTouch.clientX = center.clientX
lastTouch.clientY = center.clientY
const scale = (lastScale * newTouchDist) / touchDist

app.canvas.setDirty(true, true)
zoomCanvasAroundPinch(canvas, scale, center, lastTouch)
}
},
true
Expand Down
71 changes: 71 additions & 0 deletions src/extensions/core/touchCanvasAccess.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import { fromPartial } from '@total-typescript/shoehorn'
import { describe, expect, it, vi } from 'vitest'
import { shallowRef } from 'vue'

import type { LGraphCanvas } from '@/lib/litegraph/src/litegraph'
import { bootstrapTracer } from '@/platform/telemetry/perf/bootstrapTracer'
import { reportError } from '@/platform/telemetry/reportError'
import { app } from '@/scripts/app'

import { canvasOrReport } from './touchCanvasAccess'

vi.mock(import('@/platform/telemetry/reportError'))

const canvasEl = vi.hoisted(() => document.createElement('canvas'))

vi.mock(import('@/scripts/app'), () => ({
app: fromPartial<typeof app>({
canvasElRef: shallowRef(canvasEl),
get canvasOrUndefined(): LGraphCanvas | undefined {
return undefined
}
})
}))

describe('canvasOrReport', () => {
it('returns the canvas without reporting when one exists', () => {
const canvas = fromPartial<LGraphCanvas>({})
vi.spyOn(app, 'canvasOrUndefined', 'get').mockReturnValue(canvas)

expect(canvasOrReport('has-canvas', new Event('touchstart'))).toBe(canvas)
expect(reportError).not.toHaveBeenCalled()
})

it('reports why the canvas is missing, once per entry point', () => {
document.body.append(canvasEl)
bootstrapTracer.complete('failed')
const press = new PointerEvent('pointerdown', {
pointerType: 'touch',
isPrimary: true
})

expect(canvasOrReport('press', press)).toBeUndefined()
canvasOrReport('press', press)

expect(reportError).toHaveBeenCalledExactlyOnceWith(
expect.any(Error),
expect.objectContaining({
errorType: 'error_handling_touch_input_missing_canvas',
context: {
entryPoint: 'press',
eventType: 'pointerdown',
pointerType: 'touch',
isPrimary: true,
documentReadyState: document.readyState,
bootstrapOutcome: 'failed',
hasWindowApp: false,
hasCanvasEl: true,
canvasElConnected: true,
targetIsCanvasEl: false
}
})
)
})

it('reports each entry point separately', () => {
canvasOrReport('pinch-start', new Event('touchstart'))
canvasOrReport('pinch-move', new Event('touchmove'))

expect(reportError).toHaveBeenCalledTimes(2)
})
})
37 changes: 37 additions & 0 deletions src/extensions/core/touchCanvasAccess.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { bootstrapTracer } from '@/platform/telemetry/perf/bootstrapTracer'
import { reportError } from '@/platform/telemetry/reportError'
import { app } from '@/scripts/app'

const reportedEntryPoints = new Set<string>()

function reportMissingCanvas(entryPoint: string, e: Event) {
if (reportedEntryPoints.has(entryPoint)) return
reportedEntryPoints.add(entryPoint)

const canvasEl = app.canvasElRef.value
const pointer = e instanceof PointerEvent ? e : undefined
reportError(new Error('Canvas unavailable while handling touch input'), {
errorType: 'error_handling_touch_input_missing_canvas',
surface: 'graph',
level: 'warning',
tags: { feature_area: 'extensions', operation: entryPoint },
context: {
entryPoint,
eventType: e.type,
pointerType: pointer?.pointerType,
isPrimary: pointer?.isPrimary,
documentReadyState: document.readyState,
bootstrapOutcome: bootstrapTracer.outcome,
hasWindowApp: window.app !== undefined,
hasCanvasEl: canvasEl !== undefined,
canvasElConnected: canvasEl?.isConnected,
targetIsCanvasEl: canvasEl !== undefined && e.target === canvasEl
}
})
}

export function canvasOrReport(entryPoint: string, e: Event) {
const canvas = app.canvasOrUndefined
if (!canvas) reportMissingCanvas(entryPoint, e)
return canvas
}
5 changes: 3 additions & 2 deletions src/extensions/core/touchPointerGuards.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { LGraphCanvas } from '@/lib/litegraph/src/litegraph'

import { app } from '@/scripts/app'
import { canvasOrReport } from './touchCanvasAccess'

/** Touch gesture state shared by the canvas pointer guards and the touch listeners. */
export const touchGesture = { count: 0, zooming: false }
Expand All @@ -26,7 +26,8 @@ export function installTouchPointerGuards() {
if (touchGesture.zooming || touchGesture.count) {
return
}
app.canvas.pointer.isDown = false
canvasOrReport('processMouseDown', e)
this.pointer.isDown = false
return processMouseDown.apply(this, [e])
}

Expand Down
43 changes: 43 additions & 0 deletions src/lib/litegraph/src/LGraphCanvas.startRendering.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { describe, expect, it, vi } from 'vitest'

import { LGraph, LGraphCanvas } from '@/lib/litegraph/src/litegraph'
import { reportError } from '@/platform/telemetry/reportError'
import { createTestCanvasElement } from '@/utils/__tests__/canvasTestUtils'

vi.mock(import('@/platform/telemetry/reportError'))

describe('LGraphCanvas.startRendering', () => {
it('finishes constructing and reports when the first frame throws', () => {
const drawFailure = new TypeError('drawImage is patched')
vi.spyOn(LGraphCanvas.prototype, 'draw').mockImplementation(() => {
throw drawFailure
})

const canvas = new LGraphCanvas(createTestCanvasElement(), new LGraph(), {
skip_events: true
})

expect(canvas.is_rendering).toBe(false)
expect(reportError).toHaveBeenCalledExactlyOnceWith(
drawFailure,
expect.objectContaining({
errorType: 'error_rendering_canvas_initial_frame'
})
)
})

it('keeps scheduling frames when the first frame succeeds', () => {
const requestAnimationFrame = vi
.spyOn(window, 'requestAnimationFrame')
.mockReturnValue(0)
vi.spyOn(LGraphCanvas.prototype, 'draw').mockImplementation(() => {})

const canvas = new LGraphCanvas(createTestCanvasElement(), new LGraph(), {
skip_events: true
})

expect(canvas.is_rendering).toBe(true)
expect(requestAnimationFrame).toHaveBeenCalledOnce()
expect(reportError).not.toHaveBeenCalled()
})
})
Loading
Loading