From 4966964568b5251ce64c96e7df71049e22e8bc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Po=C5=99=C3=ADzek?= <33246262+Rhiz3K@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:47:11 +0100 Subject: [PATCH 1/9] fix: pan VNC with two fingers and show a soft keyboard on mobile Two-finger moves on VNC/ARD no longer only pinch-zoom: a stable spread pans the overflow container, and a pinch zooms around the fingers. preventDefault runs only after the gesture is classified. Mobile browsers never showed an on-screen keyboard because input was bound to a display div. A toolbar button on touch devices now focuses a hidden textarea and forwards keys to the session. --- locales/en.json | 9 +- locales/translations-lock.json | 3 + src/frontend/GuacamoleDisplay.tsx | 211 ++++++++++++++++-- src/frontend/GuacamoleToolbar.tsx | 30 +++ src/frontend/guacamole-soft-keyboard.ts | 53 +++++ src/frontend/guacamole-touch-gestures.ts | 108 +++++++++ src/frontend/toolbar-items.ts | 9 + tests/frontend/GuacamoleToolbar.test.tsx | 35 +++ .../frontend/guacamole-soft-keyboard.test.ts | 30 +++ .../frontend/guacamole-touch-gestures.test.ts | 124 ++++++++++ 10 files changed, 593 insertions(+), 19 deletions(-) create mode 100644 src/frontend/guacamole-soft-keyboard.ts create mode 100644 src/frontend/guacamole-touch-gestures.ts create mode 100644 tests/frontend/guacamole-soft-keyboard.test.ts create mode 100644 tests/frontend/guacamole-touch-gestures.test.ts diff --git a/locales/en.json b/locales/en.json index 23ed82f..76f6fc0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -84,7 +84,8 @@ "spotlight": "Spotlight", "logoff": "Sign out", "record": "Record", - "monitors": "Monitors" + "monitors": "Monitors", + "softKeyboard": "Keyboard" }, "hints": { "files": "Shows once the RDP drive is on", @@ -92,7 +93,8 @@ "modifiers": "Ctrl, Alt, Shift and the system key stay held until the next key", "logoff": "Signs out of Windows by running logoff from the Run box. The session must be open and showing the desktop.", "record": "Starts or stops a recording of this session. Needs the Session Recording plugin and the built-in engine.", - "monitors": "Switch between the remote monitors" + "monitors": "Switch between the remote monitors", + "softKeyboard": "Shows the on-screen keyboard on a phone or tablet" }, "option": "Option", "logoffConfirm": "Sign out of this Windows session? Unsaved work in it is lost.", @@ -103,7 +105,8 @@ "recordFailed": "Could not change the recording", "recordRequired": "An admin made recording required on this host", "recordRequiredLabel": "Recording (required)", - "monitor": "Monitor {{number}}" + "monitor": "Monitor {{number}}", + "showKeyboard": "Show keyboard" }, "disabled": "Remote Desktop is turned off. An admin can turn it on in Settings.", "rustdesk": { diff --git a/locales/translations-lock.json b/locales/translations-lock.json index adf5dd3..4e4ca44 100644 --- a/locales/translations-lock.json +++ b/locales/translations-lock.json @@ -63,12 +63,14 @@ "remoteDesktop.toolbar.zoomOut": "a4ae4b24a1f5", "remoteDesktop.toolbar.resetZoom": "e5737972ceac", "remoteDesktop.toolbar.fitToScreen": "0b3d8edb3a20", + "remoteDesktop.toolbar.showKeyboard": "2ea8fecfd82c", "remoteDesktop.toolbar.switchToTrackpad": "b2c47be4b034", "remoteDesktop.toolbar.switchToTouch": "91e3891f8c27", "remoteDesktop.toolbar.label": "d9b936543cb5", "remoteDesktop.toolbar.more": "4bab2d8fe13f", "remoteDesktop.toolbar.items.files": "6ce6c512ea43", "remoteDesktop.toolbar.items.touchMode": "441936f06779", + "remoteDesktop.toolbar.items.softKeyboard": "6662c40bac78", "remoteDesktop.toolbar.items.zoom": "9b3cbed5c490", "remoteDesktop.toolbar.items.lock": "6e2b667afdd9", "remoteDesktop.toolbar.items.metaKey": "6bc3c061ee45", @@ -82,6 +84,7 @@ "remoteDesktop.toolbar.items.monitors": "08cc50680045", "remoteDesktop.toolbar.hints.files": "56aa3b78d694", "remoteDesktop.toolbar.hints.touchMode": "6b98f8b9d5ad", + "remoteDesktop.toolbar.hints.softKeyboard": "fe5277aafc1a", "remoteDesktop.toolbar.hints.modifiers": "0b2572d18fc4", "remoteDesktop.toolbar.hints.logoff": "9bb352bbb4bd", "remoteDesktop.toolbar.hints.record": "644f2115aa8f", diff --git a/src/frontend/GuacamoleDisplay.tsx b/src/frontend/GuacamoleDisplay.tsx index d535331..be66662 100644 --- a/src/frontend/GuacamoleDisplay.tsx +++ b/src/frontend/GuacamoleDisplay.tsx @@ -56,6 +56,18 @@ import { type GuacamoleFileStreamClient, } from "./guacamole-filesystem.ts"; import { clampGuacamoleZoom, stepGuacamoleZoom } from "./guacamole-zoom.ts"; +import { + classifyTwoFingerMove, + nextPinchZoom, + scrollAfterPan, + scrollAfterZoom, + touchDistance, + touchMidpoint, +} from "./guacamole-touch-gestures.ts"; +import { + specialKeysymFromKeyboardEvent, + textToKeysyms, +} from "./guacamole-soft-keyboard.ts"; import { createRdpReadyGate } from "./guacamole-ready.ts"; import { remoteKeysym, @@ -98,6 +110,10 @@ export interface GuacamoleDisplayHandle { zoomIn: () => number; zoomOut: () => number; resetZoom: () => number; + /** Focuses the hidden textarea so a phone can show its software keyboard. */ + openSoftKeyboard: () => void; + closeSoftKeyboard: () => void; + isSoftKeyboardOpen: () => boolean; /** Shows another RustDesk monitor. */ selectDisplay: (index: number) => void; } @@ -199,6 +215,7 @@ export const GuacamoleDisplay = forwardRef< null, ); const hasKeyboardFocusRef = useRef(false); + const softKeyboardRef = useRef(null); const sessionProtocolRef = useRef( connectionConfig.protocol ?? connectionConfig.type, ); @@ -324,6 +341,17 @@ export const GuacamoleDisplay = forwardRef< zoomIn: () => applyZoom(stepGuacamoleZoom(zoomRef.current, 1)), zoomOut: () => applyZoom(stepGuacamoleZoom(zoomRef.current, -1)), resetZoom: () => applyZoom(1), + openSoftKeyboard: () => { + const input = softKeyboardRef.current; + if (!input) return; + input.focus({ preventScroll: true }); + }, + closeSoftKeyboard: () => { + softKeyboardRef.current?.blur(); + }, + isSoftKeyboardOpen: () => + softKeyboardRef.current != null && + document.activeElement === softKeyboardRef.current, })); const getWebSocketConnection = useCallback( @@ -1001,28 +1029,106 @@ export const GuacamoleDisplay = forwardRef< if (!container || !isReady || (protocol !== "vnc" && protocol !== "ard")) return; - let pinchDistance = 0; - let pinchZoom = zoomRef.current; - const distance = (touches: TouchList) => - Math.hypot( - touches[0].clientX - touches[1].clientX, - touches[0].clientY - touches[1].clientY, - ); + type Gesture = { + startDistance: number; + startZoom: number; + startScale: number; + startScrollLeft: number; + startScrollTop: number; + startMid: { x: number; y: number }; + lastMid: { x: number; y: number }; + kind: "undecided" | "pinch" | "pan"; + }; + let gesture: Gesture | null = null; + + const pair = (touches: TouchList) => { + if (touches.length !== 2) return null; + return { + a: { x: touches[0].clientX, y: touches[0].clientY }, + b: { x: touches[1].clientX, y: touches[1].clientY }, + }; + }; + const relative = (point: { x: number; y: number }, rect: DOMRect) => ({ + x: point.x - rect.left, + y: point.y - rect.top, + }); + const onTouchStart = (event: TouchEvent) => { - if (event.touches.length !== 2) return; - event.preventDefault(); - event.stopPropagation(); - pinchDistance = distance(event.touches); - pinchZoom = zoomRef.current; + const touches = pair(event.touches); + if (!touches) { + gesture = null; + return; + } + const mid = touchMidpoint(touches.a, touches.b); + gesture = { + startDistance: touchDistance(touches.a, touches.b), + startZoom: zoomRef.current, + startScale: scaleRef.current, + startScrollLeft: container.scrollLeft, + startScrollTop: container.scrollTop, + startMid: mid, + lastMid: mid, + kind: "undecided", + }; }; const onTouchMove = (event: TouchEvent) => { - if (event.touches.length !== 2 || pinchDistance === 0) return; + if (!gesture) return; + const touches = pair(event.touches); + if (!touches) return; + const mid = touchMidpoint(touches.a, touches.b); + const distance = touchDistance(touches.a, touches.b); + const moved = + Math.hypot(mid.x - gesture.startMid.x, mid.y - gesture.startMid.y) > + 0.5; + const kind = classifyTwoFingerMove({ + startDistance: gesture.startDistance, + currentDistance: distance, + current: gesture.kind, + moved, + }); + if (kind === "undecided") { + gesture.lastMid = mid; + return; + } event.preventDefault(); event.stopPropagation(); - applyZoom(pinchZoom * (distance(event.touches) / pinchDistance)); + gesture.kind = kind; + if (kind === "pinch") { + const newZoom = nextPinchZoom( + gesture.startZoom, + gesture.startDistance, + distance, + ); + applyZoom(newZoom); + const rect = container.getBoundingClientRect(); + const from = relative(gesture.startMid, rect); + const to = relative(mid, rect); + const next = scrollAfterZoom({ + scrollLeft: gesture.startScrollLeft, + scrollTop: gesture.startScrollTop, + fromX: from.x, + fromY: from.y, + pointX: to.x, + pointY: to.y, + oldScale: gesture.startScale, + newScale: scaleRef.current, + }); + container.scrollLeft = next.scrollLeft; + container.scrollTop = next.scrollTop; + } else { + const next = scrollAfterPan({ + scrollLeft: container.scrollLeft, + scrollTop: container.scrollTop, + dx: mid.x - gesture.lastMid.x, + dy: mid.y - gesture.lastMid.y, + }); + container.scrollLeft = next.scrollLeft; + container.scrollTop = next.scrollTop; + } + gesture.lastMid = mid; }; - const onTouchEnd = () => { - pinchDistance = 0; + const onTouchEnd = (event: TouchEvent) => { + if (event.touches.length !== 2) gesture = null; }; container.addEventListener("touchstart", onTouchStart, { @@ -1034,10 +1140,12 @@ export const GuacamoleDisplay = forwardRef< capture: true, }); container.addEventListener("touchend", onTouchEnd, { capture: true }); + container.addEventListener("touchcancel", onTouchEnd, { capture: true }); return () => { container.removeEventListener("touchstart", onTouchStart, true); container.removeEventListener("touchmove", onTouchMove, true); container.removeEventListener("touchend", onTouchEnd, true); + container.removeEventListener("touchcancel", onTouchEnd, true); }; }, [applyZoom, connectionConfig.protocol, connectionConfig.type, isReady]); @@ -1103,6 +1211,77 @@ export const GuacamoleDisplay = forwardRef< }} /> +