From fb5a3f23cf88d2364a1045bae332a8c24d9b984e Mon Sep 17 00:00:00 2001 From: Fourier Date: Tue, 28 Apr 2026 19:09:01 +0800 Subject: [PATCH] feat: add click-xy and eval-val for robust UI automation (#24) Addresses issue #24 lessons from browser-harness. - neo click-xy : bypass a11y tree, click at absolute viewport coordinates via CDP Input.dispatchMouseEvent. Fixes combobox options, shadow DOM, portals, cross-origin iframes where resolveRef fails. - neo eval-val : set input/textarea/select value using the framework-safe native setter pattern. Works with React/Vue/Angular controlled inputs that ignore naive '.value = x'. Dispatches input + change events with bubbles:true. - Extract buildClickXyEvents() and buildEvalValExpression() as pure helpers + unit tests (+6 tests, 173 passed). - Update help text (header comment + runtime usage). --- tools/neo.cjs | 90 ++++++++++++++++++++++++++++++++++++++++++++++ tools/neo.test.cjs | 77 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) diff --git a/tools/neo.cjs b/tools/neo.cjs index ce5ecc0..c780ca1 100755 --- a/tools/neo.cjs +++ b/tools/neo.cjs @@ -39,7 +39,9 @@ // neo cookies clear [domain] Clear cookies // neo snapshot [-i] [-C] [--json] [--diff] Snapshot a11y tree with compact ref mapping // neo click [--new-tab] Click element by ref +// neo click-xy Click at absolute viewport coordinates (bypasses a11y tree) // neo fill "text" Clear then fill element by ref +// neo eval-val Set input value via framework-safe native setter // neo type "text" Type text without clearing // neo press Press keyboard key (supports Ctrl+a) // neo hover Hover over element by ref @@ -1523,6 +1525,39 @@ const PRESS_KEY_MAP = { pagedown: { key: 'PageDown', code: 'PageDown' }, }; +function buildClickXyEvents(x, y) { + const nx = Number(x); + const ny = Number(y); + if (!Number.isFinite(nx) || !Number.isFinite(ny)) { + throw new Error('click-xy: x and y must be finite numbers'); + } + const base = { x: nx, y: ny, button: 'left', clickCount: 1 }; + return [ + { ...base, type: 'mousePressed' }, + { ...base, type: 'mouseReleased' }, + ]; +} + +function buildEvalValExpression(selector, value) { + const selJson = JSON.stringify(String(selector)); + const valJson = JSON.stringify(String(value == null ? '' : value)); + return `(function(){ + var el = document.querySelector(${selJson}); + if (!el) return { ok: false, error: 'not_found' }; + var tag = (el.tagName || '').toLowerCase(); + var proto; + if (tag === 'textarea') proto = HTMLTextAreaElement.prototype; + else if (tag === 'select') proto = HTMLSelectElement.prototype; + else proto = HTMLInputElement.prototype; + var setter = Object.getOwnPropertyDescriptor(proto, 'value'); + if (!setter || typeof setter.set !== 'function') return { ok: false, error: 'no_setter' }; + setter.set.call(el, ${valJson}); + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + return { ok: true, tag: tag }; +})()`; +} + function parsePressKey(rawKey) { const input = String(rawKey || '').trim(); if (!input) return null; @@ -4581,6 +4616,31 @@ commands.click = async function(args, context = {}) { console.log(`Clicked ${ref}`); }; +// neo click-xy +// Bypass the a11y tree entirely and click at absolute viewport coordinates via +// CDP Input.dispatchMouseEvent. Useful when resolveRef fails (stale objectId, +// portal/popover options, elements inside shadow DOM / cross-origin iframes). +commands['click-xy'] = async function(args, context = {}) { + const { positional } = parseArgs(args || []); + if (positional.length !== 2) { + console.error('Usage: neo click-xy '); + process.exit(1); + } + let events; + try { + events = buildClickXyEvents(positional[0], positional[1]); + } catch (err) { + console.error(`Usage: neo click-xy (${err && err.message ? err.message : String(err)})`); + process.exit(1); + } + + const sessionName = context.sessionName || DEFAULT_SESSION_NAME; + const pageWsUrl = getSessionPageWsUrl(sessionName); + await cdpSend(pageWsUrl, 'Input.dispatchMouseEvent', events[0]); + await cdpSend(pageWsUrl, 'Input.dispatchMouseEvent', events[1]); + console.log(`Clicked (${events[0].x}, ${events[0].y})`); +}; + // neo fill "text" commands.fill = async function(args, context = {}) { const { positional } = parseArgs(args || []); @@ -4624,6 +4684,34 @@ commands.fill = async function(args, context = {}) { console.log(`Filled ${ref}`); }; +// neo eval-val +// Set an input/textarea/select value using the framework-safe native setter +// pattern (React/Vue/Angular controlled inputs ignore naive .value = x). +commands['eval-val'] = async function(args, context = {}) { + const { positional } = parseArgs(args || []); + const selector = positional[0]; + const value = positional.length > 1 ? positional.slice(1).join(' ') : null; + if (!selector || value === null) { + console.error('Usage: neo eval-val '); + process.exit(1); + } + + const sessionName = context.sessionName || DEFAULT_SESSION_NAME; + const pageWsUrl = getSessionPageWsUrl(sessionName); + const expression = buildEvalValExpression(selector, value); + const res = await cdpSend(pageWsUrl, 'Runtime.evaluate', { + expression, + returnByValue: true, + }); + const out = res && res.result && res.result.value; + if (!out || !out.ok) { + const reason = out && out.error ? out.error : 'unknown'; + console.error(`eval-val failed (${reason}) for selector: ${selector}`); + process.exit(1); + } + console.log(`Set value on ${selector}`); +}; + // neo type "text" commands.type = async function(args, context = {}) { const { positional } = parseArgs(args || []); @@ -7510,7 +7598,9 @@ Commands: neo cookies list|export|import|clear Manage browser cookies in the active session neo snapshot [-i] [-C] [--json] [--diff] Snapshot a11y tree with compact refs neo click [--new-tab] Click element by ref + neo click-xy Click at absolute viewport coordinates (bypasses a11y tree) neo fill "text" Clear then fill element by ref + neo eval-val Set input value via framework-safe native setter neo type "text" Type text without clearing neo press Press keyboard key (supports Ctrl+a) neo hover Hover over element by ref diff --git a/tools/neo.test.cjs b/tools/neo.test.cjs index 2c15ef1..195fc87 100644 --- a/tools/neo.test.cjs +++ b/tools/neo.test.cjs @@ -992,6 +992,39 @@ const PRESS_KEY_MAP = { pagedown: { key: 'PageDown', code: 'PageDown' }, }; +function buildClickXyEvents(x, y) { + const nx = Number(x); + const ny = Number(y); + if (!Number.isFinite(nx) || !Number.isFinite(ny)) { + throw new Error('click-xy: x and y must be finite numbers'); + } + const base = { x: nx, y: ny, button: 'left', clickCount: 1 }; + return [ + { ...base, type: 'mousePressed' }, + { ...base, type: 'mouseReleased' }, + ]; +} + +function buildEvalValExpression(selector, value) { + const selJson = JSON.stringify(String(selector)); + const valJson = JSON.stringify(String(value == null ? '' : value)); + return `(function(){ + var el = document.querySelector(${selJson}); + if (!el) return { ok: false, error: 'not_found' }; + var tag = (el.tagName || '').toLowerCase(); + var proto; + if (tag === 'textarea') proto = HTMLTextAreaElement.prototype; + else if (tag === 'select') proto = HTMLSelectElement.prototype; + else proto = HTMLInputElement.prototype; + var setter = Object.getOwnPropertyDescriptor(proto, 'value'); + if (!setter || typeof setter.set !== 'function') return { ok: false, error: 'no_setter' }; + setter.set.call(el, ${valJson}); + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + return { ok: true, tag: tag }; +})()`; +} + function parsePressKey(rawKey) { const input = String(rawKey || '').trim(); if (!input) return null; @@ -2760,6 +2793,50 @@ test('diff reports no changes for identical snapshots', () => { assert.strictEqual(changed.length, 0); }); +test('buildClickXyEvents produces mousePressed + mouseReleased at given coords', () => { + const events = buildClickXyEvents(100, 200); + assert.strictEqual(events.length, 2); + assert.deepStrictEqual(events[0], { x: 100, y: 200, button: 'left', clickCount: 1, type: 'mousePressed' }); + assert.deepStrictEqual(events[1], { x: 100, y: 200, button: 'left', clickCount: 1, type: 'mouseReleased' }); +}); + +test('buildClickXyEvents accepts numeric strings and floats', () => { + const events = buildClickXyEvents('44.5', '88'); + assert.strictEqual(events[0].x, 44.5); + assert.strictEqual(events[0].y, 88); + assert.strictEqual(events[1].type, 'mouseReleased'); +}); + +test('buildClickXyEvents rejects non-finite coordinates', () => { + assert.throws(() => buildClickXyEvents('abc', 10), /finite numbers/); + assert.throws(() => buildClickXyEvents(10, NaN), /finite numbers/); + assert.throws(() => buildClickXyEvents(undefined, undefined), /finite numbers/); +}); + +test('buildEvalValExpression JSON-escapes selector and value safely', () => { + const expr = buildEvalValExpression('#email', 'user@test.com'); + assert.ok(expr.includes('"#email"'), 'selector JSON-quoted'); + assert.ok(expr.includes('"user@test.com"'), 'value JSON-quoted'); + assert.ok(expr.includes('querySelector'), 'uses querySelector'); + assert.ok(expr.includes("Object.getOwnPropertyDescriptor"), 'uses native setter pattern'); + assert.ok(expr.includes("new Event('input'"), 'dispatches input'); + assert.ok(expr.includes("new Event('change'"), 'dispatches change'); + assert.ok(expr.includes('HTMLInputElement.prototype'), 'includes input proto'); + assert.ok(expr.includes('HTMLTextAreaElement.prototype'), 'includes textarea proto'); +}); + +test('buildEvalValExpression escapes quotes and backslashes in value', () => { + const expr = buildEvalValExpression('input[name="q"]', 'hello "world" \\n'); + // JSON.stringify should turn the value into a safely-quoted JS string literal + assert.ok(expr.includes(JSON.stringify('input[name="q"]')), 'selector escaped'); + assert.ok(expr.includes(JSON.stringify('hello "world" \\n')), 'value escaped'); +}); + +test('buildEvalValExpression coerces null/undefined value to empty string', () => { + const expr = buildEvalValExpression('#x', null); + assert.ok(expr.includes('""')); +}); + Promise.all(pendingTests) .finally(() => { resetSessionFile();