test: harness follow-ups from PR #10's review (no DNS lookups, deferred evaluateHandle, SV-25/SV-26, perf)

scripts/capture/record.mjs

@@ -219,6 +219,8 @@ function makeToolkit(app, shell, fx) {
  * while loopback, where the fixture pages are served, bypasses proxies.
  */
 const OFFLINE_PROXY = '127.0.0.1:9'
+/** And no host name resolves but localhost: Chromium looks up a link's host before anyone clicks it (PR #10 review F2). */
+const NO_LOOKUPS = '--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE localhost, EXCLUDE 127.0.0.1'
 
 /**
  * Puts every session Orbit has, and every one it makes later, on the dead proxy, and cancels any request
@@ -254,7 +256,12 @@ let saved = false
 try {
   log('launching Orbit')
   app = await electron.launch({
-    args: [...(process.platform === 'linux' ? ['--no-sandbox'] : []), `--proxy-server=http://${OFFLINE_PROXY}`, main],
+    args: [
+      ...(process.platform === 'linux' ? ['--no-sandbox'] : []),
+      `--proxy-server=http://${OFFLINE_PROXY}`,
+      NO_LOOKUPS,
+      main,
+    ],
     env: { ...process.env, ORBIT_E2E: '1', ORBIT_USER_DATA: userData },
   })
   let shell

scripts/perf/run.mjs

@@ -6,6 +6,7 @@
  * scenarios in Chromium on the same machine.
  *
  * Linux: xvfb-run -a -s "-screen 0 1440x900x24" node scripts/perf/run.mjs [--runs 5] [--out perf-results/latest.json]
+ *   [--port 3350] (the fixture server's first port; it takes the next free one up to 3399)
  */
 import { execFileSync } from 'node:child_process'
 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
@@ -22,6 +23,7 @@ const opt = (name, fallback) => {
 const RUNS = Number(opt('runs', '5'))
 const OUT = resolve(opt('out', 'perf-results/latest.json'))
 const SKIP_CHROME = args.includes('--no-chrome')
+const FIRST_PORT = Number(opt('port', '3350'))
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
 const median = (xs) => {
   const s = [...xs].sort((a, b) => a - b)
@@ -45,7 +47,7 @@ async function startFixtures() {
     res.writeHead(200, { 'content-type': 'text/html' })
     res.end(PAGE(n))
   })
-  let port = 3350
+  let port = FIRST_PORT
   await new Promise((ok, fail) => {
     server.on('error', (err) =>
       err.code === 'EADDRINUSE' && port < 3399 ? server.listen(++port, '127.0.0.1') : fail(err),
@@ -77,9 +79,18 @@ function treeRssMb(rootPid) {
 
 // ---------- Orbit helpers ----------
 const MAIN = resolve('out/main/index.js')
+/**
+ * Offline, as in the e2e tests (tests/e2e/helpers.ts): a dead local proxy, which loopback (the
+ * fixture server) bypasses, and no host name resolves but localhost. Orbit and the Chromium baseline
+ * both run with these.
+ */
+const OFFLINE = [
+  '--proxy-server=http://127.0.0.1:9',
+  '--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE localhost, EXCLUDE 127.0.0.1',
+]
 async function launchOrbit(userData) {
   const app = await electron.launch({
-    args: process.platform === 'linux' ? ['--no-sandbox', MAIN] : [MAIN],
+    args: [...(process.platform === 'linux' ? ['--no-sandbox'] : []), ...OFFLINE, MAIN],
     env: { ...process.env, ORBIT_E2E: '1', ORBIT_USER_DATA: userData },
   })
   const deadline = Date.now() + 30_000
@@ -169,9 +180,11 @@ async function orbitInteractive(fx) {
   }
   await sleep(1500)
   const rss10 = treeRssMb(pid)
-  // Put every background tab to sleep and measure what is freed.
+  // Put every background tab to sleep and measure what is freed. Between tasks, as __orbit.run does:
+  // the inspector could otherwise run this part-way through other main-process code.
   await app.evaluate(
-    (_e, ids) => {
+    async (_e, ids) => {
+      await new Promise((r) => setImmediate(r))
       for (const id of ids) globalThis.__orbit.ctx.tabs.sleep(id)
     },
     tabs.slice(0, -1),
@@ -214,8 +227,10 @@ async function orbitInteractive(fx) {
     const t0 = Date.now()
     const inMain = await app.evaluate(async (_e, id) => {
       const o = globalThis.__orbit
+      // Start between tasks, as __orbit.run does, but time the command alone, not that hop.
+      await new Promise((r) => setImmediate(r))
       const s = performance.now()
-      await o.run('tabs.activate', { tabId: id }, o.windowIds()[0])
+      await o.ctx.commands.run('tabs.activate', { tabId: id }, { windowId: o.windowIds()[0] })
       const visible = o.ctx.tabs.get(id)?.getVisible() ?? false
       return { ms: performance.now() - s, visible }
     }, target)
@@ -420,14 +435,14 @@ async function chromeBaseline(fx) {
   for (let i = 0; i < RUNS; i++) {
     const dir = mkdtempSync(join(tmpdir(), 'chrome-perf-'))
     const t0 = Date.now()
-    const ctx = await chromium.launchPersistentContext(dir, { headless: false })
+    const ctx = await chromium.launchPersistentContext(dir, { headless: false, args: OFFLINE })
     const page = ctx.pages()[0] ?? (await ctx.newPage())
     await page.goto(fx.url(`c${i}`))
     coldStart.push(Date.now() - t0)
     await ctx.close()
     rmSync(dir, { recursive: true, force: true })
   }
-  const server = await chromium.launchServer({ headless: false })
+  const server = await chromium.launchServer({ headless: false, args: OFFLINE })
   const pid = server.process().pid
   const browser = await chromium.connect(server.wsEndpoint())
   const ctx = await browser.newContext()

tests/e2e/core.spec.ts

@@ -407,6 +407,36 @@ test("[TB-22] a quit Orbit cancels fails the test: Playwright's own quit doesn't
   await orbit.app.close()
 })
 
+test('[TB-22] main-process evaluates and handles run between tasks, method shorthands too (PR #10 review F4)', async () => {
+  // Main stays busy in timers, so an evaluate through the inspector lands inside one unless it
+  // waits for a task of its own (helpers.ts, evaluateBetweenTasks).
+  await orbit.app.evaluate(() => {
+    const g = globalThis as unknown as { inTimer?: boolean }
+    for (let i = 0; i < 10; i++) {
+      setInterval(() => {
+        g.inTimer = true
+        const end = Date.now() + 10
+        while (Date.now() < end) {
+          // busy
+        }
+        g.inTimer = false
+      }, 0)
+    }
+  })
+  const inTimer = () => (globalThis as unknown as { inTimer?: boolean }).inTimer === true
+  for (let i = 0; i < 5; i++) {
+    expect(await orbit.app.evaluate(inTimer)).toBe(false)
+    expect(await (await orbit.app.evaluateHandle(inTimer)).jsonValue()).toBe(false)
+  }
+  // A method shorthand is a function too, as in Playwright's own evaluate.
+  const calc = {
+    add(_electron: unknown, n: number) {
+      return n + 1
+    },
+  }
+  expect(await orbit.app.evaluate(calc.add, 1)).toBe(2)
+})
+
 test('[TB-02] opening tabs keeps every UI mirror in step without resyncs (review M2, F4)', async () => {
   const errors: string[] = []
   orbit.shell.on('console', (m) => m.type() === 'error' && errors.push(m.text()))

tests/e2e/helpers.ts

@@ -31,9 +31,35 @@ export interface Orbit {
 /**
  * Tests stay offline. Orbit runs behind a dead local proxy, which loopback (where the fixture pages
  * are served) bypasses, so a search or a link to a real site gets its URL but loads nothing.
+ * No host name resolves either, except localhost: Chromium looks up a link's host before anyone
+ * clicks it, and that lookup would leave the machine (PR #10 review F2).
  */
 const DEAD_PROXY = '127.0.0.1:9'
-const OFFLINE = `--proxy-server=http://${DEAD_PROXY}`
+const RESOLVER_RULES = '--host-resolver-rules='
+/**
+ * These rules don't stop an mDNS lookup: Chromium still multicasts the mapped name on the local
+ * network. Nothing in Orbit or the tests asks for one (`resolveHost` with `source: 'mdns'`).
+ */
+const NO_LOOKUPS = 'MAP * ~NOTFOUND, EXCLUDE localhost, EXCLUDE 127.0.0.1'
+/** Hosts a test may exclude from the rules: Chromium or the hosts file answers them, on this machine. */
+const LOOPBACK_HOST = /^(localhost|127\.0\.0\.1|::1|[\w.-]+\.localhost)$/
+
+/**
+ * The switches that keep Orbit offline. A test's own resolver rules go first, so its MAP rules win
+ * (the first MAP that matches a host does), and every other host still fails to resolve. Chromium
+ * checks every EXCLUDE before any MAP, wherever it is in the list, and an excluded host is looked up
+ * for real, so a test may only exclude loopback hosts (PR #19 review F3).
+ */
+export function offlineArgs(args: string[] = []): string[] {
+  const own = args.filter((a) => a.startsWith(RESOLVER_RULES)).map((a) => a.slice(RESOLVER_RULES.length))
+  for (const rule of own.flatMap((r) => r.split(','))) {
+    const host = /^\s*EXCLUDE\s+(\S+)\s*$/i.exec(rule)?.[1]
+    if (host !== undefined && !LOOPBACK_HOST.test(host)) {
+      throw new Error(`a test's resolver rules exclude ${host}, which would be looked up for real`)
+    }
+  }
+  return [`--proxy-server=http://${DEAD_PROXY}`, `${RESOLVER_RULES}${[...own, NO_LOOKUPS].join(', ')}`]
+}
 
 /**
  * Launches the built app (out/) with an isolated profile directory. `args` are extra Chromium
@@ -47,7 +73,9 @@ export async function launchOrbit(
   for (const old of quitAlone) await old.close()
   const userData = opts.userData ?? mkdtempSync(join(tmpdir(), 'orbit-e2e-'))
   const motion = opts.reducedMotion ? '--force-prefers-reduced-motion' : '--force-prefers-no-reduced-motion'
-  const args = [OFFLINE, motion, ...(opts.args ?? []), resolve('out/main/index.js')]
+  const extra = opts.args ?? []
+  const rest = extra.filter((a) => !a.startsWith(RESOLVER_RULES))
+  const args = [...offlineArgs(extra), motion, ...rest, resolve('out/main/index.js')]
   // Linux CI/dev boxes often lack the setuid sandbox helper; macOS runs fully sandboxed.
   if (process.platform === 'linux') args.unshift('--no-sandbox')
   const app = await electron.launch({
@@ -86,7 +114,9 @@ async function waitOffline(app: ElectronApplication): Promise<void> {
 /**
  * Playwright runs main-process evaluates through the inspector, which can interrupt main-process
  * code part-way, even inside a store update. A change made there is lost when that update finishes:
- * a background tab a test had just created vanished. Each evaluate starts in a task of its own instead.
+ * a background tab a test had just created vanished. Each evaluate and evaluateHandle starts in a
+ * task of its own instead. A handle's own `evaluate` (such as `app.browserWindow(page)`'s) still
+ * runs by interrupt, so tests reach main-process objects through `app.evaluate`.
  */
 export function evaluateBetweenTasks(app: ElectronApplication): void {
   const evaluate = app.evaluate.bind(app)
@@ -97,9 +127,34 @@ export function evaluateBetweenTasks(app: ElectronApplication): void {
         await new Promise((resolve) => setImmediate(resolve))
         return new Function(`return (${source})`)()(electron, a)
       },
-      [fn.toString(), arg] as const,
+      [functionSource(fn), arg] as const,
     )
   }) as ElectronApplication['evaluate']
+  const evaluateHandle = app.evaluateHandle.bind(app)
+  app.evaluateHandle = ((fn: unknown, arg?: unknown) => {
+    if (typeof fn !== 'function') return evaluateHandle(fn as never, arg as never)
+    return evaluateHandle(
+      async (electron, [source, a]) => {
+        await new Promise((resolve) => setImmediate(resolve))
+        return new Function(`return (${source})`)()(electron, a)
+      },
+      [functionSource(fn), arg] as const,
+    )
+  }) as ElectronApplication['evaluateHandle']
+}
+
+/**
+ * A function's source as an expression. A method shorthand (`m(a) {…}`) becomes a function, as in
+ * Playwright's own evaluate (PR #10 review F4).
+ */
+function functionSource(fn: { toString(): string }): string {
+  const source = fn.toString()
+  try {
+    new Function(`(${source})`)
+    return source
+  } catch {
+    return source.startsWith('async ') ? `async function ${source.slice('async '.length)}` : `function ${source}`
+  }
 }
 
 /**

tests/e2e/settings-support.ts

@@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'
 import { join, resolve } from 'node:path'
 import { type ElectronApplication, _electron as electron, expect, type Page } from '@playwright/test'
 import type { MainContext } from '../../src/main/context'
-import { checkQuit, evaluateBetweenTasks, run } from './helpers'
+import { checkQuit, evaluateBetweenTasks, offlineArgs, run } from './helpers'
 
 /** The main process's test handle (src/main/test-hooks.ts), for app.evaluate() bodies. */
 export type OrbitGlobal = { __orbit: { ctx: MainContext; windowIds: () => string[] } }
@@ -91,15 +91,15 @@ export interface FirstRun {
 }
 
 /**
- * Starts Orbit as launchOrbit does (offline, behind the same dead proxy; isolated profile), and also
- * records every network request Chromium makes from the first moment, for OB-02.
+ * Starts Orbit as launchOrbit does (offline: the same dead proxy and resolver rules; isolated
+ * profile), and also records every network request Chromium makes from the first moment, for OB-02.
  */
 export async function launchFirstRun(
   opts: { userData?: string; env?: Record<string, string> } = {},
 ): Promise<FirstRun> {
   const userData = opts.userData ?? mkdtempSync(join(tmpdir(), 'orbit-e2e-'))
   const netLog = join(mkdtempSync(join(tmpdir(), 'orbit-netlog-')), 'netlog.json')
-  const args = ['--proxy-server=http://127.0.0.1:9', `--log-net-log=${netLog}`, resolve('out/main/index.js')]
+  const args = [...offlineArgs(), `--log-net-log=${netLog}`, resolve('out/main/index.js')]
   if (process.platform === 'linux') args.unshift('--no-sandbox')
   const app = await electron.launch({
     args,

tests/e2e/smoke.spec.ts

@@ -54,6 +54,30 @@ test('opens a page from the command bar into a real web view', async () => {
   expect(Object.keys(s.tabs)).toHaveLength(1)
 })
 
+test('stays offline: no host name but localhost resolves, in any profile (PR #10 review F2)', async () => {
+  const { profiles } = await state(orbit.app)
+  const partitions = Object.values(profiles).map((p) => p.partition)
+  const lookups = await orbit.app.evaluate(async ({ session }, parts) => {
+    const sessions = [session.defaultSession, ...parts.map((p) => session.fromPartition(p))]
+    const lookUp = (s: Electron.Session, host: string) =>
+      s.resolveHost(host).then(
+        (r) => r.endpoints.map((e) => e.address).join(' '),
+        (e: Error) => e.message,
+      )
+    // An address needs no lookup, so only the rules refuse 127.0.0.2, and no query can leave the
+    // machine even when the rules are missing (a host name would then be looked up; PR #19 review F5).
+    return Promise.all(
+      sessions.map(async (s) => ({ mapped: await lookUp(s, '127.0.0.2'), local: await lookUp(s, 'localhost') })),
+    )
+  }, partitions)
+  expect(lookups.length).toBeGreaterThan(1)
+  for (const { mapped, local } of lookups) {
+    // Refused by the resolver rules at once, as every host but localhost is.
+    expect(mapped).toContain('ERR_NAME_NOT_RESOLVED')
+    expect(local).toMatch(/127\.0\.0\.1|::1/)
+  }
+})
+
 test('[FL-01] every CSS animation in the shell and the overlay names keyframes defined in that page', async () => {
   for (const view of ['shell', 'overlay'] as const) {
     const page = await viewPage(orbit.app, view)