Add a keyless browser E2E that drives the scripted mock

.github/workflows/live-e2e.yml

@@ -13,14 +13,27 @@ name: Live E2E
 # (#794). So a missing key warns, skips the six steps that need it, and leaves the run
 # green while the step summary says plainly that nothing ran and nothing was measured. A
 # real failure of the suite still fails the run: the skip is never a pass.
+#
+# The `mock` job below is the keyless half (#2565): the same bench, adapter and evidence
+# path, with DEEPSEEK_BASE_URL pointed at the scripted mock (core/dev/mock-llm.mjs). It
+# reads no secret, so it runs on every pull request that touches core/ or plugins/, not only
+# on main — and its step summary names each check it ran, so its green is visibly different
+# from the live job's skip. The `live` job is unchanged.
 
 on:
   schedule:
     - cron: "30 16 * * *" # 00:30 Asia/Shanghai daily; clear of compat's 16:00Z
   workflow_dispatch:
+  pull_request:
+    paths:
+      - "core/**"
+      - "plugins/**"
+      - "dev/live-e2e/**"
+      - ".github/workflows/live-e2e.yml"
 
+# Per ref: a pull request's mock run must not sit in the nightly's queue (or start one).
 concurrency:
-  group: live-e2e
+  group: live-e2e-${{ github.ref }}
   cancel-in-progress: false
 
 permissions:
@@ -29,6 +42,7 @@ permissions:
 jobs:
   window:
     name: window check
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
     timeout-minutes: 5
     outputs:
@@ -176,3 +190,126 @@ jobs:
           path: evidence/
           if-no-files-found: ignore
           retention-days: 14
+
+  # The keyless browser suite (#2565). Same bench, web adapter and evidence path as the live
+  # job, with DEEPSEEK_BASE_URL on the scripted mock and no secret in the environment at all:
+  # the two #204 task checks cannot run here (a scripted model answers a task prompt with a
+  # canned acknowledgement and writes no files), so this job measures the class of regression
+  # the repository had no browser-level check for — a message goes in through the composer, a
+  # reply comes out, and both survive a reload.
+  mock:
+    name: mock · ubuntu · Node 24 · scripted model
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      WEB_PORT: '3197'
+      MOCK_PORT: '8765'
+      BENCH: live-e2e-mock
+      MODEL: deepseek-official/deepseek-flash
+      LIVE_E2E_MODE: mock
+      DEEPSEEK_BASE_URL: http://127.0.0.1:8765
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1  # v7.1.0
+        with:
+          node-version: 24
+          cache: npm
+          cache-dependency-path: core/package-lock.json
+      - name: npm ci (core)
+        working-directory: core
+        run: npm ci --no-fund --no-audit
+        timeout-minutes: 5
+      # The runtime is the published package, not a source checkout: a checkout has no built
+      # client artifacts (dsh web exits with ENOENT on packages/client/*/lib/client.js), and
+      # this job must not spend ten minutes building DSH to prove a reply still renders. pnpm
+      # keeps the node_modules/.pnpm layout the adapter resolves playwright-core from.
+      - name: Install the pinned DSH runtime
+        run: |
+          DSH=$(node -p "require('./core/dev/harness/package.json').dependencies['@deepseek-ai/dsh']")
+          RUNTIME="$RUNNER_TEMP/dsh-runtime"
+          mkdir -p "$RUNTIME"
+          cd "$RUNTIME"
+          printf '{"name":"dsh-live-e2e-runtime","private":true,"packageManager":"pnpm@12.16.1"}\n' > package.json
+          # pnpm 12 refuses dependency build scripts by default; the pinned runtime's native
+          # helpers (subprocess spawn, pty, protobuf) are the same packages the live job runs.
+          printf 'dangerouslyAllowAllBuilds: true\n' > pnpm-workspace.yaml
+          corepack enable
+          corepack pnpm add "@deepseek-ai/dsh@$DSH" playwright-core@1.61.1
+          echo "DSH_HARNESS=$RUNTIME" >> "$GITHUB_ENV"
+        timeout-minutes: 8
+      # The headless shell is what the adapter launches. No --with-deps: the runner image
+      # already carries Chrome's system libraries, and an apt-get round there is the slow,
+      # flaky half (a run timed out on it); one retry covers the download itself.
+      - name: Chromium for the browser
+        run: |
+          for attempt in 1 2; do
+            node "$DSH_HARNESS/node_modules/playwright-core/cli.js" install chromium --only-shell && exit 0
+            echo "::warning::chromium install attempt $attempt failed; retrying"
+            sleep 5
+          done
+          exit 1
+        timeout-minutes: 8
+      - name: Start the scripted mock
+        run: |
+          # The bench home lives outside the checkout (#398), exactly as in the live job.
+          LIVE_E2E_HOME="$HOME/.cache/flybot-live-e2e/$BENCH"
+          echo "LIVE_E2E_HOME=$LIVE_E2E_HOME" >> "$GITHUB_ENV"
+          mkdir -p "$LIVE_E2E_HOME"
+          MOCK_LOG="$LIVE_E2E_HOME/mock.jsonl" setsid node core/dev/mock-llm.mjs \
+            </dev/null >"$LIVE_E2E_HOME/mock.out" 2>&1 & echo $! >"$LIVE_E2E_HOME/mock.pid"
+          i=0
+          until (exec 3<>"/dev/tcp/127.0.0.1/$MOCK_PORT") 2>/dev/null; do
+            i=$((i + 1))
+            [ $i -ge 30 ] && { echo "::error::the scripted mock did not start"; cat "$LIVE_E2E_HOME/mock.out"; exit 1; }
+            sleep 1
+          done
+        timeout-minutes: 2
+      - name: Start the mock bench
+        run: node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" reset
+        timeout-minutes: 8
+      # The summary carries the runner's own lines: one `ok`/`FAIL` per check it ran, the
+      # mode, and the outcome — so a green that measured nothing cannot look like this one.
+      - name: Run the keyless checks
+        id: run
+        run: |
+          set -o pipefail
+          {
+            echo "### Live E2E (mock, no key)"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+          # The fence closes and the step still fails on the runner's exit code: a summary
+          # that ends in a stray `echo` would make a red run look green.
+          status=0
+          node dev/live-e2e/run.mjs "$LIVE_E2E_HOME" "$WEB_PORT" \
+            --model "$MODEL" --harness "$DSH_HARNESS" --evidence evidence --tz Asia/Shanghai \
+            | tee -a "$GITHUB_STEP_SUMMARY" || status=$?
+          echo '```' >> "$GITHUB_STEP_SUMMARY"
+          exit $status
+      - name: Stop the bench and the mock
+        if: always()
+        run: |
+          node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" stop || true
+          if [ -f "$LIVE_E2E_HOME/mock.pid" ]; then
+            kill -TERM "-$(cat "$LIVE_E2E_HOME/mock.pid")" 2>/dev/null || true
+          fi
+      - name: Sanitized web log (diagnostics only, never uploaded raw)
+        if: failure()
+        run: tail -80 "$LIVE_E2E_HOME/web.log" | grep -v 'token=' || true
+      - name: Evidence gate
+        id: evidence-gate
+        if: always() && steps.run.outcome != 'skipped'
+        run: |
+          if [ ! -f evidence/sanitized.txt ]; then
+            echo "::error::no sanitizer-cleared evidence bundle (the run failed before the scan, or the scan found something); nothing is uploaded"
+            exit 1
+          fi
+      - name: Upload evidence (allow-list, sanitizer-stamped)
+        if: always() && steps.evidence-gate.outcome == 'success'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
+        with:
+          name: live-e2e-mock-evidence
+          path: evidence/
+          if-no-files-found: ignore
+          retention-days: 14

changelog.d/2565-keyless-browser-e2e.added.md

@@ -0,0 +1 @@
+A keyless browser suite now runs in CI: the scripted mock drives the same bench and web adapter as the real-model suite, so the web UI's message-and-reply path is measured on every pull request that touches `core/` or `plugins/` instead of only existing as a nightly that skips without a secret. The job reads no secret, its step summary names each check it ran, and the real-model `live` job is unchanged. ([#2565](https://github.com/ThinkFlowLab/FlyBot/issues/2565))

dev/live-e2e/README.md

@@ -14,8 +14,9 @@ only `adapter.mjs` knows FlyBot. Nothing here ships in a package or runs in
 | --- | --- |
 | `fixtures/orders.csv` | The fixed 5-row fixture the expected totals derive from. |
 | `scenarios.mjs` | The four checks, task texts, pure assertions (parsed JSON, SHA-256 immutability). |
+| `smoke.mjs` | The keyless checks (#2565): a message sent through the composer gets a reply, and both survive a reload. Run by `LIVE_E2E_MODE=mock`; no files, no model wording asserted. |
 | `adapter.mjs` | The FlyBot web adapter: playwright-core from the harness checkout, UI task steps, read-only observation. |
-| `bench.mjs` | A dsh web bench on this checkout with the real endpoint; refuses missing/mock keys, keeps its home outside the checkout, never touches `~/.dsh`. |
+| `bench.mjs` | A dsh web bench on this checkout. `LIVE_E2E_MODE=live` refuses a missing/mock key, `LIVE_E2E_MODE=mock` needs the loopback mock and reads no key; both keep the home outside the checkout and never touch `~/.dsh`. |
 | `run.mjs` | The runner: guards, budgets, classification, report, evidence sanitizer. |
 | `report.mjs` | Outcome contract, guard logic, the tamper rule (`integrityOutcome`), usage/classification, report assembly (pure, tested keyless). |
 | `integrity.mjs` | The grader's own hash: `treeManifest` / `changedPaths` for the tamper check, and the path test the guards use (#398). |
@@ -76,12 +77,35 @@ A run costs real tokens (the pilot used ~116K); budgets default to 180 s per che
 250K tokens per run (`LIVE_E2E_CHECK_MS`, `LIVE_E2E_QUIET_MS`, `LIVE_E2E_MAX_TOKENS`),
 with zero retries.
 
+## Run the keyless mock suite
+
+No key, no network to a model endpoint: the same bench and adapter, with the scripted mock
+from `core/dev/mock-llm.mjs` answering (`LIVE_E2E_MODE=mock` also allows the loopback base
+URL the live mode refuses). The checks are the two in `smoke.mjs`, not the four #204 task
+checks — a scripted model does not write `summary.json`.
+
+```sh
+export LIVE_E2E_MODE=mock
+export DEEPSEEK_BASE_URL=http://127.0.0.1:8765
+MOCK_PORT=8765 node core/dev/mock-llm.mjs &          # the scripted model
+node dev/live-e2e/bench.mjs live-e2e 3197 reset
+node dev/live-e2e/run.mjs "$LIVE_E2E_HOME" 3197 --model deepseek-official/deepseek-flash
+node dev/live-e2e/bench.mjs live-e2e 3197 stop
+```
+
+The runtime can be a published `@deepseek-ai/dsh` install instead of a source checkout: the
+bench only needs the `dsh` command (a source tree has no built `lib/client.js`) and the
+adapter `playwright-core` from `node_modules/.pnpm/node_modules`.
+
 ## In CI
 
 `.github/workflows/live-e2e.yml` runs the same flow nightly at 00:30 Asia/Shanghai (and
 on dispatch) on `main` only, with a window gate so a quiet `main` spends no tokens: it
 clones the harness at the tag pinned in `core/dev/harness/package.json`, starts the tap,
 resets the bench, runs the four checks and uploads the sanitizer-cleared allow-list.
+The keyless `mock` job runs the two checks above on every pull request that touches `core/`
+or `plugins/` and on the same nightly schedule, with no `secrets.*` in its environment and
+the runner's summary appended to the job's step summary ([#2565](https://github.com/ThinkFlowLab/FlyBot/issues/2565)).
 The four decisions behind this setup are recorded in
 [#204](https://github.com/ThinkFlowLab/issues/204). "Stable" means five consecutive
 scheduled runs green at one recorded configuration; until then the suite is evidence,

dev/live-e2e/adapter.mjs

@@ -151,9 +151,16 @@ export async function openFlyBotWeb({ harness, home, port, model, outDir, quietM
       await page.waitForTimeout(1200)
     },
     // Who said what is app-specific to render; for the scenarios it is enough that the
-    // task texts are still visible and that some reply beside them is.
+    // task texts are still visible and that some reply beside them is. The reading is
+    // scoped to the conversation flow, never the whole page: the sidebar's own text (Bot
+    // names, previews, group lines) would otherwise count as "a reply", and a chat that
+    // rendered nothing would still look non-empty (#2565).
     async conversationLines() {
-      const text = await page.evaluate(() => document.body.innerText)
+      const text = await page.evaluate(() => {
+        const flow = document.querySelector('[data-conversation-scroll] [data-chat-flow]')
+          ?? document.querySelector('[data-conversation-scroll]')
+        return (flow ?? document.body).innerText
+      })
       const lines = text.split('\n').map(line => line.trim()).filter(line => line.length > 0)
       return lines.map(line => ({ who: sent.some(task => task.slice(0, 24) && line.includes(task.slice(0, 24))) ? 'user' : 'app', text: line }))
     },

dev/live-e2e/bench.mjs

@@ -1,7 +1,8 @@
-// Real-model test bench for the live E2E suite (#204): a dsh web whose profile links
-// this checkout, talking to the real DeepSeek endpoint with DEEPSEEK_API_KEY from the
-// caller's environment. The mock bench stays core/dev/bench.sh; this one never starts a
-// mock and refuses to run without a real key, so the two suites cannot mix.
+// Test bench for the E2E suites: a dsh web whose profile links this checkout, talking to
+// whatever DEEPSEEK_BASE_URL names. LIVE_E2E_MODE=live (#204) is the real DeepSeek endpoint
+// with a real key from the caller's environment; LIVE_E2E_MODE=mock (#2565) is the scripted
+// mock already listening on loopback and reads no key at all. This bench never starts a
+// mock itself, and each mode refuses the other's configuration, so the two cannot mix.
 //
 //   node dev/live-e2e/bench.mjs <name> <web-port> [start|stop|reset|status]
 //
@@ -10,14 +11,16 @@
 // checkout is refused: that workspace would hold the grader (#398). Environment: DEEPSEEK_API_KEY
 // (required; fails loudly when the suite's own reading refuses it — see key.mjs),
 // DEEPSEEK_BASE_URL (optional; defaults to the official endpoint — point it at
-// dev/llm-tap.mjs to record per-call usage), DSH_HARNESS (a deepseek-harness checkout with
-// its dependencies installed).
+// dev/llm-tap.mjs to record per-call usage, or at core/dev/mock-llm.mjs in mock mode),
+// DSH_HARNESS (a deepseek-harness checkout with its dependencies installed), LIVE_E2E_MODE
+// (live, the default, or mock).
 import { spawn, spawnSync } from 'node:child_process'
 import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { dirname, join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { benchHome } from './integrity.mjs'
-import { keyProblem, keyReading } from './key.mjs'
+import { keyProblem, keyReading, MOCK_KEY } from './key.mjs'
+import { readMode } from './report.mjs'
 
 const ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url))))
 const [NAME, PORT_TEXT, ACTION = 'start'] = process.argv.slice(2)
@@ -30,9 +33,18 @@ if (WEB_PORT === 3080) die('port 3080 belongs to the real service')
 const { home: HOME_DIR, problem: homeProblem } = benchHome({ name: NAME, root: ROOT })
 if (homeProblem) die(`${homeProblem}\n(the bench home must be outside ${ROOT}; export LIVE_E2E_HOME to choose it)`)
 
+const MODE = readMode(process.env.LIVE_E2E_MODE)
+if (MODE !== 'live' && MODE !== 'mock') die(`LIVE_E2E_MODE is ${MODE}; use live or mock`)
+const loopbackEndpoint = () => {
+  try { return /^(localhost|127\.|0\.0\.0\.0|\[::1\]|::1)/.test(new URL(process.env.DEEPSEEK_BASE_URL ?? '').host) } catch { return false }
+}
 const apiKey = process.env.DEEPSEEK_API_KEY ?? ''
-const key = keyReading(apiKey)
-if (!key.usable) die(`DEEPSEEK_API_KEY ${keyProblem(key.reason)}; a real key is required (never printed)`)
+if (MODE === 'live') {
+  const key = keyReading(apiKey)
+  if (!key.usable) die(`DEEPSEEK_API_KEY ${keyProblem(key.reason)}; a real key is required (never printed)`)
+} else if (!loopbackEndpoint()) {
+  die('mock mode needs DEEPSEEK_BASE_URL on loopback, where the scripted mock listens; nothing was started')
+}
 
 const harness = process.env.DSH_HARNESS ?? join(ROOT, 'deepseek-harness')
 if (!existsSync(join(harness, 'package.json'))) die(`no deepseek-harness checkout at ${harness}; set DSH_HARNESS (see CONTRIBUTING.md)`)
@@ -87,14 +99,21 @@ async function start() {
   // anything is uploaded. Detached, the child is its own process group: the pid file
   // names the group, and a later stop reaches the whole tree.
   const out = openSync(logFile, 'a')
-  const child = spawn('pnpm', ['-s', 'dsh', '--profile', 'web', '--patch', 'apps/web/tests/pin-browse-picker.overlay.yml', '--patch', join(HOME_DIR, 'bench.patch.yml'), '--no-open', '--port', String(WEB_PORT)], {
+  // The overlay pins the in-browser directory picker. A source checkout carries it; the
+  // published runtime the mock job installs has no apps/ tree, and the smoke opens no
+  // picker, so its absence is not a failure.
+  const overlay = join(harness, 'apps/web/tests/pin-browse-picker.overlay.yml')
+  const patches = [...(existsSync(overlay) ? ['--patch', 'apps/web/tests/pin-browse-picker.overlay.yml'] : []), '--patch', join(HOME_DIR, 'bench.patch.yml')]
+  const child = spawn('pnpm', ['-s', 'dsh', '--profile', 'web', ...patches, '--no-open', '--port', String(WEB_PORT)], {
     cwd: harness,
     detached: true,
     stdio: ['ignore', out, out],
     env: {
       ...process.env,
       DSH_HOME: HOME_DIR,
-      DEEPSEEK_API_KEY: apiKey,
+      // In mock mode the bench never sees a key from this environment: the scripted mock
+      // ignores auth, and a real key has no business in a run that measures no model.
+      DEEPSEEK_API_KEY: MODE === 'mock' ? MOCK_KEY : apiKey,
       ...(process.env.DEEPSEEK_BASE_URL ? { DEEPSEEK_BASE_URL: process.env.DEEPSEEK_BASE_URL } : {}),
     },
   })
@@ -120,7 +139,7 @@ function tailSanitized() {
 function status() {
   const running = existsSync(webPidFile) && alive(Number(readFileSync(webPidFile, 'utf8')))
   console.log(`web: ${running ? `running (pid ${readFileSync(webPidFile, 'utf8').trim()})` : 'stopped'}`)
-  console.log(`home: ${HOME_DIR}  web: http://127.0.0.1:${WEB_PORT}  model: real (key from the environment)`)
+  console.log(`home: ${HOME_DIR}  web: http://127.0.0.1:${WEB_PORT}  model: ${MODE === 'mock' ? 'scripted mock (no key)' : 'real (key from the environment)'}`)
 }
 
 switch (ACTION) {