ci(e2e): compare auth tests on playwright and endform

.github/workflows/e2e-auth.yml

@@ -4,15 +4,35 @@ on:
   pull_request:
   push:
     branches: [main]
+  workflow_dispatch:
+    inputs:
+      repeat_each:
+        description: "Run each test N times (passed to both runners as --repeat-each)"
+        required: false
+        default: "1"
+      retries:
+        description: "Override the config's retries (empty keeps the config value)"
+        required: false
+        default: ""
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   auth-e2e:
-    timeout-minutes: 15
+    name: Auth E2E (${{ matrix.runner }})
+    timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 45 || 15 }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # The same suite on plain Playwright and on Endform, so every run
+        # yields a side-by-side comparison on the same commit.
+        runner: [playwright, endform]
+    permissions:
+      contents: read
+      pull-requests: read
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
       TURBO_TEAM: ${{ vars.TURBO_TEAM }}
@@ -21,35 +41,61 @@ jobs:
 
       - uses: ./.github/actions/detect-code-changes
         id: changes
+        if: github.event_name != 'workflow_dispatch'
 
       - uses: ./.github/actions/setup
-        if: steps.changes.outputs.has_changes == 'true'
+        if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true'
 
       - name: Store Playwright's Version
-        if: steps.changes.outputs.has_changes == 'true'
+        if: matrix.runner == 'playwright' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true')
         id: playwright-version
         run: |
           PLAYWRIGHT_VERSION=$(pnpm exec playwright --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
           echo "version=${PLAYWRIGHT_VERSION}" >> "$GITHUB_OUTPUT"
 
       - name: Cache Playwright Browsers
-        if: steps.changes.outputs.has_changes == 'true'
+        if: matrix.runner == 'playwright' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true')
         id: cache-playwright-browsers
         uses: actions/cache@v6
         with:
           path: ~/.cache/ms-playwright
           key: ${{ steps.playwright-version.outputs.version }}-playwright-browsers-chromium
 
       - name: Install Playwright Browsers
-        if: steps.changes.outputs.has_changes == 'true' && steps.cache-playwright-browsers.outputs.cache-hit != 'true'
+        if: matrix.runner == 'playwright' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true') && steps.cache-playwright-browsers.outputs.cache-hit != 'true'
         run: pnpm exec playwright install --with-deps chromium
 
       - name: Build packages and auth-test-studio
-        if: steps.changes.outputs.has_changes == 'true'
+        if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true'
         run: pnpm build --filter=auth-test-studio...
 
-      - name: Run auth E2E tests
-        if: steps.changes.outputs.has_changes == 'true'
-        run: pnpm --filter e2e test:auth
+      - name: Run auth E2E tests (Playwright)
+        if: matrix.runner == 'playwright' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true')
+        working-directory: e2e
+        run: |
+          args=(--repeat-each "${REPEAT_EACH:-1}")
+          if [ -n "${RETRIES}" ]; then args+=(--retries "${RETRIES}"); fi
+          pnpm exec playwright test --config=playwright.auth.config.ts "${args[@]}"
+        env:
+          CI: "true"
+          REPEAT_EACH: ${{ inputs.repeat_each }}
+          RETRIES: ${{ inputs.retries }}
+
+      # Runs every test on its own Endform machine. The preview server from the
+      # config's webServer keeps running on this job, and Endform routes the
+      # runners' localhost traffic back to it. The runners have no studio
+      # node_modules, so the DOM prop leak guard's vocabulary is written here
+      # first and shipped to them (see e2e/endform.config.ts).
+      - name: Run auth E2E tests (Endform)
+        if: matrix.runner == 'endform' && (github.event_name == 'workflow_dispatch' || steps.changes.outputs.has_changes == 'true')
+        working-directory: e2e
+        run: |
+          pnpm run dom-prop-vocabulary
+          args=(--repeat-each "${REPEAT_EACH:-1}")
+          if [ -n "${RETRIES}" ]; then args+=(--retries "${RETRIES}"); fi
+          npx --yes endform@0.81.5 test --organization-id OQXolTtG --config=playwright.auth.config.ts "${args[@]}"
         env:
           CI: "true"
+          ENDFORM_API_KEY: ${{ secrets.ENDFORM_API_KEY }}
+          REPEAT_EACH: ${{ inputs.repeat_each }}
+          RETRIES: ${{ inputs.retries }}

e2e/.gitignore

@@ -4,3 +4,4 @@
 /results-auth
 /blob-report
 /diagnostics
+/helpers/domPropLeaks/vocabulary.generated.json

e2e/endform.config.ts

@@ -0,0 +1,7 @@
+// Endform sends each runner the files the tests import. Files they read at runtime without
+// importing them are listed here.
+export default {
+  // Written by `pnpm --filter e2e dom-prop-vocabulary` before the run, see
+  // `helpers/domPropLeaks/vocabulary.ts`.
+  additionalFiles: ['helpers/domPropLeaks/vocabulary.generated.json'],
+}

e2e/helpers/domPropLeaks/vocabulary.ts

@@ -1,6 +1,7 @@
 import {readFileSync} from 'node:fs'
 import {createRequire} from 'node:module'
 import path from 'node:path'
+import {fileURLToPath} from 'node:url'
 
 import {loadReactDomInternals, STUDIO_PACKAGE_JSON} from './reactDomInternals'
 
@@ -125,9 +126,25 @@ function toSource(regExp: RegExp): RegExpSource {
 
 let cachedVocabulary: DomPropVocabulary | undefined
 
+/**
+ * The vocabulary as written ahead of the run by `scripts/writeDomPropVocabulary.ts`. Endform runs
+ * each test on a machine that has the e2e package but not the studio's `node_modules`, so the
+ * runner reads this file (shipped through `endform.config.ts`) instead of the studio's `react-dom`.
+ */
+export const PREBUILT_VOCABULARY_PATH = fileURLToPath(
+  new URL('./vocabulary.generated.json', import.meta.url),
+)
+
 export function loadDomPropVocabulary(): DomPropVocabulary {
   if (cachedVocabulary) return cachedVocabulary
 
+  if (process.env.ENDFORM === 'true') {
+    cachedVocabulary = JSON.parse(
+      readFileSync(PREBUILT_VOCABULARY_PATH, 'utf8'),
+    ) as DomPropVocabulary
+    return cachedVocabulary
+  }
+
   const react = loadReactDomInternals()
   const isPropValid = loadIsPropValid()
   cachedVocabulary = {

e2e/package.json

@@ -10,6 +10,7 @@
     "cleanup": "tsx -r dotenv-flow/config scripts/cleanupDatasets",
     "codegen": "node -r dotenv-flow/config ./node_modules/.bin/sanity-test codegen",
     "dev": "pnpm --filter studio-e2e-testing dev",
+    "dom-prop-vocabulary": "tsx scripts/writeDomPropVocabulary.ts",
     "extract-diagnostics": "tsx scripts/flakeReport/extractDiagnostics.ts",
     "flake-report": "tsx scripts/flakeReport/index.ts",
     "preview": "pnpm e2e:build && pnpm --filter studio-e2e-testing preview",

e2e/playwright.auth.config.ts

@@ -31,7 +31,8 @@ export default defineConfig({
   reporter: [['list']],
   use: {
     actionTimeout: 10_000,
-    trace: 'on-first-retry',
+    // Endform shows the trace of every failed attempt in its dashboard
+    trace: process.env.ENDFORM === 'true' ? 'retain-on-failure' : 'on-first-retry',
     viewport: {width: 1728, height: 1000},
     video: 'retain-on-failure',
     baseURL: `http://localhost:${PORT}`,

e2e/scripts/writeDomPropVocabulary.ts

@@ -0,0 +1,9 @@
+/**
+ * Writes the DOM prop leak guard's vocabulary to a JSON file, for test runners that cannot read
+ * the studio's `react-dom` themselves (Endform). Run it after `pnpm install` and before the suite.
+ */
+import {writeFileSync} from 'node:fs'
+
+import {loadDomPropVocabulary, PREBUILT_VOCABULARY_PATH} from '../helpers/domPropLeaks/vocabulary'
+
+writeFileSync(PREBUILT_VOCABULARY_PATH, `${JSON.stringify(loadDomPropVocabulary(), null, 2)}\n`)