Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
4e4acf6
test: harden flaky errorsTab and widget E2E tests
DrJKL Apr 8, 2026
9f6421e
test: harden E2E tests across 14 additional spec files
DrJKL Apr 8, 2026
1c500cf
fix: harden more flaky e2e waits
DrJKL Apr 8, 2026
48812fd
test: harden change tracker undo assertion
DrJKL Apr 8, 2026
ee7560f
test: fix flaky interaction.spec.ts toggle and prompt dialog tests
DrJKL Apr 8, 2026
a18a7ea
Merge remote-tracking branch 'origin/main' into drjkl/harden-flaky-e2…
DrJKL Apr 8, 2026
4fad83d
[automated] Update test expectations
invalid-email-address Apr 8, 2026
97ab90f
test: deflake subgraphPromotion promote/un-promote menu clicks
DrJKL Apr 8, 2026
2624d67
test: harden 28 spec files for determinism (expect.poll, menu waits, …
DrJKL Apr 9, 2026
e3526be
fix: remoteWidgets backoff test — collect 4 retries and compare first…
DrJKL Apr 9, 2026
a4bd700
fix: remoteWidgets backoff test needs canvas redraws to drive retry s…
DrJKL Apr 9, 2026
4bf899a
fix: restore initial canvas click and correct coordinates in backoff …
DrJKL Apr 9, 2026
94d052b
fix: harden remaining flaky e2e waits
DrJKL Apr 9, 2026
b2082a8
Merge remote-tracking branch 'origin/main' into drjkl/harden-flaky-e2…
DrJKL Apr 9, 2026
f09a8c8
test: harden change tracker readiness waits
DrJKL Apr 9, 2026
64ccb47
fix: harden dropdown setting test with combobox role and aria-expanded
DrJKL Apr 9, 2026
9497077
fix: replace flaky URL drag-drop screenshot with behavioral assertion
DrJKL Apr 9, 2026
5df99c4
fix: remove force:true from BaseDialog.close() to fix animation race
DrJKL Apr 9, 2026
6d23122
fix: retry combobox click in settingsDialog dropdown test
DrJKL Apr 9, 2026
4b1579a
Merge branch 'main' into drjkl/harden-flaky-e2e-tests
DrJKL Apr 9, 2026
3c2fe2d
fix: re-dirty canvas in animated webp preview poll to avoid stale paint
DrJKL Apr 9, 2026
bc6d8e9
test: replace snapshot-then-assert with auto-retrying Playwright asse…
DrJKL Apr 9, 2026
4bf8553
fix: replace snapshot-then-assert with retrying assertions in e2e tests
DrJKL Apr 9, 2026
9821761
test: convert remaining racy assertions to expect.poll() across 27 sp…
DrJKL Apr 9, 2026
a250854
test: fix 28 additional racy assertions found by deep scan
DrJKL Apr 9, 2026
f2ba7c1
test: fix 27 more racy assertions from comprehensive deep scan
DrJKL Apr 9, 2026
c66ee62
test: convert all remaining page.evaluate snapshot assertions to expe…
DrJKL Apr 9, 2026
7d2a420
test: convert final 44 racy boundingBox/evaluate assertions to expect…
DrJKL Apr 9, 2026
50cef98
test: add hardening-flaky-e2e-tests agent skill
DrJKL Apr 9, 2026
c4e671f
fix: address CodeRabbit review feedback across 14 browser test files
DrJKL Apr 9, 2026
80de1d2
test: address review warnings from PR #10967
DrJKL Apr 9, 2026
d747679
fix: select single-output assets in bulk selection count test
DrJKL Apr 9, 2026
3907075
Merge branch 'main' into drjkl/harden-flaky-e2e-tests
DrJKL Apr 9, 2026
a6c9b9d
test: address PR review feedback — consolidate polls, fix regex, clea…
DrJKL Apr 9, 2026
b84f616
test: address PR review feedback from coderabbit and christian-byrne
DrJKL Apr 10, 2026
fb2d35e
test: remove redundant count methods in favor of locator getters
DrJKL Apr 10, 2026
290d092
merge: resolve conflict with origin/main, fix stale relative imports
DrJKL Apr 10, 2026
17bb341
fix: address review feedback — consolidate polls, remove side-effects
DrJKL Apr 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
246 changes: 246 additions & 0 deletions .claude/skills/hardening-flaky-e2e-tests/SKILL.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
---
name: hardening-flaky-e2e-tests
description: 'Diagnoses and fixes flaky Playwright e2e tests by replacing race-prone patterns with retry-safe alternatives. Use when triaging CI flakes, hardening spec files, fixing timing races, or asked to stabilize browser tests. Triggers on: flaky, flake, harden, stabilize, race condition in e2e, intermittent failure.'
---

# Hardening Flaky E2E Tests

Fix flaky Playwright specs by identifying race-prone patterns and replacing them with retry-safe alternatives. This skill covers diagnosis, pattern matching, and mechanical transforms — not writing new tests (see `writing-playwright-tests` for that).

## Workflow

### 1. Gather CI Evidence

```bash
gh run list --workflow=ci-test.yaml --limit=5
gh run download <run-id> -n playwright-report
```

- Open `report.json` and search for `"status": "flaky"` entries.
- Collect file paths, test titles, and error messages.
- Do NOT trust green checks alone — flaky tests that passed on retry still need fixing.
- Use `error-context.md`, traces, and page snapshots before editing code.
- Pull the newest run after each push instead of assuming the flaky set is unchanged.

### 2. Classify the Flake

Read the failing assertion and match it against the pattern table. Most flakes fall into one of these categories:

| # | Pattern | Signature in Code | Fix |
| --- | ------------------------------------- | --------------------------------------------------------- | ---------------------------------------------------------------- |
| 1 | **Snapshot-then-assert** | `expect(await evaluate()).toBe(x)` | `await expect.poll(() => evaluate()).toBe(x)` |
| 2 | **Immediate count** | `const n = await loc.count(); expect(n).toBe(3)` | `await expect(loc).toHaveCount(3)` |
| 3 | **nextFrame after menu click** | `clickMenuItem(x); nextFrame()` | `clickMenuItem(x); contextMenu.waitForHidden()` |
| 4 | **Tight poll timeout** | `expect.poll(..., { timeout: 250 })` | ≥2000 ms; prefer default 5000 ms |
| 5 | **Immediate evaluate after mutation** | `setSetting(k, v); expect(await evaluate()).toBe(x)` | `await expect.poll(() => evaluate()).toBe(x)` |
| 6 | **Screenshot without readiness** | `loadWorkflow(); nextFrame(); toHaveScreenshot()` | `waitForNodes()` or poll state first |
| 7 | **Non-deterministic node order** | `getNodeRefsByType('X')[0]` with >1 match | `getNodeRefById(id)` or guard `toHaveLength(1)` |
| 8 | **Fake readiness helper** | Helper clicks but doesn't assert state | Remove; poll the actual value |
| 9 | **Immediate graph state after drop** | `expect(await getLinkCount()).toBe(1)` | `await expect.poll(() => getLinkCount()).toBe(1)` |
| 10 | **Immediate boundingBox/layout read** | `const box = await loc.boundingBox(); expect(box!.width)` | `await expect.poll(() => loc.boundingBox().then(b => b?.width))` |

### 3. Apply the Transform

#### Rule: Choose the Smallest Correct Assertion

- **Locator state** → use built-in retrying assertions: `toBeVisible()`, `toHaveText()`, `toHaveCount()`, `toHaveClass()`
- **Single async value** → `expect.poll(() => asyncFn()).toBe(expected)`
- **Multiple assertions that must settle together** → `expect(async () => { ... }).toPass()`
- **Never** use `waitForTimeout()` to hide a race.

```typescript
// ✅ Single value — use expect.poll
await expect
.poll(() => comfyPage.page.evaluate(() => window.app!.graph.links.length))
.toBe(3)

// ✅ Locator count — use toHaveCount
await expect(comfyPage.page.locator('.dom-widget')).toHaveCount(2)

// ✅ Multiple conditions — use toPass
await expect(async () => {
expect(await node1.getValue()).toBe('foo')
expect(await node2.getValue()).toBe('bar')
}).toPass({ timeout: 5000 })
```

#### Rule: Wait for the Real Readiness Boundary

Visible is not always ready. Prefer user-facing assertions when possible; poll internal state only when there is no UI surface to assert on.

Common readiness boundaries:

| After this action... | Wait for... |
| -------------------------------------- | ------------------------------------------------------------ |
| Canvas interaction (drag, click node) | `await comfyPage.nextFrame()` |
| Menu item click | `await contextMenu.waitForHidden()` |
| Workflow load | `await comfyPage.workflow.loadWorkflow(...)` (built-in wait) |
| Settings write | Poll the setting value with `expect.poll()` |
| Node pin/bypass/collapse toggle | `await expect.poll(() => nodeRef.isPinned()).toBe(true)` |
| Graph mutation (add/remove node, link) | Poll link/node count |
| Clipboard write | Poll pasted value |
| Screenshot | Ensure nodes are rendered: `waitForNodes()` or poll state |

#### Rule: Expose Locators for Retrying Assertions

When a helper returns a count via `await loc.count()`, callers can't use `toHaveCount()`. Expose the underlying `Locator` as a getter so callers choose between:

```typescript
// Helper exposes locator
get domWidgets(): Locator {
return this.page.locator('.dom-widget')
}

// Caller uses retrying assertion
await expect(comfyPage.domWidgets).toHaveCount(2)
```

Replace count methods with locator getters so callers can use retrying assertions directly.

#### Rule: Fix Check-then-Act Races in Helpers

```typescript
// ❌ Race: count can change between check and waitFor
const count = await locator.count()
if (count > 0) {
await locator.waitFor({ state: 'hidden' })
}

// ✅ Direct: waitFor handles both cases
await locator.waitFor({ state: 'hidden' })
```

#### Rule: Remove force:true from Clicks

`force: true` bypasses actionability checks, hiding real animation/visibility races. Remove it and fix the underlying timing issue.

```typescript
// ❌ Hides the race
await closeButton.click({ force: true })

// ✅ Surfaces the real issue — fix with proper wait
await closeButton.click()
await dialog.waitForHidden()
```

#### Rule: Handle Non-deterministic Element Order

When `getNodeRefsByType` returns multiple nodes, the order is not guaranteed. Don't use index `[0]` blindly.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could be a linting rule probably

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Love that idea. Out of scope here but would be a solid eslint-plugin-playwright custom rule. Want me to file an issue?


```typescript
// ❌ Assumes order
const node = (await comfyPage.nodeOps.getNodeRefsByType('CLIPTextEncode'))[0]

// ✅ Find by ID or proximity
const nodes = await comfyPage.nodeOps.getNodeRefsByType('CLIPTextEncode')
let target = nodes[0]
for (const n of nodes) {
const pos = await n.getPosition()
if (Math.abs(pos.y - expectedY) < minDist) target = n
}
```

Or guard the assumption:

```typescript
const nodes = await comfyPage.nodeOps.getNodeRefsByType('CLIPTextEncode')
expect(nodes).toHaveLength(1)
const node = nodes[0]
```

#### Rule: Use toPass for Timing-sensitive Dismiss Guards

Some UI elements (e.g. LiteGraph's graphdialog) have built-in dismiss delays. Retry the entire dismiss action:

```typescript
// ✅ Retry click+assert together
await expect(async () => {
await comfyPage.canvas.click({ position: { x: 10, y: 10 } })
await expect(dialog).toBeHidden({ timeout: 500 })
}).toPass({ timeout: 5000 })
```

### 4. Keep Changes Narrow

- Shared helpers should drive setup to a stable boundary.
- Do not encode one-spec timing assumptions into generic helpers.
- If a race only matters to one spec, prefer a local wait in that spec.
- If a helper fails before the real test begins, remove or relax the brittle precondition and let downstream UI interaction prove readiness.

### 5. Verify Narrowly

```bash
# Targeted rerun with repetition
pnpm test:browser:local -- browser_tests/tests/myFile.spec.ts --repeat-each 10

# Single test by line number (avoids grep quoting issues on Windows)
pnpm test:browser:local -- browser_tests/tests/myFile.spec.ts:42
```

- Use `--repeat-each 10` for targeted flake verification (use 20 for single test cases).
- Verify with the smallest command that exercises the flaky path.

### 6. Watch CI E2E Runs

After pushing, use `gh` to monitor the E2E workflow:

```bash
# Find the run for the current branch
gh run list --workflow="CI: Tests E2E" --branch=$(git branch --show-current) --limit=1

# Watch it live (blocks until complete, streams logs)
gh run watch <run-id>

# One-liner: find and watch the latest E2E run for the current branch
gh run list --workflow="CI: Tests E2E" --branch=$(git branch --show-current) --limit=1 --json databaseId --jq ".[0].databaseId" | xargs gh run watch
```

On Windows (PowerShell):

```powershell
# One-liner equivalent
gh run watch (gh run list --workflow="CI: Tests E2E" --branch=$(git branch --show-current) --limit=1 --json databaseId --jq ".[0].databaseId")
```

After the run completes:

```bash
# Download the Playwright report artifact
gh run download <run-id> -n playwright-report

# View the run summary in browser
gh run view <run-id> --web
```

Also watch the unit test workflow in parallel if you changed helpers:

```bash
gh run list --workflow="CI: Tests Unit" --branch=$(git branch --show-current) --limit=1
```

### 7. Pre-merge Checklist

Before merging a flaky-test fix, confirm:

- [ ] The latest CI artifact was inspected directly
- [ ] The root cause is stated as a race or readiness mismatch
- [ ] The fix waits on the real readiness boundary
- [ ] The assertion primitive matches the job (poll vs toHaveCount vs toPass)
- [ ] The fix stays local unless a shared helper truly owns the race
- [ ] Local verification uses a targeted rerun
- [ ] No behavioral changes to the test — only timing/retry strategy updated

## Local Noise — Do Not Fix

These are local distractions, not CI root causes:

- Missing local input fixture files required by the test path
- Missing local models directory
- Teardown `EPERM` while restoring the local browser-test user data directory

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This happens for you? We should fix the global teardown scripts to account for it. @coderabbitai Can you make an Issue to do that?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🧩 Analysis chain

Sure! I'll create that issue now.


🏁 Script executed:

#!/bin/bash
gh issue create \
  --repo Comfy-Org/ComfyUI_frontend \
  --title "Fix global teardown scripts to handle EPERM when restoring browser-test user data directory" \
  --body "## Summary

During local browser test runs, a teardown \`EPERM\` error occurs while attempting to restore the browser-test user data directory. The global teardown scripts do not currently handle this case gracefully.

This was surfaced in PR `#10967`, where the error was documented as a known local noise item in \`.claude/skills/hardening-flaky-e2e-tests/SKILL.md\`:

> Teardown \`EPERM\` while restoring the local browser-test user data directory

## Goal

Update the global teardown scripts (e.g., \`browser_tests/\` global setup/teardown) to catch and handle \`EPERM\` errors when restoring the user data directory, so that local test runs do not surface spurious failures or warnings on teardown.

## References

- PR: https://github.com/Comfy-Org/ComfyUI_frontend/pull/10967
- Comment: https://github.com/Comfy-Org/ComfyUI_frontend/pull/10967#discussion_r3061550360

Raised by `@christian-byrne`." \
  --assignee christian-byrne

Length of output: 58


@christian-byrne, done! I've opened issue #11009 — "Fix global teardown scripts to handle EPERM when restoring browser-test user data directory" — assigned to you. It includes context from this PR and the relevant reference in SKILL.md.


🧠 Learnings used
Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: browser_tests/AGENTS.md:0-0
Timestamp: 2026-04-08T02:47:53.153Z
Learning: Applies to browser_tests/tests/**/*.spec.ts : Refer to `browser_tests/FLAKE_PREVENTION_RULES.md` when triaging or editing flaky browser tests

Learnt from: benceruleanlu
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: :0-0
Timestamp: 2026-03-27T05:26:01.429Z
Learning: In Comfy-Org/ComfyUI_frontend PR `#10526`, the async scope-disposal race in `src/platform/cloud/notification/components/DesktopCloudNotificationController.vue` (desktop/mac-only cloud promo flow) is intentionally covered only at the unit level via `DesktopCloudNotificationController.test.ts`, using a deferred `settingStore.load()` promise and fake timers to deterministically verify that the dialog is not scheduled or shown after the owning scope is disposed. A Playwright E2E test is not appropriate here because: (1) the non-deterministic ordering cannot be reliably reproduced in Playwright, (2) the feature requires a real macOS Electron environment not available in CI, and (3) a timing-driven pseudo-E2E would be flaky and weaker than the unit test. If broader coverage is needed later, the right approach is a separate desktop promo smoke test, not a race-specific E2E regression. Do not request a Playwright/E2E test for this specific race condition scenario.

Learnt from: pythongosssss
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: :0-0
Timestamp: 2026-03-18T15:38:16.169Z
Learning: In Comfy-Org/ComfyUI_frontend, race conditions between WebSocket events and HTTP responses (e.g., the WebSocket/HTTP race in job tracking in `src/renderer/extensions/linearMode/linearOutputStore.ts`) are not suitable candidates for E2E testing because the non-deterministic ordering cannot be reliably reproduced in Playwright. Unit tests (e.g., in `linearOutputStore.test.ts`) that deterministically simulate event ordering are the correct approach. Do not request E2E tests for such race condition scenarios.

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: coderabbit-custom-pre-merge-checks-unique-id-file-non-traceable-F7F2B60C-1728-4C9A-8889-4F2235E186CA.txt:0-0
Timestamp: 2026-03-27T05:07:49.322Z
Learning: For end-to-end regression coverage on bug fixes: Use only PR metadata (title, commit subjects, changed-file list, PR description) without shell commands. Pass if: (1) Neither PR title nor commit subjects use bug-fix language (fix, fixed, fixes, fixing, bugfix, hotfix), OR (2) PR changes at least one file under `browser_tests/`, OR (3) PR description includes concrete, non-placeholder explanation of why end-to-end regression test was not added. Otherwise fail and ask author to add or update Playwright regression test under `browser_tests/` or provide concrete explanation in PR description. Mark check inconclusive if changed-file list or commit subjects unavailable.

Learnt from: jaeone94
Repo: Comfy-Org/ComfyUI_frontend PR: 10309
File: browser_tests/tests/missingMedia.spec.ts:141-145
Timestamp: 2026-03-25T15:30:09.361Z
Learning: In `browser_tests/tests/missingMedia.spec.ts` (Comfy-Org/ComfyUI_frontend PR `#10309`), the library-select E2E test uses `test.skip()` when no options are found because the OSS test server's input directory content varies per environment and cannot be guaranteed to contain specific library items. The upload flow test already covers the full 2-step confirm path. Do not flag this `test.skip()` guard as a regression risk or suggest pre-uploading files (which creates inter-test dependency). If deterministic coverage is needed in the future, suggest mocking the input-file list endpoint via Playwright's `page.route()` instead.

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: browser_tests/AGENTS.md:0-0
Timestamp: 2026-04-08T02:47:53.153Z
Learning: Applies to browser_tests/**/*.spec.ts : Reference Playwright best practices from `docs/guidance/playwright.md` when writing end-to-end tests

Learnt from: DrJKL
Repo: Comfy-Org/ComfyUI_frontend PR: 10958
File: browser_tests/helpers/builderTestUtils.ts:65-66
Timestamp: 2026-04-08T18:10:30.277Z
Learning: In `browser_tests/helpers/builderTestUtils.ts` (Comfy-Org/ComfyUI_frontend), the `ksampler.convertToSubgraph()` followed by a single `comfyPage.nextFrame()` wait before returning `{ inputNodeTitle: 'New Subgraph', widgetNames: ['seed'] }` is a pre-existing, CI-verified pattern. Do not flag this as a race condition or request additional polling/assertions for promoted-widget hydration in this helper; it works reliably in CI.

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: AGENTS.md:0-0
Timestamp: 2026-04-08T21:15:04.091Z
Learning: Applies to browser_tests/**/*.spec.ts : Use Playwright best practices and do not use waitForTimeout; use Locator actions and retrying assertions instead

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: coderabbit-custom-pre-merge-checks-unique-id-file-non-traceable-F7F2B60C-1728-4C9A-8889-4F2235E186CA.txt:0-0
Timestamp: 2026-03-24T17:16:12.386Z
Learning: For bug-fix PRs, ensure end-to-end regression coverage by either: (1) changing at least one file under `browser_tests/`, or (2) providing a concrete, non-placeholder explanation in the PR description of why an end-to-end regression test was not added. Use only PR metadata available in the review context (PR title, commit subjects, files changed relative to base, PR description). Do not rely on shell commands or inspect files outside the PR.

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: coderabbit-custom-pre-merge-checks-unique-id-file-non-traceable-F7F2B60C-1728-4C9A-8889-4F2235E186CA.txt:0-0
Timestamp: 2026-03-17T07:39:57.336Z
Learning: End-to-end regression coverage for fixes: Use PR metadata (PR title, commit subjects, files changed relative to PR base, PR description) to verify bug-fix PRs include end-to-end regression test coverage. Pass if: (1) PR title and commit subjects don't use bug-fix language (fix, fixed, fixes, fixing, bugfix, hotfix), OR (2) PR changes at least one file under browser_tests/, OR (3) PR description includes concrete non-placeholder explanation for why end-to-end regression test wasn't added. Fail otherwise, mentioning the bug-fix signal found and asking to add/update Playwright regression test under browser_tests/ or provide concrete explanation in PR description.

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: browser_tests/AGENTS.md:0-0
Timestamp: 2026-04-08T02:47:53.154Z
Learning: Consult the Playwright test-writing skill at `.claude/skills/writing-playwright-tests/SKILL.md` for meta-level guidance on gotchas, anti-patterns, and decision guides

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: coderabbit-custom-pre-merge-checks-unique-id-file-non-traceable-F7F2B60C-1728-4C9A-8889-4F2235E186CA.txt:0-0
Timestamp: 2026-03-27T04:23:01.330Z
Learning: End-to-end regression coverage for fixes: For PRs that include bug-fix language in the title or commit subjects (fix, fixed, fixes, fixing, bugfix, hotfix), ensure at least one of the following: (1) files are changed under browser_tests/, or (2) the PR description includes a concrete, non-placeholder explanation of why an end-to-end regression test was not added. Only use PR metadata available in the review context (PR title, commit subjects, files changed relative to base, PR description). Do not rely on shell commands, reverse diffs, or files outside the PR.

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: coderabbit-custom-pre-merge-checks-unique-id-file-non-traceable-F7F2B60C-1728-4C9A-8889-4F2235E186CA.txt:0-0
Timestamp: 2026-04-09T23:18:11.029Z
Learning: End-to-end regression coverage for fixes: If a PR uses bug-fix language (`fix`, `fixed`, `fixes`, `fixing`, `bugfix`, or `hotfix`) in the title or commit subjects, ensure the PR either changes at least one file under `browser_tests/` or includes a concrete, non-placeholder explanation in the PR description of why an end-to-end regression test was not added.

Learnt from: CR
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: coderabbit-custom-pre-merge-checks-unique-id-file-non-traceable-F7F2B60C-1728-4C9A-8889-4F2235E186CA.txt:0-0
Timestamp: 2026-03-25T22:30:34.819Z
Learning: For bug-fix PRs (containing language like 'fix', 'fixed', 'fixes', 'fixing', 'bugfix', or 'hotfix' in PR title or commit subjects), ensure end-to-end regression coverage by either: (1) changing at least one file under `browser_tests/`, or (2) providing a concrete, non-placeholder explanation in the PR description of why an end-to-end regression test is not practical. Use only PR metadata available in the review context (PR title, commit subjects, files changed, PR description). Do not rely on shell commands, reverse diffs, or inspect files outside the PR.

Learnt from: pythongosssss
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: :0-0
Timestamp: 2026-04-09T19:42:11.362Z
Learning: In Comfy-Org/ComfyUI_frontend PR `#10993` ("fix/feat: App mode - Persist user resized widget heights"), end-to-end regression tests for the new `useAppModeWidgetResizing` composable and persisted widget height behavior were intentionally deferred to a follow-up PR. The reason given by the author (pythongosssss) is to allow the fix to be backported more easily. Do not flag the absence of browser_tests/ changes in PR `#10993` as a coverage gap.

Learnt from: pythongosssss
Repo: Comfy-Org/ComfyUI_frontend PR: 0
File: :0-0
Timestamp: 2026-04-09T19:41:57.643Z
Learning: In Comfy-Org/ComfyUI_frontend PR `#10993` ("fix/feat: App mode - Persist user resized widget heights"), e2e regression tests for persisting user-resized widget heights are intentionally deferred to a follow-up PR to make backporting easier. Do not flag the absence of `browser_tests/` changes in PR `#10993` as a missing regression test.

Learnt from: christian-byrne
Repo: Comfy-Org/ComfyUI_frontend PR: 10140
File: apps/website/package.json:1-79
Timestamp: 2026-03-17T07:28:27.677Z
Learning: In Comfy-Org/ComfyUI_frontend, PRs managed via Graphite stacked sets (using `gt split`) may show browser_tests/ files in the diff as artifacts of the split process. These files contain no actual changes relative to origin/main and should not be flagged as unrelated scope changes. Verify against the PR title/description stacking context before raising scope concerns.

Learnt from: jtydhr88
Repo: Comfy-Org/ComfyUI_frontend PR: 9381
File: .claude/skills/threejs-postprocessing/SKILL.md:525-533
Timestamp: 2026-03-05T01:07:51.626Z
Learning: Treat all files under .claude/skills/ in this repository as verbatim upstream content sourced from CloudAI-X/threejs-skills. Do not propose or apply edits to their content. Any modifications should be limited to metadata, licensing notes, or repository organization; preserve the original upstream content unless an upstream update is requested/requested changes are handled upstream.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, it's a Windows thing — fs.rm races with Chromium still holding handles on the user-data dir. CodeRabbit already filed #11009 for it. I'll take a look.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like the humans are having a chat. I'll hop back into my burrow for now. If you need me again, just tag @coderabbitai in a new comment, and I'll come hopping out!

- Local screenshot baseline differences on Windows

Rules:

- First confirm whether it blocks the exact flaky path under investigation.
- Do not commit temporary local assets used only for verification.
- Do not commit local screenshot baselines.
15 changes: 15 additions & 0 deletions browser_tests/FLAKE_PREVENTION_RULES.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,21 @@ await expect
a different reproduction pattern.
- Verify with the smallest command that exercises the flaky path.

## 7. Common Flake Patterns

| Pattern | Bad | Fix |
| ------------------------------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------ |
| **Snapshot-then-assert** | `expect(await evaluate()).toBe(x)` | `await expect.poll(() => evaluate()).toBe(x)` |
| **Immediate boundingBox/layout read** | `const box = await loc.boundingBox(); expect(box!.width).toBe(w)` | `await expect.poll(() => loc.boundingBox().then(b => b?.width)).toBe(w)` |
| **Immediate graph state after drop** | `expect(await getLinkCount()).toBe(1)` | `await expect.poll(() => getLinkCount()).toBe(1)` |
| **Fake readiness helper** | Helper that clicks but doesn't assert state | Remove; poll the actual value |
| **nextFrame after menu click** | `clickMenuItem(x); nextFrame()` | `clickMenuItem(x); contextMenu.waitForHidden()` |
| **Tight poll timeout** | `expect.poll(..., { timeout: 250 })` | ≥2000ms; prefer default (5000ms) |
| **Immediate count()** | `const n = await loc.count(); expect(n).toBe(3)` | `await expect(loc).toHaveCount(3)` |
| **Immediate evaluate after mutation** | `setSetting(); expect(await evaluate()).toBe(x)` | `await expect.poll(() => evaluate()).toBe(x)` |
| **Screenshot without readiness** | `loadWorkflow(); nextFrame(); toHaveScreenshot()` | `waitForNodes()` or poll state first |
| **Non-deterministic node order** | `getNodeRefsByType('X')[0]` with >1 match | `getNodeRefById(id)` or guard `toHaveLength(1)` |

## Current Local Noise

These are local distractions, not automatic CI root causes:
Expand Down
4 changes: 2 additions & 2 deletions browser_tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,8 @@ Most common testing needs are already addressed by these helpers, which will mak

```typescript
// Prefer this:
expect(await node.isPinned()).toBe(true)
expect(await node.getProperty('title')).toBe('Expected Title')
await expect.poll(() => node.isPinned()).toBe(true)
await expect.poll(() => node.getProperty('title')).toBe('Expected Title')
Comment thread
DrJKL marked this conversation as resolved.
Resolved

// Over this - only use when needed:
await expect(comfyPage.canvas).toHaveScreenshot('state.png')
Expand Down
5 changes: 2 additions & 3 deletions browser_tests/fixtures/ComfyPage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -392,9 +392,8 @@ export class ComfyPage {
await modal.waitFor({ state: 'hidden' })
}

/** Get number of DOM widgets on the canvas. */
async getDOMWidgetCount() {
return await this.page.locator('.dom-widget').count()
get domWidgets(): Locator {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion: (non-blocking) domWidgets is added but has no call-sites in this PR. getDOMWidgetCount() right below duplicates the same .dom-widget selector instead of delegating to this.domWidgets.count(). Same applies to toastErrors in ToastHelper.ts vs getToastErrorCount(). Consider either wiring up the call-sites or deferring these getters to a follow-up.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

domWidgets does have call-sites — it's used in domWidget.spec.ts (lines 45 and 55). Similarly, toastErrors is used in 4 specs (changeTracker, colorPalette, commands, toastNotifications). Neither getDOMWidgetCount nor getToastErrorCount exist — the locator getters are the primary API. No change needed here.

return this.page.locator('.dom-widget')
}

async setFocusMode(focusMode: boolean) {
Expand Down
7 changes: 0 additions & 7 deletions browser_tests/fixtures/VueNodeHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,6 @@ export class VueNodeHelpers {
return await this.nodes.count()
}

/**
* Get count of selected Vue nodes
*/
async getSelectedNodeCount(): Promise<number> {
return await this.selectedNodes.count()
}

/**
* Get all Vue node IDs currently in the DOM
*/
Expand Down
2 changes: 1 addition & 1 deletion browser_tests/fixtures/components/BaseDialog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export class BaseDialog {
}

async close(): Promise<void> {
await this.closeButton.click({ force: true })
await this.closeButton.click()
await this.waitForHidden()
}
}
16 changes: 2 additions & 14 deletions browser_tests/fixtures/components/ContextMenu.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,21 +65,9 @@ export class ContextMenu {
}

async waitForHidden(): Promise<void> {
const waitIfExists = async (locator: Locator, menuName: string) => {
const count = await locator.count()
if (count > 0) {
await locator.waitFor({ state: 'hidden' }).catch((error: Error) => {
console.warn(
`[waitForHidden] ${menuName} waitFor failed:`,
error.message
)
})
}
}

await Promise.all([
waitIfExists(this.primeVueMenu, 'primeVueMenu'),
waitIfExists(this.litegraphMenu, 'litegraphMenu')
this.primeVueMenu.waitFor({ state: 'hidden' }),
this.litegraphMenu.waitFor({ state: 'hidden' })
])
}
}
10 changes: 7 additions & 3 deletions browser_tests/fixtures/components/SidebarTab.ts
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,14 @@ export class WorkflowsSidebarTab extends SidebarTab {
.allInnerTexts()
}

get activeWorkflowLabel(): Locator {
return this.root.locator(
'.comfyui-workflows-open .p-tree-node-selected .node-label'
)
}

async getActiveWorkflowName() {
return await this.root
.locator('.comfyui-workflows-open .p-tree-node-selected .node-label')
.innerText()
return await this.activeWorkflowLabel.innerText()
}

async getTopLevelSavedWorkflowNames() {
Expand Down
2 changes: 1 addition & 1 deletion browser_tests/fixtures/helpers/SubgraphHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ export class SubgraphHelper {
await this.rightClickOutputSlot(slotName)
}
await this.comfyPage.contextMenu.clickLitegraphMenuItem('Remove Slot')
await this.comfyPage.nextFrame()
await this.comfyPage.contextMenu.waitForHidden()
}

async findSubgraphNodeId(): Promise<string> {
Expand Down
10 changes: 2 additions & 8 deletions browser_tests/fixtures/helpers/ToastHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,8 @@ export class ToastHelper {
return this.page.locator('.p-toast-message:visible')
}

async getToastErrorCount(): Promise<number> {
return await this.page
.locator('.p-toast-message.p-toast-message-error')
.count()
}

async getVisibleToastCount(): Promise<number> {
return await this.visibleToasts.count()
get toastErrors(): Locator {
return this.page.locator('.p-toast-message.p-toast-message-error')
}

async closeToasts(requireCount = 0): Promise<void> {
Expand Down
7 changes: 5 additions & 2 deletions browser_tests/helpers/manageGroupNode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,12 @@ export class ManageGroupNode {
await this.footer.getByText('Close').click()
}

get selectedNodeTypeSelect(): Locator {
return this.header.locator('select').first()
}

async getSelectedNodeType() {
const select = this.header.locator('select').first()
return await select.inputValue()
return await this.selectedNodeTypeSelect.inputValue()
}

async selectNode(name: string) {
Expand Down
Loading
Loading