Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
67203e4
test: ratchet expected-failure inventory
christian-byrne Oct 4, 2026
4373b06
fix: close expected-failure ratchet gaps
christian-byrne Oct 5, 2026
e6576aa
fix: recognize playwright fixture runners
christian-byrne Oct 5, 2026
6c71190
fix: cover all expected-failure runners
christian-byrne Oct 5, 2026
a69d6d5
fix: detect app Playwright failure pins
christian-byrne Oct 5, 2026
d44dee4
test: stabilize computed expected-failure titles
christian-byrne Oct 5, 2026
e2382f3
test: preserve callback-form failure titles
christian-byrne Oct 5, 2026
08e8807
fix(ci): bound expected-failure source inspection
christian-byrne Oct 5, 2026
0306aee
perf(ci): skip non-test files in failure scan
christian-byrne Oct 5, 2026
7cd3abd
Merge remote-tracking branch 'origin/main' into repair-20140-run7
christian-byrne Oct 13, 2026
32ae6a8
test(ci): refresh expected-failure baseline
christian-byrne Oct 13, 2026
c6bc07b
Merge branch 'main' into christian/expected-failure-ratchet
christian-byrne Oct 16, 2026
990f4da
test(ci): lower expected-failure ceiling after restored pins
christian-byrne Oct 17, 2026
ec6dda9
test: cover expected-failure baseline comparison
christian-byrne Oct 19, 2026
858d8e9
Merge remote-tracking branch 'origin/main' into update-20140-run12
christian-byrne Oct 20, 2026
a7d34e0
test: name both actions in the live-defect ceiling message
christian-byrne Oct 20, 2026
7d21ee4
refactor: move expected-failure ratchet into a tools workspace
DrJKL Oct 20, 2026
133fdaf
refactor: store the expected-failure baseline as a top-level array
DrJKL Oct 20, 2026
b5ad479
Merge remote-tracking branch 'origin/main' into takeover/pr20140
DrJKL Oct 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ci-lint-format.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -204,3 +204,6 @@ jobs:

- name: Verify knip, locale integrity and ADR naming
uses: ./.github/actions/repo-checks

- name: Prevent untracked expected failures
run: pnpm test:expected-failure-ratchet
3 changes: 3 additions & 0 deletions knip.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ const config: KnipConfig = {
'tools/design-system-audits': {
project: ['src/**/*.ts']
},
'tools/expected-failures': {
project: ['src/**/*.ts']
},
'tools/test-recorder': {
project: ['src/**/*.ts']
}
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"stylelint": "stylelint --cache --cache-strategy content '{apps,packages,src}/**/*.{css,vue}'",
"comfy-test": "tsx tools/test-recorder/src/index.ts",
"test:assets-flag-artifact": "tsx scripts/checkAssetsFlagArtifact.ts",
"test:expected-failure-ratchet": "tsx tools/expected-failures/src/cli.ts",
"test:browser": "pnpm exec playwright test",
"test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 tsx scripts/test-browser-local.ts",
"custom-node-quarantine": "tsx scripts/customNodeQuarantine.ts",
Expand Down
16 changes: 16 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

90 changes: 90 additions & 0 deletions tools/expected-failures/baseline.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
[
{
"id": "playwright:browser_tests/tests/agent/agentLayoutQuality.spec.ts:draws every node without overlapping another:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts:<expression:'wipes a duplicated node and replaces it with a phantom node when ' + \"the agent's write lands on the same id\">:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts:<expression:\"leaves the agent's own graph read disagreeing with the \" + 'still-visible canvas after its write is silently dropped'>:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/agent/agentRateLimitDuplicateSubmit.spec.ts:keeps repeated Enter presses to one rejected send:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/agent/agentRefusedDocSubscribe.spec.ts:reports a canvas edit as succeeded when the document never arrived:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/agent/agentReplyDrafts.spec.ts:<expression:`a completed turn keeps its ${selection} approval action live`>:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/agent/agentTurnProgressIsLegible.spec.ts:names how long the finished turn took:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/agent/agentTwoSessionCrdtSubscription.spec.ts:returning to the displaced build shows it on the canvas:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts:<expression:`blocks POST ${path} and fails even when the app catches it`>:1",
"classification": "infrastructure-assertion"
},
{
"id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts:<expression:`${client} retains the API origin and mutation guards`>:1",
"classification": "infrastructure-assertion"
},
{
"id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts:retains the external WebSocket guard:1",
"classification": "infrastructure-assertion"
},
{
"id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:<expression:`${name} reports external requests at teardown`>:1",
"classification": "infrastructure-assertion"
},
{
"id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test even when an external fetch error is caught:1",
"classification": "infrastructure-assertion"
},
{
"id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an external WebSocket:1",
"classification": "infrastructure-assertion"
},
{
"id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an unmocked popup:1",
"classification": "infrastructure-assertion"
},
{
"id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:guards every API verb, including context.request:1",
"classification": "infrastructure-assertion"
},
{
"id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts:<expression:title>:1",
"classification": "live-defect"
},
{
"id": "playwright:browser_tests/tests/workspace/workflowDraftForeignWindowSignIn.spec.ts:a second window signing in neither wipes drafts nor stops persistence:1",
"classification": "live-defect"
},
{
"id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:releases a hovered node with the pointerup event:1",
"classification": "live-defect"
},
{
"id": "vitest:src/platform/workflow/persistence/base/hashUtil.test.ts:produces different hashes for known collision paths:1",
"classification": "live-defect"
},
{
"id": "vitest:src/stores/assetDownloadStore.test.ts:excludes a failed-then-actually-completed task from activeDownloads (PM-1302):1",
"classification": "live-defect"
},
{
"id": "vitest:src/workbench/extensions/agent/crdt/semanticLayoutSeparation.guard.test.ts:omits pos and size from a minted node payload:1",
"classification": "live-defect"
}
]
15 changes: 15 additions & 0 deletions tools/expected-failures/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"name": "@comfyorg/expected-failures",
"version": "0.1.0",
"private": true,
"description": "Inventory of Vitest and Playwright expected failures, ratcheted against a classified baseline",
"type": "module",
"dependencies": {
"typescript": "catalog:",
"zod": "catalog:"
},
"devDependencies": {
"@types/node": "catalog:",
"vitest": "catalog:"
}
}
29 changes: 29 additions & 0 deletions tools/expected-failures/src/cli.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import { readFile } from 'node:fs/promises'
import path from 'node:path'

import { buildInventory, compareBaseline } from './inventory'

const repositoryRoot = path.resolve(import.meta.dirname, '../../..')
const baselinePath = path.resolve(import.meta.dirname, '../baseline.json')

async function main(): Promise<number> {
const { errors, liveDefects, infrastructure } = compareBaseline(
await buildInventory(repositoryRoot),
JSON.parse(await readFile(baselinePath, 'utf8'))
)

if (errors.length === 0) {
process.stdout.write(
`Expected-failure baseline matches: ${liveDefects} live defects, ${infrastructure} infrastructure assertions.\n`
)
return 0
}

for (const error of errors) console.error(error)
console.error(
'Update tools/expected-failures/baseline.json: remove resolved entries and classify each addition as "live-defect" or "infrastructure-assertion".'
)
return 1
}

if (import.meta.main) process.exitCode = await main()
Loading
Loading