From 67203e4fe704e140bdbeaa7f5b0dca7614072fe1 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 4 Oct 2026 23:58:41 +0000 Subject: [PATCH 01/15] test: ratchet expected-failure inventory --- .github/workflows/ci-lint-format.yaml | 3 + package.json | 1 + scripts/cicd/expected-failure-baseline.json | 132 +++++++++++ scripts/cicd/expected-failure-ratchet.test.ts | 35 +++ scripts/cicd/expected-failure-ratchet.ts | 207 ++++++++++++++++++ 5 files changed, 378 insertions(+) create mode 100644 scripts/cicd/expected-failure-baseline.json create mode 100644 scripts/cicd/expected-failure-ratchet.test.ts create mode 100644 scripts/cicd/expected-failure-ratchet.ts diff --git a/.github/workflows/ci-lint-format.yaml b/.github/workflows/ci-lint-format.yaml index 6434861bbf6..044b20563e4 100644 --- a/.github/workflows/ci-lint-format.yaml +++ b/.github/workflows/ci-lint-format.yaml @@ -177,3 +177,6 @@ jobs: - name: Verify knip, locale integrity and ADR index uses: ./.github/actions/repo-checks + + - name: Prevent untracked expected failures + run: pnpm test:expected-failure-ratchet diff --git a/package.json b/package.json index ad060e9663a..be254189279 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "stylelint": "stylelint --cache --cache-strategy content '{apps,packages,src}/**/*.{css,vue}'", "comfy-test": "tsx tools/test-recorder/src/index.ts", "test:assets-flag-artifact": "tsx scripts/checkAssetsFlagArtifact.ts", + "test:expected-failure-ratchet": "tsx scripts/cicd/expected-failure-ratchet.ts", "test:browser": "pnpm exec playwright test", "test:browser:coverage": "cross-env COLLECT_COVERAGE=true pnpm test:browser", "test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 tsx scripts/test-browser-local.ts", diff --git a/scripts/cicd/expected-failure-baseline.json b/scripts/cicd/expected-failure-baseline.json new file mode 100644 index 00000000000..616003aa9a5 --- /dev/null +++ b/scripts/cicd/expected-failure-baseline.json @@ -0,0 +1,132 @@ +{ + "entries": [ + { + "id": "playwright:browser_tests/tests/agent/agentLayoutQuality.spec.ts:draws every node without overlapping another:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentNestedOutputSentPreview.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::2", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentRateLimitDuplicateSubmit.spec.ts:keeps repeated Enter presses to one rejected send:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentRefusedDocSubscribe.spec.ts:reports a canvas edit as succeeded when the document never arrived:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentReplyDrafts.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::2", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts:retains the external WebSocket guard:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test even when an external fetch error is caught:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an external WebSocket:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an unmocked popup:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:guards every API verb, including context.request:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/pointerGestures.spec.ts:node-title movement below ClickDrift is a click:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/pointerGestures.spec.ts:pressing an unselected unpinned node brings it to front before release:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/pointerGestures.spec.ts:slow group-title click selects without moving the group:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/selection.spec.ts:shift-toggling a group off also deselects its cascaded children:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/workspace/workflowDraftForeignWindowSignIn.spec.ts:a second window signing in neither wipes drafts nor stops persistence:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:releases a hovered node with the pointerup event:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:slow movement within the drift threshold stays a click:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:a clearing click reports one change:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:a replacing click reports one change:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:deselect() reports the change:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:live marquee reports one change:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:nested groups receive onSelected:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:select() reports the change:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:shift+alt marquee resolves the same way in both modes:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/platform/workflow/persistence/base/hashUtil.test.ts:produces different hashes for known collision paths:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/stores/assetDownloadStore.test.ts:excludes a failed-then-actually-completed task from activeDownloads (PM-1302):1", + "classification": "live-defect" + }, + { + "id": "vitest:src/workbench/extensions/agent/crdt/semanticLayoutSeparation.guard.test.ts:omits pos and size from a minted node payload:1", + "classification": "live-defect" + } + ] +} diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts new file mode 100644 index 00000000000..af6eb763d51 --- /dev/null +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest' + +import { inspectSource } from './expected-failure-ratchet' + +describe('expected failure inventory', () => { + it('finds Vitest and Playwright pins without matching inert text', () => { + const source = ` + // it.fails('commented out', () => {}) + const explanation = "test.fail() is mentioned" + it.fails('known unit defect', () => {}) + test('conditional browser defect', () => { + test.fail(!available) + }) + test.fail(true, 'declaration-level defect') + ` + + expect(inspectSource(source, 'example.test.ts')).toEqual([ + expect.objectContaining({ + id: 'vitest:example.test.ts:known unit defect:1', + kind: 'vitest', + title: 'known unit defect' + }), + expect.objectContaining({ + id: 'playwright:example.test.ts:conditional browser defect:1', + kind: 'playwright', + title: 'conditional browser defect' + }), + expect.objectContaining({ + id: 'playwright:example.test.ts::1', + kind: 'playwright', + title: '' + }) + ]) + }) +}) diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts new file mode 100644 index 00000000000..3d57b161f4b --- /dev/null +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -0,0 +1,207 @@ +import { readFile, readdir } from 'node:fs/promises' +import path from 'node:path' + +import ts from 'typescript' + +export type ExpectedFailureKind = 'playwright' | 'vitest' + +export interface ExpectedFailure { + id: string + kind: ExpectedFailureKind + file: string + line: number + title: string +} + +interface BaselineEntry { + id: string + classification: 'infrastructure-assertion' | 'live-defect' +} + +const SOURCE_ROOTS = ['browser_tests', 'src'] +const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx']) + +const propertyCall = ( + node: ts.CallExpression +): { object: string; property: string } | undefined => { + if (!ts.isPropertyAccessExpression(node.expression)) { + return + } + + return { + object: node.expression.expression.getText(), + property: node.expression.name.text + } +} + +const isTestExpression = (expression: ts.Expression): boolean => { + if (ts.isIdentifier(expression)) { + return expression.text === 'it' || expression.text === 'test' + } + if (!ts.isPropertyAccessExpression(expression)) return false + + const object = expression.expression.getText() + return ( + (object === 'it' || object === 'test') && + ['fixme', 'only', 'skip'].includes(expression.name.text) + ) +} + +const testTitle = (node: ts.Node): string => { + for ( + let parent = node.parent; + parent !== node.getSourceFile(); + parent = parent.parent + ) { + if (!ts.isCallExpression(parent)) continue + + if (!isTestExpression(parent.expression)) continue + + const title = parent.arguments.at(0) + if ( + title && + (ts.isStringLiteral(title) || ts.isNoSubstitutionTemplateLiteral(title)) + ) { + return title.text + } + } + + return '' +} + +const expectedFailureKind = ( + node: ts.CallExpression +): ExpectedFailureKind | undefined => { + const call = propertyCall(node) + if (call?.property === 'fails' && ['it', 'test'].includes(call.object)) { + return 'vitest' + } + if (call?.object === 'test' && call.property === 'fail') return 'playwright' +} + +const vitestTitle = (node: ts.CallExpression): string => { + const value = node.arguments.at(0) + return value && + (ts.isStringLiteral(value) || ts.isNoSubstitutionTemplateLiteral(value)) + ? value.text + : '' +} + +export const inspectSource = ( + source: string, + file: string +): ExpectedFailure[] => { + const sourceFile = ts.createSourceFile( + file, + source, + ts.ScriptTarget.Latest, + true + ) + const found: Omit[] = [] + + const visit = (node: ts.Node) => { + if (!ts.isCallExpression(node)) { + ts.forEachChild(node, visit) + return + } + + const kind = expectedFailureKind(node) + + if (kind) { + const position = sourceFile.getLineAndCharacterOfPosition(node.getStart()) + const title = kind === 'vitest' ? vitestTitle(node) : testTitle(node) + found.push({ kind, file, line: position.line + 1, title }) + } + + ts.forEachChild(node, visit) + } + visit(sourceFile) + + const occurrences = new Map() + return found.map((entry) => { + const fingerprint = `${entry.kind}:${entry.file}:${entry.title}` + const occurrence = (occurrences.get(fingerprint) ?? 0) + 1 + occurrences.set(fingerprint, occurrence) + return { ...entry, id: `${fingerprint}:${occurrence}` } + }) +} + +const sourceFiles = async (root: string): Promise => { + const entries = await readdir(root, { withFileTypes: true }) + const nested = await Promise.all( + entries.map(async (entry) => { + const entryPath = path.join(root, entry.name) + if (entry.isDirectory()) return sourceFiles(entryPath) + return SOURCE_EXTENSIONS.has(path.extname(entry.name)) ? [entryPath] : [] + }) + ) + return nested.flat() +} + +export const buildInventory = async ( + repositoryRoot = process.cwd() +): Promise => { + const files = ( + await Promise.all( + SOURCE_ROOTS.map((root) => sourceFiles(path.join(repositoryRoot, root))) + ) + ).flat() + + return ( + await Promise.all( + files.map(async (absoluteFile) => { + const file = path.relative(repositoryRoot, absoluteFile) + return inspectSource(await readFile(absoluteFile, 'utf8'), file) + }) + ) + ) + .flat() + .sort((left, right) => left.id.localeCompare(right.id)) +} + +const run = async () => { + const inventory = await buildInventory() + if (process.argv.includes('--print')) { + console.log(JSON.stringify(inventory, null, 2)) + return + } + + const baselinePath = path.join( + process.cwd(), + 'scripts/cicd/expected-failure-baseline.json' + ) + const baseline = JSON.parse(await readFile(baselinePath, 'utf8')) as { + entries: BaselineEntry[] + } + const actualIds = new Set(inventory.map(({ id }) => id)) + const baselineIds = new Set(baseline.entries.map(({ id }) => id)) + const added = inventory.filter(({ id }) => !baselineIds.has(id)) + const removed = baseline.entries.filter(({ id }) => !actualIds.has(id)) + + if (added.length === 0 && removed.length === 0) { + const liveDefects = baseline.entries.filter( + ({ classification }) => classification === 'live-defect' + ).length + console.log( + `Expected-failure baseline matches: ${liveDefects} live defects, ${baseline.entries.length - liveDefects} infrastructure assertions.` + ) + return + } + + for (const entry of added) { + console.error( + `Unclassified expected failure: ${entry.file}:${entry.line} — ${entry.title}` + ) + } + for (const entry of removed) { + console.error( + `Resolved or renamed expected failure still in baseline: ${entry.id}` + ) + } + console.error( + 'Classify additions and remove resolved entries in expected-failure-baseline.json.' + ) + process.exitCode = 1 +} + +if (import.meta.url === `file://${process.argv[1]}`) await run() From 4373b068c3c4b85d6deee63d6f3eb60a1ca2407f Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Oct 2026 05:09:41 +0000 Subject: [PATCH 02/15] fix: close expected-failure ratchet gaps --- scripts/cicd/expected-failure-baseline.json | 6 +- scripts/cicd/expected-failure-ratchet.test.ts | 58 +++++- scripts/cicd/expected-failure-ratchet.ts | 186 ++++++++++++------ 3 files changed, 175 insertions(+), 75 deletions(-) diff --git a/scripts/cicd/expected-failure-baseline.json b/scripts/cicd/expected-failure-baseline.json index 616003aa9a5..a9aef8623fd 100644 --- a/scripts/cicd/expected-failure-baseline.json +++ b/scripts/cicd/expected-failure-baseline.json @@ -5,7 +5,7 @@ "classification": "live-defect" }, { - "id": "playwright:browser_tests/tests/agent/agentNestedOutputSentPreview.spec.ts::1", + "id": "playwright:browser_tests/tests/agent/agentNestedOutputSentPreview.spec.ts:sends a nested video output by its display name, not its asset id:1", "classification": "live-defect" }, { @@ -40,6 +40,10 @@ "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts:retains the external WebSocket guard:1", "classification": "infrastructure-assertion" }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts::1", + "classification": "infrastructure-assertion" + }, { "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test even when an external fetch error is caught:1", "classification": "infrastructure-assertion" diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index af6eb763d51..9462b2aa622 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -12,24 +12,62 @@ describe('expected failure inventory', () => { test.fail(!available) }) test.fail(true, 'declaration-level defect') + test.fail.only('focused browser defect', () => {}) + fixture.fail(true, 'fixture teardown defect') + it.skip.fails('skipped unit defect', () => {}) + it.fails.each([1])('parameterized unit defect', () => {}) ` - expect(inspectSource(source, 'example.test.ts')).toEqual([ - expect.objectContaining({ - id: 'vitest:example.test.ts:known unit defect:1', + expect(inspectSource(source, 'browser_tests/example.spec.ts')).toEqual([ + { + id: 'vitest:browser_tests/example.spec.ts:known unit defect:1', kind: 'vitest', + file: 'browser_tests/example.spec.ts', + line: 4, title: 'known unit defect' - }), - expect.objectContaining({ - id: 'playwright:example.test.ts:conditional browser defect:1', + }, + { + id: 'playwright:browser_tests/example.spec.ts:conditional browser defect:1', kind: 'playwright', + file: 'browser_tests/example.spec.ts', + line: 6, title: 'conditional browser defect' - }), - expect.objectContaining({ - id: 'playwright:example.test.ts::1', + }, + { + id: 'playwright:browser_tests/example.spec.ts::1', kind: 'playwright', + file: 'browser_tests/example.spec.ts', + line: 8, title: '' - }) + }, + { + id: 'playwright:browser_tests/example.spec.ts:focused browser defect:1', + kind: 'playwright', + file: 'browser_tests/example.spec.ts', + line: 9, + title: 'focused browser defect' + }, + { + id: 'playwright:browser_tests/example.spec.ts::2', + kind: 'playwright', + file: 'browser_tests/example.spec.ts', + line: 10, + title: '' + }, + { + id: 'vitest:browser_tests/example.spec.ts:skipped unit defect:1', + kind: 'vitest', + file: 'browser_tests/example.spec.ts', + line: 11, + title: 'skipped unit defect' + }, + { + id: 'vitest:browser_tests/example.spec.ts:parameterized unit defect:1', + kind: 'vitest', + file: 'browser_tests/example.spec.ts', + line: 12, + title: 'parameterized unit defect' + } ]) }) }) diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index 3d57b161f4b..3fd233e77d9 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -2,10 +2,13 @@ import { readFile, readdir } from 'node:fs/promises' import path from 'node:path' import ts from 'typescript' +import { z } from 'zod' -export type ExpectedFailureKind = 'playwright' | 'vitest' +import { isMainModule } from '../isMainModule' -export interface ExpectedFailure { +type ExpectedFailureKind = 'playwright' | 'vitest' + +interface ExpectedFailure { id: string kind: ExpectedFailureKind file: string @@ -13,28 +16,19 @@ export interface ExpectedFailure { title: string } -interface BaselineEntry { - id: string - classification: 'infrastructure-assertion' | 'live-defect' -} +const baselineEntrySchema = z.object({ + id: z.string().min(1), + classification: z.enum(['infrastructure-assertion', 'live-defect']) +}) +const baselineSchema = z.object({ entries: z.array(baselineEntrySchema) }) +type BaselineEntry = z.infer + +const LIVE_DEFECT_CEILING = 25 const SOURCE_ROOTS = ['browser_tests', 'src'] const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx']) -const propertyCall = ( - node: ts.CallExpression -): { object: string; property: string } | undefined => { - if (!ts.isPropertyAccessExpression(node.expression)) { - return - } - - return { - object: node.expression.expression.getText(), - property: node.expression.name.text - } -} - -const isTestExpression = (expression: ts.Expression): boolean => { +function isTestExpression(expression: ts.Expression): boolean { if (ts.isIdentifier(expression)) { return expression.text === 'it' || expression.text === 'test' } @@ -47,7 +41,26 @@ const isTestExpression = (expression: ts.Expression): boolean => { ) } -const testTitle = (node: ts.Node): string => { +function literalTitle(node: ts.Expression | undefined): string | undefined { + return node && + (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) + ? node.text + : undefined +} + +function hasTestBody(node: ts.CallExpression): boolean { + const body = node.arguments.at(-1) + return Boolean( + body && (ts.isArrowFunction(body) || ts.isFunctionExpression(body)) + ) +} + +function testTitle(node: ts.CallExpression): string { + if (hasTestBody(node)) { + const declaredTitle = literalTitle(node.arguments.at(0)) + if (declaredTitle) return declaredTitle + } + for ( let parent = node.parent; parent !== node.getSourceFile(); @@ -57,40 +70,64 @@ const testTitle = (node: ts.Node): string => { if (!isTestExpression(parent.expression)) continue - const title = parent.arguments.at(0) - if ( - title && - (ts.isStringLiteral(title) || ts.isNoSubstitutionTemplateLiteral(title)) - ) { - return title.text - } + const title = literalTitle(parent.arguments.at(0)) + if (title) return title } return '' } -const expectedFailureKind = ( - node: ts.CallExpression -): ExpectedFailureKind | undefined => { - const call = propertyCall(node) - if (call?.property === 'fails' && ['it', 'test'].includes(call.object)) { - return 'vitest' +function callChain(expression: ts.Expression): { + root?: string + properties: string[] +} { + if (ts.isIdentifier(expression)) { + return { root: expression.text, properties: [] } } - if (call?.object === 'test' && call.property === 'fail') return 'playwright' + if (ts.isCallExpression(expression)) return callChain(expression.expression) + if (!ts.isPropertyAccessExpression(expression)) return { properties: [] } + + const chain = callChain(expression.expression) + return { ...chain, properties: [...chain.properties, expression.name.text] } +} + +function isVitestFailure(root: string | undefined, properties: string[]) { + return ( + root !== undefined && + ['it', 'test'].includes(root) && + properties.includes('fails') + ) } -const vitestTitle = (node: ts.CallExpression): string => { - const value = node.arguments.at(0) - return value && - (ts.isStringLiteral(value) || ts.isNoSubstitutionTemplateLiteral(value)) - ? value.text - : '' +function isPlaywrightFailure( + root: string | undefined, + properties: string[], + file: string +) { + return ( + root !== undefined && + properties.includes('fail') && + (root === 'test' || file.startsWith('browser_tests/')) + ) } -export const inspectSource = ( - source: string, +function expectedFailureKind( + node: ts.CallExpression, file: string -): ExpectedFailure[] => { +): ExpectedFailureKind | undefined { + if (ts.isCallExpression(node.parent) && node.parent.expression === node) + return + + const { root, properties } = callChain(node.expression) + if (isVitestFailure(root, properties)) return 'vitest' + if (isPlaywrightFailure(root, properties, file)) return 'playwright' +} + +function vitestTitle(node: ts.CallExpression): string { + return literalTitle(node.arguments.at(0)) ?? '' +} + +export function inspectSource(source: string, file: string): ExpectedFailure[] { const sourceFile = ts.createSourceFile( file, source, @@ -99,13 +136,13 @@ export const inspectSource = ( ) const found: Omit[] = [] - const visit = (node: ts.Node) => { + function visit(node: ts.Node) { if (!ts.isCallExpression(node)) { ts.forEachChild(node, visit) return } - const kind = expectedFailureKind(node) + const kind = expectedFailureKind(node, file) if (kind) { const position = sourceFile.getLineAndCharacterOfPosition(node.getStart()) @@ -126,7 +163,7 @@ export const inspectSource = ( }) } -const sourceFiles = async (root: string): Promise => { +async function sourceFiles(root: string): Promise { const entries = await readdir(root, { withFileTypes: true }) const nested = await Promise.all( entries.map(async (entry) => { @@ -138,9 +175,9 @@ const sourceFiles = async (root: string): Promise => { return nested.flat() } -export const buildInventory = async ( +async function buildInventory( repositoryRoot = process.cwd() -): Promise => { +): Promise { const files = ( await Promise.all( SOURCE_ROOTS.map((root) => sourceFiles(path.join(repositoryRoot, root))) @@ -150,7 +187,10 @@ export const buildInventory = async ( return ( await Promise.all( files.map(async (absoluteFile) => { - const file = path.relative(repositoryRoot, absoluteFile) + const file = path + .relative(repositoryRoot, absoluteFile) + .split(path.sep) + .join('/') return inspectSource(await readFile(absoluteFile, 'utf8'), file) }) ) @@ -159,7 +199,16 @@ export const buildInventory = async ( .sort((left, right) => left.id.localeCompare(right.id)) } -const run = async () => { +function validateBaseline(value: unknown): BaselineEntry[] { + const entries = baselineSchema.parse(value).entries + const ids = entries.map(({ id }) => id) + if (new Set(ids).size !== ids.length) { + throw new Error('Expected-failure baseline contains duplicate IDs.') + } + return entries +} + +async function run() { const inventory = await buildInventory() if (process.argv.includes('--print')) { console.log(JSON.stringify(inventory, null, 2)) @@ -170,27 +219,31 @@ const run = async () => { process.cwd(), 'scripts/cicd/expected-failure-baseline.json' ) - const baseline = JSON.parse(await readFile(baselinePath, 'utf8')) as { - entries: BaselineEntry[] - } + const baseline = validateBaseline( + JSON.parse(await readFile(baselinePath, 'utf8')) as unknown + ) const actualIds = new Set(inventory.map(({ id }) => id)) - const baselineIds = new Set(baseline.entries.map(({ id }) => id)) + const baselineIds = new Set(baseline.map(({ id }) => id)) const added = inventory.filter(({ id }) => !baselineIds.has(id)) - const removed = baseline.entries.filter(({ id }) => !actualIds.has(id)) + const removed = baseline.filter(({ id }) => !actualIds.has(id)) + const liveDefects = baseline.filter( + ({ classification }) => classification === 'live-defect' + ).length - if (added.length === 0 && removed.length === 0) { - const liveDefects = baseline.entries.filter( - ({ classification }) => classification === 'live-defect' - ).length + if ( + added.length === 0 && + removed.length === 0 && + liveDefects <= LIVE_DEFECT_CEILING + ) { console.log( - `Expected-failure baseline matches: ${liveDefects} live defects, ${baseline.entries.length - liveDefects} infrastructure assertions.` + `Expected-failure baseline matches: ${liveDefects} live defects, ${baseline.length - liveDefects} infrastructure assertions.` ) return } for (const entry of added) { console.error( - `Unclassified expected failure: ${entry.file}:${entry.line} — ${entry.title}` + `Unclassified expected failure: ${entry.file}:${entry.line} — ${entry.id}` ) } for (const entry of removed) { @@ -198,10 +251,15 @@ const run = async () => { `Resolved or renamed expected failure still in baseline: ${entry.id}` ) } + if (liveDefects > LIVE_DEFECT_CEILING) { + console.error( + `Live-defect count ${liveDefects} exceeds the fixed ceiling ${LIVE_DEFECT_CEILING}.` + ) + } console.error( - 'Classify additions and remove resolved entries in expected-failure-baseline.json.' + 'Update scripts/cicd/expected-failure-baseline.json: remove resolved entries and classify each addition as "live-defect" or "infrastructure-assertion".' ) process.exitCode = 1 } -if (import.meta.url === `file://${process.argv[1]}`) await run() +if (isMainModule(import.meta.url)) await run() From e6576aa216edf77e35570f889e877d75ccc7b2f6 Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Oct 2026 07:43:45 +0000 Subject: [PATCH 03/15] fix: recognize playwright fixture runners --- scripts/cicd/expected-failure-ratchet.test.ts | 26 +++- scripts/cicd/expected-failure-ratchet.ts | 127 +++++++++++++++--- 2 files changed, 133 insertions(+), 20 deletions(-) diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index 9462b2aa622..18d4a2dcdd6 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -13,6 +13,7 @@ describe('expected failure inventory', () => { }) test.fail(true, 'declaration-level defect') test.fail.only('focused browser defect', () => {}) + const fixture = test.extend({}) fixture.fail(true, 'fixture teardown defect') it.skip.fails('skipped unit defect', () => {}) it.fails.each([1])('parameterized unit defect', () => {}) @@ -51,23 +52,42 @@ describe('expected failure inventory', () => { id: 'playwright:browser_tests/example.spec.ts::2', kind: 'playwright', file: 'browser_tests/example.spec.ts', - line: 10, + line: 11, title: '' }, { id: 'vitest:browser_tests/example.spec.ts:skipped unit defect:1', kind: 'vitest', file: 'browser_tests/example.spec.ts', - line: 11, + line: 12, title: 'skipped unit defect' }, { id: 'vitest:browser_tests/example.spec.ts:parameterized unit defect:1', kind: 'vitest', file: 'browser_tests/example.spec.ts', - line: 12, + line: 13, title: 'parameterized unit defect' } ]) }) + + it('uses extended fixture declaration titles in inventory IDs', () => { + const inventoryId = (title: string) => + inspectSource( + ` + import { baseFixture } from '@e2e/fixtures/baseFixture' + const fixture = baseFixture.extend({}) + fixture('${title}', () => fixture.fail(true)) + `, + 'browser_tests/fixture.spec.ts' + )[0].id + + expect(inventoryId('old title')).toBe( + 'playwright:browser_tests/fixture.spec.ts:old title:1' + ) + expect(inventoryId('new title')).toBe( + 'playwright:browser_tests/fixture.spec.ts:new title:1' + ) + }) }) diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index 3fd233e77d9..0d58b81c5a8 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -28,15 +28,17 @@ const LIVE_DEFECT_CEILING = 25 const SOURCE_ROOTS = ['browser_tests', 'src'] const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx']) -function isTestExpression(expression: ts.Expression): boolean { - if (ts.isIdentifier(expression)) { - return expression.text === 'it' || expression.text === 'test' - } +function isTestExpression( + expression: ts.Expression, + playwrightRunners: ReadonlySet +): boolean { + if (ts.isIdentifier(expression)) return playwrightRunners.has(expression.text) if (!ts.isPropertyAccessExpression(expression)) return false - const object = expression.expression.getText() + const { root } = callChain(expression) return ( - (object === 'it' || object === 'test') && + root !== undefined && + playwrightRunners.has(root) && ['fixme', 'only', 'skip'].includes(expression.name.text) ) } @@ -55,7 +57,11 @@ function hasTestBody(node: ts.CallExpression): boolean { ) } -function testTitle(node: ts.CallExpression): string { +function testTitle( + node: ts.CallExpression, + runner: string, + playwrightRunners: ReadonlySet +): string { if (hasTestBody(node)) { const declaredTitle = literalTitle(node.arguments.at(0)) if (declaredTitle) return declaredTitle @@ -68,7 +74,12 @@ function testTitle(node: ts.CallExpression): string { ) { if (!ts.isCallExpression(parent)) continue - if (!isTestExpression(parent.expression)) continue + const { root } = callChain(parent.expression) + if ( + root !== runner || + !isTestExpression(parent.expression, playwrightRunners) + ) + continue const title = literalTitle(parent.arguments.at(0)) if (title) return title @@ -91,6 +102,82 @@ function callChain(expression: ts.Expression): { return { ...chain, properties: [...chain.properties, expression.name.text] } } +function importedPlaywrightRunners(node: ts.Node): string[] { + if ( + !ts.isImportDeclaration(node) || + !ts.isStringLiteral(node.moduleSpecifier) || + !node.moduleSpecifier.text.includes('/fixtures/') + ) + return [] + const bindings = node.importClause?.namedBindings + if (!bindings || !ts.isNamedImports(bindings)) return [] + return bindings.elements + .map(({ name }) => name.text) + .filter((name) => /(?:fixture|test)$/i.test(name)) +} + +function extendedPlaywrightRunner( + node: ts.Node, + runners: ReadonlySet +): string | undefined { + if ( + !ts.isVariableDeclaration(node) || + !ts.isIdentifier(node.name) || + !node.initializer || + !ts.isCallExpression(node.initializer) + ) + return + const { root, properties } = callChain(node.initializer.expression) + if (root && runners.has(root) && properties.includes('extend')) + return node.name.text +} + +function forOfRunnerAliases( + node: ts.Node, + runners: ReadonlySet +): string[] { + if (!ts.isForOfStatement(node)) return [] + const valuesExpression = ts.isAsExpression(node.expression) + ? node.expression.expression + : node.expression + if ( + !ts.isVariableDeclarationList(node.initializer) || + !ts.isArrayLiteralExpression(valuesExpression) + ) + return [] + const binding = node.initializer.declarations[0].name + if (!ts.isArrayBindingPattern(binding)) return [] + + return binding.elements.flatMap((element, index) => { + if (!ts.isBindingElement(element) || !ts.isIdentifier(element.name)) + return [] + const values = valuesExpression.elements.map((value) => + ts.isArrayLiteralExpression(value) ? value.elements[index] : null + ) + return values.length > 0 && + values.every( + (value) => value && ts.isIdentifier(value) && runners.has(value.text) + ) + ? [element.name.text] + : [] + }) +} + +function playwrightRunnerNames(sourceFile: ts.SourceFile): Set { + const runners = new Set(['test']) + + function visit(node: ts.Node) { + importedPlaywrightRunners(node).forEach((name) => runners.add(name)) + const extendedRunner = extendedPlaywrightRunner(node, runners) + if (extendedRunner) runners.add(extendedRunner) + forOfRunnerAliases(node, runners).forEach((name) => runners.add(name)) + ts.forEachChild(node, visit) + } + + visit(sourceFile) + return runners +} + function isVitestFailure(root: string | undefined, properties: string[]) { return ( root !== undefined && @@ -102,25 +189,26 @@ function isVitestFailure(root: string | undefined, properties: string[]) { function isPlaywrightFailure( root: string | undefined, properties: string[], - file: string + playwrightRunners: ReadonlySet ) { return ( root !== undefined && properties.includes('fail') && - (root === 'test' || file.startsWith('browser_tests/')) + playwrightRunners.has(root) ) } function expectedFailureKind( node: ts.CallExpression, - file: string + playwrightRunners: ReadonlySet ): ExpectedFailureKind | undefined { if (ts.isCallExpression(node.parent) && node.parent.expression === node) return const { root, properties } = callChain(node.expression) if (isVitestFailure(root, properties)) return 'vitest' - if (isPlaywrightFailure(root, properties, file)) return 'playwright' + if (isPlaywrightFailure(root, properties, playwrightRunners)) + return 'playwright' } function vitestTitle(node: ts.CallExpression): string { @@ -134,6 +222,7 @@ export function inspectSource(source: string, file: string): ExpectedFailure[] { ts.ScriptTarget.Latest, true ) + const playwrightRunners = playwrightRunnerNames(sourceFile) const found: Omit[] = [] function visit(node: ts.Node) { @@ -142,11 +231,15 @@ export function inspectSource(source: string, file: string): ExpectedFailure[] { return } - const kind = expectedFailureKind(node, file) + const kind = expectedFailureKind(node, playwrightRunners) if (kind) { const position = sourceFile.getLineAndCharacterOfPosition(node.getStart()) - const title = kind === 'vitest' ? vitestTitle(node) : testTitle(node) + const { root } = callChain(node.expression) + const title = + kind === 'vitest' + ? vitestTitle(node) + : testTitle(node, root!, playwrightRunners) found.push({ kind, file, line: position.line + 1, title }) } @@ -233,7 +326,7 @@ async function run() { if ( added.length === 0 && removed.length === 0 && - liveDefects <= LIVE_DEFECT_CEILING + liveDefects === LIVE_DEFECT_CEILING ) { console.log( `Expected-failure baseline matches: ${liveDefects} live defects, ${baseline.length - liveDefects} infrastructure assertions.` @@ -251,9 +344,9 @@ async function run() { `Resolved or renamed expected failure still in baseline: ${entry.id}` ) } - if (liveDefects > LIVE_DEFECT_CEILING) { + if (liveDefects !== LIVE_DEFECT_CEILING) { console.error( - `Live-defect count ${liveDefects} exceeds the fixed ceiling ${LIVE_DEFECT_CEILING}.` + `Live-defect count ${liveDefects} must equal LIVE_DEFECT_CEILING (${LIVE_DEFECT_CEILING}); lower the ceiling when a pin is fixed.` ) } console.error( From 6c71190273007e6523bebe705cb2c2d23027279f Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Oct 2026 10:09:36 +0000 Subject: [PATCH 04/15] fix: cover all expected-failure runners --- scripts/cicd/expected-failure-ratchet.test.ts | 36 +++++++++ scripts/cicd/expected-failure-ratchet.ts | 80 ++++++++++++++----- 2 files changed, 98 insertions(+), 18 deletions(-) diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index 18d4a2dcdd6..9167c873bec 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -90,4 +90,40 @@ describe('expected failure inventory', () => { 'playwright:browser_tests/fixture.spec.ts:new title:1' ) }) + + it('recognizes aliased, extended, and merged Playwright runners', () => { + const source = ` + import { baseFixture as runner } from '@e2e/fixtures/baseFixture' + const extended = runner.extend({}) + const merged = mergeTests(extended, anotherFixture) + merged('known defect', () => merged.fail(true)) + ` + + expect(inspectSource(source, 'browser_tests/fixture.spec.ts')).toEqual([ + { + id: 'playwright:browser_tests/fixture.spec.ts:known defect:1', + kind: 'playwright', + file: 'browser_tests/fixture.spec.ts', + line: 5, + title: 'known defect' + } + ]) + }) + + it('does not classify extended Vitest runners as Playwright failures', () => { + const source = ` + const customTest = test.extend({}) + customTest.fails('known defect', () => {}) + ` + + expect(inspectSource(source, 'src/example.test.ts')).toEqual([ + { + id: 'vitest:src/example.test.ts:known defect:1', + kind: 'vitest', + file: 'src/example.test.ts', + line: 3, + title: 'known defect' + } + ]) + }) }) diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index 0d58b81c5a8..a45e422d1d0 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -25,8 +25,16 @@ type BaselineEntry = z.infer const LIVE_DEFECT_CEILING = 25 -const SOURCE_ROOTS = ['browser_tests', 'src'] -const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx']) +const SOURCE_ROOTS = [ + 'apps', + 'browser_tests', + 'build', + 'packages', + 'scripts', + 'src', + 'tools' +] +const SOURCE_EXTENSIONS = new Set(['.cts', '.mts', '.ts', '.tsx']) function isTestExpression( expression: ts.Expression, @@ -59,7 +67,7 @@ function hasTestBody(node: ts.CallExpression): boolean { function testTitle( node: ts.CallExpression, - runner: string, + runner: string | undefined, playwrightRunners: ReadonlySet ): string { if (hasTestBody(node)) { @@ -112,8 +120,25 @@ function importedPlaywrightRunners(node: ts.Node): string[] { const bindings = node.importClause?.namedBindings if (!bindings || !ts.isNamedImports(bindings)) return [] return bindings.elements + .filter(({ propertyName, name }) => + /(?:fixture|test)$/i.test((propertyName ?? name).text) + ) .map(({ name }) => name.text) - .filter((name) => /(?:fixture|test)$/i.test(name)) +} + +function createsRunner( + initializer: ts.CallExpression, + runners: ReadonlySet +): boolean { + const { root, properties } = callChain(initializer.expression) + if (root !== undefined && runners.has(root) && properties.includes('extend')) + return true + return ( + root === 'mergeTests' && + initializer.arguments.some( + (argument) => ts.isIdentifier(argument) && runners.has(argument.text) + ) + ) } function extendedPlaywrightRunner( @@ -121,14 +146,12 @@ function extendedPlaywrightRunner( runners: ReadonlySet ): string | undefined { if ( - !ts.isVariableDeclaration(node) || - !ts.isIdentifier(node.name) || - !node.initializer || - !ts.isCallExpression(node.initializer) + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.initializer && + ts.isCallExpression(node.initializer) && + createsRunner(node.initializer, runners) ) - return - const { root, properties } = callChain(node.initializer.expression) - if (root && runners.has(root) && properties.includes('extend')) return node.name.text } @@ -178,10 +201,27 @@ function playwrightRunnerNames(sourceFile: ts.SourceFile): Set { return runners } -function isVitestFailure(root: string | undefined, properties: string[]) { +function vitestRunnerNames(sourceFile: ts.SourceFile): Set { + const runners = new Set(['it', 'test']) + + function visit(node: ts.Node) { + const extendedRunner = extendedPlaywrightRunner(node, runners) + if (extendedRunner) runners.add(extendedRunner) + ts.forEachChild(node, visit) + } + + visit(sourceFile) + return runners +} + +function isVitestFailure( + root: string | undefined, + properties: string[], + vitestRunners: ReadonlySet +) { return ( root !== undefined && - ['it', 'test'].includes(root) && + vitestRunners.has(root) && properties.includes('fails') ) } @@ -200,13 +240,14 @@ function isPlaywrightFailure( function expectedFailureKind( node: ts.CallExpression, - playwrightRunners: ReadonlySet + playwrightRunners: ReadonlySet, + vitestRunners: ReadonlySet ): ExpectedFailureKind | undefined { if (ts.isCallExpression(node.parent) && node.parent.expression === node) return const { root, properties } = callChain(node.expression) - if (isVitestFailure(root, properties)) return 'vitest' + if (isVitestFailure(root, properties, vitestRunners)) return 'vitest' if (isPlaywrightFailure(root, properties, playwrightRunners)) return 'playwright' } @@ -222,7 +263,10 @@ export function inspectSource(source: string, file: string): ExpectedFailure[] { ts.ScriptTarget.Latest, true ) - const playwrightRunners = playwrightRunnerNames(sourceFile) + const playwrightRunners = file.startsWith('browser_tests/') + ? playwrightRunnerNames(sourceFile) + : new Set() + const vitestRunners = vitestRunnerNames(sourceFile) const found: Omit[] = [] function visit(node: ts.Node) { @@ -231,7 +275,7 @@ export function inspectSource(source: string, file: string): ExpectedFailure[] { return } - const kind = expectedFailureKind(node, playwrightRunners) + const kind = expectedFailureKind(node, playwrightRunners, vitestRunners) if (kind) { const position = sourceFile.getLineAndCharacterOfPosition(node.getStart()) @@ -239,7 +283,7 @@ export function inspectSource(source: string, file: string): ExpectedFailure[] { const title = kind === 'vitest' ? vitestTitle(node) - : testTitle(node, root!, playwrightRunners) + : testTitle(node, root, playwrightRunners) found.push({ kind, file, line: position.line + 1, title }) } From a69d6d59d617f162dacf1fd4c7206a930bf55778 Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Oct 2026 15:07:09 +0000 Subject: [PATCH 05/15] fix: detect app Playwright failure pins --- scripts/cicd/expected-failure-ratchet.test.ts | 24 +++++++++++++++++++ scripts/cicd/expected-failure-ratchet.ts | 8 +++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index 9167c873bec..fa499fb3587 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -110,6 +110,30 @@ describe('expected failure inventory', () => { ]) }) + it('recognizes Playwright failures in app e2e suites', () => { + const source = ` + test('known app defect', () => test.fail(true)) + test.fail('declared app defect', () => {}) + ` + + expect(inspectSource(source, 'apps/website/e2e/example.spec.ts')).toEqual([ + { + id: 'playwright:apps/website/e2e/example.spec.ts:known app defect:1', + kind: 'playwright', + file: 'apps/website/e2e/example.spec.ts', + line: 2, + title: 'known app defect' + }, + { + id: 'playwright:apps/website/e2e/example.spec.ts:declared app defect:1', + kind: 'playwright', + file: 'apps/website/e2e/example.spec.ts', + line: 3, + title: 'declared app defect' + } + ]) + }) + it('does not classify extended Vitest runners as Playwright failures', () => { const source = ` const customTest = test.extend({}) diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index a45e422d1d0..15c26235d57 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -141,7 +141,7 @@ function createsRunner( ) } -function extendedPlaywrightRunner( +function derivedRunner( node: ts.Node, runners: ReadonlySet ): string | undefined { @@ -191,7 +191,7 @@ function playwrightRunnerNames(sourceFile: ts.SourceFile): Set { function visit(node: ts.Node) { importedPlaywrightRunners(node).forEach((name) => runners.add(name)) - const extendedRunner = extendedPlaywrightRunner(node, runners) + const extendedRunner = derivedRunner(node, runners) if (extendedRunner) runners.add(extendedRunner) forOfRunnerAliases(node, runners).forEach((name) => runners.add(name)) ts.forEachChild(node, visit) @@ -205,7 +205,7 @@ function vitestRunnerNames(sourceFile: ts.SourceFile): Set { const runners = new Set(['it', 'test']) function visit(node: ts.Node) { - const extendedRunner = extendedPlaywrightRunner(node, runners) + const extendedRunner = derivedRunner(node, runners) if (extendedRunner) runners.add(extendedRunner) ts.forEachChild(node, visit) } @@ -263,7 +263,7 @@ export function inspectSource(source: string, file: string): ExpectedFailure[] { ts.ScriptTarget.Latest, true ) - const playwrightRunners = file.startsWith('browser_tests/') + const playwrightRunners = /^(?:browser_tests|apps\/[^/]+\/e2e)\//.test(file) ? playwrightRunnerNames(sourceFile) : new Set() const vitestRunners = vitestRunnerNames(sourceFile) From d44dee486635b64e068f9d44dd576e09720b6c9b Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Oct 2026 20:06:50 +0000 Subject: [PATCH 06/15] test: stabilize computed expected-failure titles --- scripts/cicd/expected-failure-baseline.json | 14 ++++----- scripts/cicd/expected-failure-ratchet.test.ts | 20 +++++++++++++ scripts/cicd/expected-failure-ratchet.ts | 29 ++++++++----------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/scripts/cicd/expected-failure-baseline.json b/scripts/cicd/expected-failure-baseline.json index a9aef8623fd..fbab1439bed 100644 --- a/scripts/cicd/expected-failure-baseline.json +++ b/scripts/cicd/expected-failure-baseline.json @@ -9,11 +9,11 @@ "classification": "live-defect" }, { - "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", + "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", "classification": "live-defect" }, { - "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::2", + "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", "classification": "live-defect" }, { @@ -25,15 +25,15 @@ "classification": "live-defect" }, { - "id": "playwright:browser_tests/tests/agent/agentReplyDrafts.spec.ts::1", + "id": "playwright:browser_tests/tests/agent/agentReplyDrafts.spec.ts::1", "classification": "live-defect" }, { - "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", "classification": "infrastructure-assertion" }, { - "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::2", + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", "classification": "infrastructure-assertion" }, { @@ -41,7 +41,7 @@ "classification": "infrastructure-assertion" }, { - "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts::1", + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts::1", "classification": "infrastructure-assertion" }, { @@ -61,7 +61,7 @@ "classification": "infrastructure-assertion" }, { - "id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts::1", + "id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts::1", "classification": "live-defect" }, { diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index fa499fb3587..3f936cbd556 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -91,6 +91,26 @@ describe('expected failure inventory', () => { ) }) + it('uses computed test titles instead of positional declaration IDs', () => { + const source = ` + for (const scenario of scenarios) { + test(\`rejects \${scenario.kind} collisions\`, () => { + test.fail(true) + }) + } + test(prefix + ' teardown', () => test.fail(true)) + ` + + expect( + inspectSource(source, 'browser_tests/computed-title.spec.ts').map( + ({ id }) => id + ) + ).toEqual([ + 'playwright:browser_tests/computed-title.spec.ts::1', + "playwright:browser_tests/computed-title.spec.ts::1" + ]) + }) + it('recognizes aliased, extended, and merged Playwright runners', () => { const source = ` import { baseFixture as runner } from '@e2e/fixtures/baseFixture' diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index 15c26235d57..ebdbe0772c2 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -51,11 +51,11 @@ function isTestExpression( ) } -function literalTitle(node: ts.Expression | undefined): string | undefined { - return node && - (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) - ? node.text - : undefined +function titleText(node: ts.Expression | undefined): string | undefined { + if (!node) return + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) + return node.text + return `` } function hasTestBody(node: ts.CallExpression): boolean { @@ -71,7 +71,7 @@ function testTitle( playwrightRunners: ReadonlySet ): string { if (hasTestBody(node)) { - const declaredTitle = literalTitle(node.arguments.at(0)) + const declaredTitle = titleText(node.arguments.at(0)) if (declaredTitle) return declaredTitle } @@ -89,7 +89,7 @@ function testTitle( ) continue - const title = literalTitle(parent.arguments.at(0)) + const title = titleText(parent.arguments.at(0)) if (title) return title } @@ -191,8 +191,8 @@ function playwrightRunnerNames(sourceFile: ts.SourceFile): Set { function visit(node: ts.Node) { importedPlaywrightRunners(node).forEach((name) => runners.add(name)) - const extendedRunner = derivedRunner(node, runners) - if (extendedRunner) runners.add(extendedRunner) + const runner = derivedRunner(node, runners) + if (runner) runners.add(runner) forOfRunnerAliases(node, runners).forEach((name) => runners.add(name)) ts.forEachChild(node, visit) } @@ -205,8 +205,8 @@ function vitestRunnerNames(sourceFile: ts.SourceFile): Set { const runners = new Set(['it', 'test']) function visit(node: ts.Node) { - const extendedRunner = derivedRunner(node, runners) - if (extendedRunner) runners.add(extendedRunner) + const runner = derivedRunner(node, runners) + if (runner) runners.add(runner) ts.forEachChild(node, visit) } @@ -253,7 +253,7 @@ function expectedFailureKind( } function vitestTitle(node: ts.CallExpression): string { - return literalTitle(node.arguments.at(0)) ?? '' + return titleText(node.arguments.at(0)) ?? '' } export function inspectSource(source: string, file: string): ExpectedFailure[] { @@ -347,11 +347,6 @@ function validateBaseline(value: unknown): BaselineEntry[] { async function run() { const inventory = await buildInventory() - if (process.argv.includes('--print')) { - console.log(JSON.stringify(inventory, null, 2)) - return - } - const baselinePath = path.join( process.cwd(), 'scripts/cicd/expected-failure-baseline.json' From e2382f3b86d465f8e20ad37e09d52a89bbfccd6a Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Oct 2026 20:52:44 +0000 Subject: [PATCH 07/15] test: preserve callback-form failure titles Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20140#discussion_r4188278012 --- scripts/cicd/expected-failure-ratchet.test.ts | 19 +++++++++++++++++++ scripts/cicd/expected-failure-ratchet.ts | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index 3f936cbd556..3973bdf6d4b 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -111,6 +111,25 @@ describe('expected failure inventory', () => { ]) }) + it('uses the enclosing title for callback-form test.fail', () => { + const inventoryId = (title: string) => + inspectSource( + ` + test('${title}', () => { + test.fail(({ browserName }) => browserName === 'webkit') + }) + `, + 'browser_tests/callback-condition.spec.ts' + )[0].id + + expect(inventoryId('old title')).toBe( + 'playwright:browser_tests/callback-condition.spec.ts:old title:1' + ) + expect(inventoryId('new title')).toBe( + 'playwright:browser_tests/callback-condition.spec.ts:new title:1' + ) + }) + it('recognizes aliased, extended, and merged Playwright runners', () => { const source = ` import { baseFixture as runner } from '@e2e/fixtures/baseFixture' diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index ebdbe0772c2..d29a8661c81 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -70,7 +70,7 @@ function testTitle( runner: string | undefined, playwrightRunners: ReadonlySet ): string { - if (hasTestBody(node)) { + if (node.arguments.length > 1 && hasTestBody(node)) { const declaredTitle = titleText(node.arguments.at(0)) if (declaredTitle) return declaredTitle } From 08e88071aa49818f3b10d2251c88f69b56913457 Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Oct 2026 21:14:51 +0000 Subject: [PATCH 08/15] fix(ci): bound expected-failure source inspection --- scripts/cicd/expected-failure-ratchet.test.ts | 9 +++++ scripts/cicd/expected-failure-ratchet.ts | 33 ++++++++++++------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index 3973bdf6d4b..0fb6a49973a 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -3,6 +3,15 @@ import { describe, expect, it } from 'vitest' import { inspectSource } from './expected-failure-ratchet' describe('expected failure inventory', () => { + it('ignores test-shaped calls in production source files', () => { + expect( + inspectSource( + `const test = { fails: () => undefined }; test.fails('not a test')`, + 'src/test-support.ts' + ) + ).toEqual([]) + }) + it('finds Vitest and Playwright pins without matching inert text', () => { const source = ` // it.fails('commented out', () => {}) diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index d29a8661c81..7774f0b3e4a 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -35,6 +35,8 @@ const SOURCE_ROOTS = [ 'tools' ] const SOURCE_EXTENSIONS = new Set(['.cts', '.mts', '.ts', '.tsx']) +const TEST_FILE_PATTERN = /\.(?:spec|test)\.[cm]?tsx?$/ +const READ_BATCH_SIZE = 64 function isTestExpression( expression: ts.Expression, @@ -257,6 +259,8 @@ function vitestTitle(node: ts.CallExpression): string { } export function inspectSource(source: string, file: string): ExpectedFailure[] { + if (!TEST_FILE_PATTERN.test(file)) return [] + const sourceFile = ts.createSourceFile( file, source, @@ -321,19 +325,24 @@ async function buildInventory( ) ).flat() - return ( - await Promise.all( - files.map(async (absoluteFile) => { - const file = path - .relative(repositoryRoot, absoluteFile) - .split(path.sep) - .join('/') - return inspectSource(await readFile(absoluteFile, 'utf8'), file) - }) + const inventory: ExpectedFailure[] = [] + for (let index = 0; index < files.length; index += READ_BATCH_SIZE) { + const batch = files.slice(index, index + READ_BATCH_SIZE) + inventory.push( + ...( + await Promise.all( + batch.map(async (absoluteFile) => { + const file = path + .relative(repositoryRoot, absoluteFile) + .split(path.sep) + .join('/') + return inspectSource(await readFile(absoluteFile, 'utf8'), file) + }) + ) + ).flat() ) - ) - .flat() - .sort((left, right) => left.id.localeCompare(right.id)) + } + return inventory.sort((left, right) => left.id.localeCompare(right.id)) } function validateBaseline(value: unknown): BaselineEntry[] { From 0306aee68778256576f1de559a4725f3b13110a6 Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 5 Oct 2026 21:21:05 +0000 Subject: [PATCH 09/15] perf(ci): skip non-test files in failure scan --- scripts/cicd/expected-failure-ratchet.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index 7774f0b3e4a..efc03ac2175 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -34,7 +34,6 @@ const SOURCE_ROOTS = [ 'src', 'tools' ] -const SOURCE_EXTENSIONS = new Set(['.cts', '.mts', '.ts', '.tsx']) const TEST_FILE_PATTERN = /\.(?:spec|test)\.[cm]?tsx?$/ const READ_BATCH_SIZE = 64 @@ -310,7 +309,7 @@ async function sourceFiles(root: string): Promise { entries.map(async (entry) => { const entryPath = path.join(root, entry.name) if (entry.isDirectory()) return sourceFiles(entryPath) - return SOURCE_EXTENSIONS.has(path.extname(entry.name)) ? [entryPath] : [] + return TEST_FILE_PATTERN.test(entry.name) ? [entryPath] : [] }) ) return nested.flat() From 32ae6a8d02e9535738382b230fb7c98b1e21c696 Mon Sep 17 00:00:00 2001 From: bymyself Date: Tue, 13 Oct 2026 03:41:33 +0000 Subject: [PATCH 10/15] test(ci): refresh expected-failure baseline --- scripts/cicd/expected-failure-baseline.json | 28 ++++++++------------- scripts/cicd/expected-failure-ratchet.ts | 2 +- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/scripts/cicd/expected-failure-baseline.json b/scripts/cicd/expected-failure-baseline.json index fbab1439bed..621a1196110 100644 --- a/scripts/cicd/expected-failure-baseline.json +++ b/scripts/cicd/expected-failure-baseline.json @@ -1,11 +1,11 @@ { "entries": [ { - "id": "playwright:browser_tests/tests/agent/agentLayoutQuality.spec.ts:draws every node without overlapping another:1", + "id": "playwright:browser_tests/tests/agent/agentAttachFileTypeParity.spec.ts:gives the same answer for a .csv whether it is picked or dropped:1", "classification": "live-defect" }, { - "id": "playwright:browser_tests/tests/agent/agentNestedOutputSentPreview.spec.ts:sends a nested video output by its display name, not its asset id:1", + "id": "playwright:browser_tests/tests/agent/agentLayoutQuality.spec.ts:draws every node without overlapping another:1", "classification": "live-defect" }, { @@ -28,6 +28,14 @@ "id": "playwright:browser_tests/tests/agent/agentReplyDrafts.spec.ts::1", "classification": "live-defect" }, + { + "id": "playwright:browser_tests/tests/agent/agentTurnProgressIsLegible.spec.ts:names how long the finished turn took:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentTwoSessionCrdtSubscription.spec.ts:returning to the displaced build shows it on the canvas:1", + "classification": "live-defect" + }, { "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", "classification": "infrastructure-assertion" @@ -64,18 +72,6 @@ "id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts::1", "classification": "live-defect" }, - { - "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/pointerGestures.spec.ts:node-title movement below ClickDrift is a click:1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/pointerGestures.spec.ts:pressing an unselected unpinned node brings it to front before release:1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/pointerGestures.spec.ts:slow group-title click selects without moving the group:1", - "classification": "live-defect" - }, { "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/selection.spec.ts:shift-toggling a group off also deselects its cascaded children:1", "classification": "live-defect" @@ -88,10 +84,6 @@ "id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:releases a hovered node with the pointerup event:1", "classification": "live-defect" }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:slow movement within the drift threshold stays a click:1", - "classification": "live-defect" - }, { "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:a clearing click reports one change:1", "classification": "live-defect" diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index efc03ac2175..ba9dac751f0 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -23,7 +23,7 @@ const baselineEntrySchema = z.object({ const baselineSchema = z.object({ entries: z.array(baselineEntrySchema) }) type BaselineEntry = z.infer -const LIVE_DEFECT_CEILING = 25 +const LIVE_DEFECT_CEILING = 23 const SOURCE_ROOTS = [ 'apps', From 990f4dac1e7a5e7c19233d418fe0bc0bc2ee23ac Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 17 Oct 2026 03:32:39 +0000 Subject: [PATCH 11/15] test(ci): lower expected-failure ceiling after restored pins --- scripts/cicd/expected-failure-baseline.json | 36 --------------------- scripts/cicd/expected-failure-ratchet.ts | 2 +- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/scripts/cicd/expected-failure-baseline.json b/scripts/cicd/expected-failure-baseline.json index 621a1196110..37db7880ed4 100644 --- a/scripts/cicd/expected-failure-baseline.json +++ b/scripts/cicd/expected-failure-baseline.json @@ -1,9 +1,5 @@ { "entries": [ - { - "id": "playwright:browser_tests/tests/agent/agentAttachFileTypeParity.spec.ts:gives the same answer for a .csv whether it is picked or dropped:1", - "classification": "live-defect" - }, { "id": "playwright:browser_tests/tests/agent/agentLayoutQuality.spec.ts:draws every node without overlapping another:1", "classification": "live-defect" @@ -72,10 +68,6 @@ "id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts::1", "classification": "live-defect" }, - { - "id": "playwright:browser_tests/tests/vueNodes/interactions/canvas/selection.spec.ts:shift-toggling a group off also deselects its cascaded children:1", - "classification": "live-defect" - }, { "id": "playwright:browser_tests/tests/workspace/workflowDraftForeignWindowSignIn.spec.ts:a second window signing in neither wipes drafts nor stops persistence:1", "classification": "live-defect" @@ -84,34 +76,6 @@ "id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:releases a hovered node with the pointerup event:1", "classification": "live-defect" }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:a clearing click reports one change:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:a replacing click reports one change:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:deselect() reports the change:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:live marquee reports one change:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:nested groups receive onSelected:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:select() reports the change:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.selection.test.ts:shift+alt marquee resolves the same way in both modes:1", - "classification": "live-defect" - }, { "id": "vitest:src/platform/workflow/persistence/base/hashUtil.test.ts:produces different hashes for known collision paths:1", "classification": "live-defect" diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index ba9dac751f0..8556f6b2d43 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -23,7 +23,7 @@ const baselineEntrySchema = z.object({ const baselineSchema = z.object({ entries: z.array(baselineEntrySchema) }) type BaselineEntry = z.infer -const LIVE_DEFECT_CEILING = 23 +const LIVE_DEFECT_CEILING = 14 const SOURCE_ROOTS = [ 'apps', From ec6dda9db46af5e7e798963da9d64c51db6c9cde Mon Sep 17 00:00:00 2001 From: bymyself Date: Mon, 19 Oct 2026 03:35:36 +0000 Subject: [PATCH 12/15] test: cover expected-failure baseline comparison Extract the inventory/baseline comparison into compareBaseline so the added, removed, ceiling-equality, and duplicate-ID checks are exercised by a table of pure cases. --- scripts/cicd/expected-failure-ratchet.test.ts | 68 +++++++++++++++++- scripts/cicd/expected-failure-ratchet.ts | 71 +++++++++++-------- 2 files changed, 109 insertions(+), 30 deletions(-) diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index 0fb6a49973a..58eb5467b28 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { inspectSource } from './expected-failure-ratchet' +import { compareBaseline, inspectSource } from './expected-failure-ratchet' describe('expected failure inventory', () => { it('ignores test-shaped calls in production source files', () => { @@ -199,3 +199,69 @@ describe('expected failure inventory', () => { ]) }) }) + +describe('expected failure baseline comparison', () => { + const pin = (id: string) => ({ + id, + kind: 'vitest' as const, + file: 'src/example.test.ts', + line: 1, + title: id + }) + const entry = ( + id: string, + classification: 'live-defect' | 'infrastructure-assertion' = 'live-defect' + ) => ({ id, classification }) + + it.for([ + { + name: 'matches an exact baseline at the ceiling', + inventory: [pin('a'), pin('b')], + entries: [entry('a'), entry('b', 'infrastructure-assertion')], + ceiling: 1, + errors: [] + }, + { + name: 'rejects an unclassified pin', + inventory: [pin('a'), pin('new')], + entries: [entry('a')], + ceiling: 1, + errors: ['Unclassified expected failure: src/example.test.ts:1 — new'] + }, + { + name: 'rejects a resolved pin left in the baseline', + inventory: [pin('a')], + entries: [entry('a'), entry('fixed', 'infrastructure-assertion')], + ceiling: 1, + errors: ['Resolved or renamed expected failure still in baseline: fixed'] + }, + { + name: 'rejects a ceiling above the live-defect count', + inventory: [pin('a')], + entries: [entry('a')], + ceiling: 2, + errors: [ + 'Live-defect count 1 must equal LIVE_DEFECT_CEILING (2); lower the ceiling when a pin is fixed.' + ] + }, + { + name: 'rejects a ceiling below the live-defect count', + inventory: [pin('a'), pin('b')], + entries: [entry('a'), entry('b')], + ceiling: 1, + errors: [ + 'Live-defect count 2 must equal LIVE_DEFECT_CEILING (1); lower the ceiling when a pin is fixed.' + ] + } + ])('$name', ({ inventory, entries, ceiling, errors }) => { + expect(compareBaseline(inventory, { entries }, ceiling).errors).toEqual( + errors + ) + }) + + it('rejects duplicate baseline IDs', () => { + expect(() => + compareBaseline([pin('a')], { entries: [entry('a'), entry('a')] }, 2) + ).toThrow('duplicate IDs') + }) +}) diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index 8556f6b2d43..03bdd7e3bad 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -353,49 +353,62 @@ function validateBaseline(value: unknown): BaselineEntry[] { return entries } +export function compareBaseline( + inventory: ExpectedFailure[], + baselineValue: unknown, + ceiling = LIVE_DEFECT_CEILING +): { errors: string[]; liveDefects: number; infrastructure: number } { + const baseline = validateBaseline(baselineValue) + const actualIds = new Set(inventory.map(({ id }) => id)) + const baselineIds = new Set(baseline.map(({ id }) => id)) + const liveDefects = baseline.filter( + ({ classification }) => classification === 'live-defect' + ).length + const errors = [ + ...inventory + .filter(({ id }) => !baselineIds.has(id)) + .map( + (entry) => + `Unclassified expected failure: ${entry.file}:${entry.line} — ${entry.id}` + ), + ...baseline + .filter(({ id }) => !actualIds.has(id)) + .map( + ({ id }) => + `Resolved or renamed expected failure still in baseline: ${id}` + ) + ] + if (liveDefects !== ceiling) { + errors.push( + `Live-defect count ${liveDefects} must equal LIVE_DEFECT_CEILING (${ceiling}); lower the ceiling when a pin is fixed.` + ) + } + return { + errors, + liveDefects, + infrastructure: baseline.length - liveDefects + } +} + async function run() { const inventory = await buildInventory() const baselinePath = path.join( process.cwd(), 'scripts/cicd/expected-failure-baseline.json' ) - const baseline = validateBaseline( + const { errors, liveDefects, infrastructure } = compareBaseline( + inventory, JSON.parse(await readFile(baselinePath, 'utf8')) as unknown ) - const actualIds = new Set(inventory.map(({ id }) => id)) - const baselineIds = new Set(baseline.map(({ id }) => id)) - const added = inventory.filter(({ id }) => !baselineIds.has(id)) - const removed = baseline.filter(({ id }) => !actualIds.has(id)) - const liveDefects = baseline.filter( - ({ classification }) => classification === 'live-defect' - ).length - if ( - added.length === 0 && - removed.length === 0 && - liveDefects === LIVE_DEFECT_CEILING - ) { + if (errors.length === 0) { console.log( - `Expected-failure baseline matches: ${liveDefects} live defects, ${baseline.length - liveDefects} infrastructure assertions.` + `Expected-failure baseline matches: ${liveDefects} live defects, ${infrastructure} infrastructure assertions.` ) return } - for (const entry of added) { - console.error( - `Unclassified expected failure: ${entry.file}:${entry.line} — ${entry.id}` - ) - } - for (const entry of removed) { - console.error( - `Resolved or renamed expected failure still in baseline: ${entry.id}` - ) - } - if (liveDefects !== LIVE_DEFECT_CEILING) { - console.error( - `Live-defect count ${liveDefects} must equal LIVE_DEFECT_CEILING (${LIVE_DEFECT_CEILING}); lower the ceiling when a pin is fixed.` - ) - } + for (const error of errors) console.error(error) console.error( 'Update scripts/cicd/expected-failure-baseline.json: remove resolved entries and classify each addition as "live-defect" or "infrastructure-assertion".' ) From a7d34e0bb469efcfb1e70548c65146e071082630 Mon Sep 17 00:00:00 2001 From: bymyself Date: Tue, 20 Oct 2026 03:35:58 +0000 Subject: [PATCH 13/15] test: name both actions in the live-defect ceiling message --- scripts/cicd/expected-failure-ratchet.test.ts | 4 ++-- scripts/cicd/expected-failure-ratchet.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/scripts/cicd/expected-failure-ratchet.test.ts index 58eb5467b28..687626fabca 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/scripts/cicd/expected-failure-ratchet.test.ts @@ -241,7 +241,7 @@ describe('expected failure baseline comparison', () => { entries: [entry('a')], ceiling: 2, errors: [ - 'Live-defect count 1 must equal LIVE_DEFECT_CEILING (2); lower the ceiling when a pin is fixed.' + 'Live-defect count 1 must equal LIVE_DEFECT_CEILING (2); lower it when a pin is fixed or raise it to admit a new live defect.' ] }, { @@ -250,7 +250,7 @@ describe('expected failure baseline comparison', () => { entries: [entry('a'), entry('b')], ceiling: 1, errors: [ - 'Live-defect count 2 must equal LIVE_DEFECT_CEILING (1); lower the ceiling when a pin is fixed.' + 'Live-defect count 2 must equal LIVE_DEFECT_CEILING (1); lower it when a pin is fixed or raise it to admit a new live defect.' ] } ])('$name', ({ inventory, entries, ceiling, errors }) => { diff --git a/scripts/cicd/expected-failure-ratchet.ts b/scripts/cicd/expected-failure-ratchet.ts index 03bdd7e3bad..b69f99c1271 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/scripts/cicd/expected-failure-ratchet.ts @@ -380,7 +380,7 @@ export function compareBaseline( ] if (liveDefects !== ceiling) { errors.push( - `Live-defect count ${liveDefects} must equal LIVE_DEFECT_CEILING (${ceiling}); lower the ceiling when a pin is fixed.` + `Live-defect count ${liveDefects} must equal LIVE_DEFECT_CEILING (${ceiling}); lower it when a pin is fixed or raise it to admit a new live defect.` ) } return { From 7d21ee44db341c6e4af38cecf73aa48d6a886dbb Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Tue, 20 Oct 2026 05:19:39 +0000 Subject: [PATCH 14/15] refactor: move expected-failure ratchet into a tools workspace Amp-Thread-ID: https://ampcode.com/threads/T-01a109dc-0b0a-72ed-b170-974724b4fbe5 --- knip.config.ts | 3 ++ package.json | 2 +- pnpm-lock.yaml | 16 +++++++++ .../expected-failures/baseline.json | 0 tools/expected-failures/package.json | 15 +++++++++ tools/expected-failures/src/cli.ts | 29 ++++++++++++++++ .../expected-failures/src/inventory.test.ts | 2 +- .../expected-failures/src/inventory.ts | 33 ++----------------- 8 files changed, 67 insertions(+), 33 deletions(-) rename scripts/cicd/expected-failure-baseline.json => tools/expected-failures/baseline.json (100%) create mode 100644 tools/expected-failures/package.json create mode 100644 tools/expected-failures/src/cli.ts rename scripts/cicd/expected-failure-ratchet.test.ts => tools/expected-failures/src/inventory.test.ts (99%) rename scripts/cicd/expected-failure-ratchet.ts => tools/expected-failures/src/inventory.ts (92%) diff --git a/knip.config.ts b/knip.config.ts index 892bf1273f9..135e9ff387a 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -64,6 +64,9 @@ const config: KnipConfig = { 'tools/design-system-audits': { project: ['src/**/*.ts'] }, + 'tools/expected-failures': { + project: ['src/**/*.ts'] + }, 'tools/test-recorder': { project: ['src/**/*.ts'] } diff --git a/package.json b/package.json index ba68eef4fed..88cb95a68d6 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "stylelint": "stylelint --cache --cache-strategy content '{apps,packages,src}/**/*.{css,vue}'", "comfy-test": "tsx tools/test-recorder/src/index.ts", "test:assets-flag-artifact": "tsx scripts/checkAssetsFlagArtifact.ts", - "test:expected-failure-ratchet": "tsx scripts/cicd/expected-failure-ratchet.ts", + "test:expected-failure-ratchet": "tsx tools/expected-failures/src/cli.ts", "test:browser": "pnpm exec playwright test", "test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 tsx scripts/test-browser-local.ts", "custom-node-quarantine": "tsx scripts/customNodeQuarantine.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6ad59039372..345058172bf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1538,6 +1538,22 @@ importers: specifier: 'catalog:' version: 5.0.2(@types/node@25.9.5)(@vitest/coverage-v8@5.0.2)(@vitest/ui@5.0.2)(happy-dom@20.14.5)(jsdom@29.1.1)(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + tools/expected-failures: + dependencies: + typescript: + specifier: 'catalog:' + version: 6.0.3 + zod: + specifier: 'catalog:' + version: 4.6.5 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.9.5 + vitest: + specifier: 'catalog:' + version: 5.0.2(@types/node@25.9.5)(@vitest/coverage-v8@5.0.2)(@vitest/ui@5.0.2)(happy-dom@20.14.5)(jsdom@29.1.1)(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + tools/snapshot-updates: devDependencies: '@playwright/test': diff --git a/scripts/cicd/expected-failure-baseline.json b/tools/expected-failures/baseline.json similarity index 100% rename from scripts/cicd/expected-failure-baseline.json rename to tools/expected-failures/baseline.json diff --git a/tools/expected-failures/package.json b/tools/expected-failures/package.json new file mode 100644 index 00000000000..1040018b626 --- /dev/null +++ b/tools/expected-failures/package.json @@ -0,0 +1,15 @@ +{ + "name": "@comfyorg/expected-failures", + "version": "0.1.0", + "private": true, + "description": "Inventory of Vitest and Playwright expected failures, ratcheted against a classified baseline", + "type": "module", + "dependencies": { + "typescript": "catalog:", + "zod": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "vitest": "catalog:" + } +} diff --git a/tools/expected-failures/src/cli.ts b/tools/expected-failures/src/cli.ts new file mode 100644 index 00000000000..a58bd330655 --- /dev/null +++ b/tools/expected-failures/src/cli.ts @@ -0,0 +1,29 @@ +import { readFile } from 'node:fs/promises' +import path from 'node:path' + +import { buildInventory, compareBaseline } from './inventory' + +const repositoryRoot = path.resolve(import.meta.dirname, '../../..') +const baselinePath = path.resolve(import.meta.dirname, '../baseline.json') + +async function main(): Promise { + const { errors, liveDefects, infrastructure } = compareBaseline( + await buildInventory(repositoryRoot), + JSON.parse(await readFile(baselinePath, 'utf8')) + ) + + if (errors.length === 0) { + process.stdout.write( + `Expected-failure baseline matches: ${liveDefects} live defects, ${infrastructure} infrastructure assertions.\n` + ) + return 0 + } + + for (const error of errors) console.error(error) + console.error( + 'Update tools/expected-failures/baseline.json: remove resolved entries and classify each addition as "live-defect" or "infrastructure-assertion".' + ) + return 1 +} + +if (import.meta.main) process.exitCode = await main() diff --git a/scripts/cicd/expected-failure-ratchet.test.ts b/tools/expected-failures/src/inventory.test.ts similarity index 99% rename from scripts/cicd/expected-failure-ratchet.test.ts rename to tools/expected-failures/src/inventory.test.ts index 687626fabca..4980bd21544 100644 --- a/scripts/cicd/expected-failure-ratchet.test.ts +++ b/tools/expected-failures/src/inventory.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { compareBaseline, inspectSource } from './expected-failure-ratchet' +import { compareBaseline, inspectSource } from './inventory' describe('expected failure inventory', () => { it('ignores test-shaped calls in production source files', () => { diff --git a/scripts/cicd/expected-failure-ratchet.ts b/tools/expected-failures/src/inventory.ts similarity index 92% rename from scripts/cicd/expected-failure-ratchet.ts rename to tools/expected-failures/src/inventory.ts index b69f99c1271..4c5af3eaaa4 100644 --- a/scripts/cicd/expected-failure-ratchet.ts +++ b/tools/expected-failures/src/inventory.ts @@ -4,8 +4,6 @@ import path from 'node:path' import ts from 'typescript' import { z } from 'zod' -import { isMainModule } from '../isMainModule' - type ExpectedFailureKind = 'playwright' | 'vitest' interface ExpectedFailure { @@ -315,8 +313,8 @@ async function sourceFiles(root: string): Promise { return nested.flat() } -async function buildInventory( - repositoryRoot = process.cwd() +export async function buildInventory( + repositoryRoot: string ): Promise { const files = ( await Promise.all( @@ -389,30 +387,3 @@ export function compareBaseline( infrastructure: baseline.length - liveDefects } } - -async function run() { - const inventory = await buildInventory() - const baselinePath = path.join( - process.cwd(), - 'scripts/cicd/expected-failure-baseline.json' - ) - const { errors, liveDefects, infrastructure } = compareBaseline( - inventory, - JSON.parse(await readFile(baselinePath, 'utf8')) as unknown - ) - - if (errors.length === 0) { - console.log( - `Expected-failure baseline matches: ${liveDefects} live defects, ${infrastructure} infrastructure assertions.` - ) - return - } - - for (const error of errors) console.error(error) - console.error( - 'Update scripts/cicd/expected-failure-baseline.json: remove resolved entries and classify each addition as "live-defect" or "infrastructure-assertion".' - ) - process.exitCode = 1 -} - -if (isMainModule(import.meta.url)) await run() From 133fdafdd56eac8391e3985299edcd1015bc29ee Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Tue, 20 Oct 2026 05:21:09 +0000 Subject: [PATCH 15/15] refactor: store the expected-failure baseline as a top-level array --- tools/expected-failures/baseline.json | 182 +++++++++--------- tools/expected-failures/src/inventory.test.ts | 6 +- tools/expected-failures/src/inventory.ts | 4 +- 3 files changed, 94 insertions(+), 98 deletions(-) diff --git a/tools/expected-failures/baseline.json b/tools/expected-failures/baseline.json index 37db7880ed4..0334d8033a4 100644 --- a/tools/expected-failures/baseline.json +++ b/tools/expected-failures/baseline.json @@ -1,92 +1,90 @@ -{ - "entries": [ - { - "id": "playwright:browser_tests/tests/agent/agentLayoutQuality.spec.ts:draws every node without overlapping another:1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/agent/agentRateLimitDuplicateSubmit.spec.ts:keeps repeated Enter presses to one rejected send:1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/agent/agentRefusedDocSubscribe.spec.ts:reports a canvas edit as succeeded when the document never arrived:1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/agent/agentReplyDrafts.spec.ts::1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/agent/agentTurnProgressIsLegible.spec.ts:names how long the finished turn took:1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/agent/agentTwoSessionCrdtSubscription.spec.ts:returning to the displaced build shows it on the canvas:1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", - "classification": "infrastructure-assertion" - }, - { - "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", - "classification": "infrastructure-assertion" - }, - { - "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts:retains the external WebSocket guard:1", - "classification": "infrastructure-assertion" - }, - { - "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts::1", - "classification": "infrastructure-assertion" - }, - { - "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test even when an external fetch error is caught:1", - "classification": "infrastructure-assertion" - }, - { - "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an external WebSocket:1", - "classification": "infrastructure-assertion" - }, - { - "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an unmocked popup:1", - "classification": "infrastructure-assertion" - }, - { - "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:guards every API verb, including context.request:1", - "classification": "infrastructure-assertion" - }, - { - "id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts::1", - "classification": "live-defect" - }, - { - "id": "playwright:browser_tests/tests/workspace/workflowDraftForeignWindowSignIn.spec.ts:a second window signing in neither wipes drafts nor stops persistence:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:releases a hovered node with the pointerup event:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/platform/workflow/persistence/base/hashUtil.test.ts:produces different hashes for known collision paths:1", - "classification": "live-defect" - }, - { - "id": "vitest:src/stores/assetDownloadStore.test.ts:excludes a failed-then-actually-completed task from activeDownloads (PM-1302):1", - "classification": "live-defect" - }, - { - "id": "vitest:src/workbench/extensions/agent/crdt/semanticLayoutSeparation.guard.test.ts:omits pos and size from a minted node payload:1", - "classification": "live-defect" - } - ] -} +[ + { + "id": "playwright:browser_tests/tests/agent/agentLayoutQuality.spec.ts:draws every node without overlapping another:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentRateLimitDuplicateSubmit.spec.ts:keeps repeated Enter presses to one rejected send:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentRefusedDocSubscribe.spec.ts:reports a canvas edit as succeeded when the document never arrived:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentReplyDrafts.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentTurnProgressIsLegible.spec.ts:names how long the finished turn took:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentTwoSessionCrdtSubscription.spec.ts:returning to the displaced build shows it on the canvas:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts:retains the external WebSocket guard:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts::1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test even when an external fetch error is caught:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an external WebSocket:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an unmocked popup:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:guards every API verb, including context.request:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/workspace/workflowDraftForeignWindowSignIn.spec.ts:a second window signing in neither wipes drafts nor stops persistence:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:releases a hovered node with the pointerup event:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/platform/workflow/persistence/base/hashUtil.test.ts:produces different hashes for known collision paths:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/stores/assetDownloadStore.test.ts:excludes a failed-then-actually-completed task from activeDownloads (PM-1302):1", + "classification": "live-defect" + }, + { + "id": "vitest:src/workbench/extensions/agent/crdt/semanticLayoutSeparation.guard.test.ts:omits pos and size from a minted node payload:1", + "classification": "live-defect" + } +] diff --git a/tools/expected-failures/src/inventory.test.ts b/tools/expected-failures/src/inventory.test.ts index 4980bd21544..c9ee2ce2ea6 100644 --- a/tools/expected-failures/src/inventory.test.ts +++ b/tools/expected-failures/src/inventory.test.ts @@ -254,14 +254,12 @@ describe('expected failure baseline comparison', () => { ] } ])('$name', ({ inventory, entries, ceiling, errors }) => { - expect(compareBaseline(inventory, { entries }, ceiling).errors).toEqual( - errors - ) + expect(compareBaseline(inventory, entries, ceiling).errors).toEqual(errors) }) it('rejects duplicate baseline IDs', () => { expect(() => - compareBaseline([pin('a')], { entries: [entry('a'), entry('a')] }, 2) + compareBaseline([pin('a')], [entry('a'), entry('a')], 2) ).toThrow('duplicate IDs') }) }) diff --git a/tools/expected-failures/src/inventory.ts b/tools/expected-failures/src/inventory.ts index 4c5af3eaaa4..1da2ff4f100 100644 --- a/tools/expected-failures/src/inventory.ts +++ b/tools/expected-failures/src/inventory.ts @@ -18,7 +18,7 @@ const baselineEntrySchema = z.object({ id: z.string().min(1), classification: z.enum(['infrastructure-assertion', 'live-defect']) }) -const baselineSchema = z.object({ entries: z.array(baselineEntrySchema) }) +const baselineSchema = z.array(baselineEntrySchema) type BaselineEntry = z.infer const LIVE_DEFECT_CEILING = 14 @@ -343,7 +343,7 @@ export async function buildInventory( } function validateBaseline(value: unknown): BaselineEntry[] { - const entries = baselineSchema.parse(value).entries + const entries = baselineSchema.parse(value) const ids = entries.map(({ id }) => id) if (new Set(ids).size !== ids.length) { throw new Error('Expected-failure baseline contains duplicate IDs.')