diff --git a/.github/workflows/ci-lint-format.yaml b/.github/workflows/ci-lint-format.yaml index c44406646ce..bbaaec73bf9 100644 --- a/.github/workflows/ci-lint-format.yaml +++ b/.github/workflows/ci-lint-format.yaml @@ -204,3 +204,6 @@ jobs: - name: Verify knip, locale integrity and ADR naming uses: ./.github/actions/repo-checks + + - name: Prevent untracked expected failures + run: pnpm test:expected-failure-ratchet diff --git a/knip.config.ts b/knip.config.ts index 892bf1273f9..135e9ff387a 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -64,6 +64,9 @@ const config: KnipConfig = { 'tools/design-system-audits': { project: ['src/**/*.ts'] }, + 'tools/expected-failures': { + project: ['src/**/*.ts'] + }, 'tools/test-recorder': { project: ['src/**/*.ts'] } diff --git a/package.json b/package.json index 43d336c9c2d..88cb95a68d6 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "stylelint": "stylelint --cache --cache-strategy content '{apps,packages,src}/**/*.{css,vue}'", "comfy-test": "tsx tools/test-recorder/src/index.ts", "test:assets-flag-artifact": "tsx scripts/checkAssetsFlagArtifact.ts", + "test:expected-failure-ratchet": "tsx tools/expected-failures/src/cli.ts", "test:browser": "pnpm exec playwright test", "test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 tsx scripts/test-browser-local.ts", "custom-node-quarantine": "tsx scripts/customNodeQuarantine.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 64610657969..afa12c0b440 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1556,6 +1556,22 @@ importers: specifier: 'catalog:' version: 5.0.2(@types/node@25.9.5)(@vitest/coverage-v8@5.0.2)(@vitest/ui@5.0.2)(happy-dom@20.14.5)(jsdom@29.1.1)(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + tools/expected-failures: + dependencies: + typescript: + specifier: 'catalog:' + version: 6.0.3 + zod: + specifier: 'catalog:' + version: 4.6.5 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.9.5 + vitest: + specifier: 'catalog:' + version: 5.0.2(@types/node@25.9.5)(@vitest/coverage-v8@5.0.2)(@vitest/ui@5.0.2)(happy-dom@20.14.5)(jsdom@29.1.1)(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + tools/published-package-build: devDependencies: '@types/node': diff --git a/tools/expected-failures/baseline.json b/tools/expected-failures/baseline.json new file mode 100644 index 00000000000..0334d8033a4 --- /dev/null +++ b/tools/expected-failures/baseline.json @@ -0,0 +1,90 @@ +[ + { + "id": "playwright:browser_tests/tests/agent/agentLayoutQuality.spec.ts:draws every node without overlapping another:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentNodeIdCollision.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentRateLimitDuplicateSubmit.spec.ts:keeps repeated Enter presses to one rejected send:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentRefusedDocSubscribe.spec.ts:reports a canvas edit as succeeded when the document never arrived:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentReplyDrafts.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentTurnProgressIsLegible.spec.ts:names how long the finished turn took:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/agent/agentTwoSessionCrdtSubscription.spec.ts:returning to the displaced build shows it on the canvas:1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts::1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts:retains the external WebSocket guard:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts::1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test even when an external fetch error is caught:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an external WebSocket:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:fails the owning test for an unmocked popup:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/infrastructure/networkIsolation.spec.ts:guards every API verb, including context.request:1", + "classification": "infrastructure-assertion" + }, + { + "id": "playwright:browser_tests/tests/sidebar/assetPanelDragOntoLoadImageWidget.spec.ts::1", + "classification": "live-defect" + }, + { + "id": "playwright:browser_tests/tests/workspace/workflowDraftForeignWindowSignIn.spec.ts:a second window signing in neither wipes drafts nor stops persistence:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/lib/litegraph/src/LGraphCanvas.gesture.test.ts:releases a hovered node with the pointerup event:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/platform/workflow/persistence/base/hashUtil.test.ts:produces different hashes for known collision paths:1", + "classification": "live-defect" + }, + { + "id": "vitest:src/stores/assetDownloadStore.test.ts:excludes a failed-then-actually-completed task from activeDownloads (PM-1302):1", + "classification": "live-defect" + }, + { + "id": "vitest:src/workbench/extensions/agent/crdt/semanticLayoutSeparation.guard.test.ts:omits pos and size from a minted node payload:1", + "classification": "live-defect" + } +] diff --git a/tools/expected-failures/package.json b/tools/expected-failures/package.json new file mode 100644 index 00000000000..1040018b626 --- /dev/null +++ b/tools/expected-failures/package.json @@ -0,0 +1,15 @@ +{ + "name": "@comfyorg/expected-failures", + "version": "0.1.0", + "private": true, + "description": "Inventory of Vitest and Playwright expected failures, ratcheted against a classified baseline", + "type": "module", + "dependencies": { + "typescript": "catalog:", + "zod": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "vitest": "catalog:" + } +} diff --git a/tools/expected-failures/src/cli.ts b/tools/expected-failures/src/cli.ts new file mode 100644 index 00000000000..a58bd330655 --- /dev/null +++ b/tools/expected-failures/src/cli.ts @@ -0,0 +1,29 @@ +import { readFile } from 'node:fs/promises' +import path from 'node:path' + +import { buildInventory, compareBaseline } from './inventory' + +const repositoryRoot = path.resolve(import.meta.dirname, '../../..') +const baselinePath = path.resolve(import.meta.dirname, '../baseline.json') + +async function main(): Promise { + const { errors, liveDefects, infrastructure } = compareBaseline( + await buildInventory(repositoryRoot), + JSON.parse(await readFile(baselinePath, 'utf8')) + ) + + if (errors.length === 0) { + process.stdout.write( + `Expected-failure baseline matches: ${liveDefects} live defects, ${infrastructure} infrastructure assertions.\n` + ) + return 0 + } + + for (const error of errors) console.error(error) + console.error( + 'Update tools/expected-failures/baseline.json: remove resolved entries and classify each addition as "live-defect" or "infrastructure-assertion".' + ) + return 1 +} + +if (import.meta.main) process.exitCode = await main() diff --git a/tools/expected-failures/src/inventory.test.ts b/tools/expected-failures/src/inventory.test.ts new file mode 100644 index 00000000000..c9ee2ce2ea6 --- /dev/null +++ b/tools/expected-failures/src/inventory.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from 'vitest' + +import { compareBaseline, inspectSource } from './inventory' + +describe('expected failure inventory', () => { + it('ignores test-shaped calls in production source files', () => { + expect( + inspectSource( + `const test = { fails: () => undefined }; test.fails('not a test')`, + 'src/test-support.ts' + ) + ).toEqual([]) + }) + + it('finds Vitest and Playwright pins without matching inert text', () => { + const source = ` + // it.fails('commented out', () => {}) + const explanation = "test.fail() is mentioned" + it.fails('known unit defect', () => {}) + test('conditional browser defect', () => { + test.fail(!available) + }) + test.fail(true, 'declaration-level defect') + test.fail.only('focused browser defect', () => {}) + const fixture = test.extend({}) + fixture.fail(true, 'fixture teardown defect') + it.skip.fails('skipped unit defect', () => {}) + it.fails.each([1])('parameterized unit defect', () => {}) + ` + + expect(inspectSource(source, 'browser_tests/example.spec.ts')).toEqual([ + { + id: 'vitest:browser_tests/example.spec.ts:known unit defect:1', + kind: 'vitest', + file: 'browser_tests/example.spec.ts', + line: 4, + title: 'known unit defect' + }, + { + id: 'playwright:browser_tests/example.spec.ts:conditional browser defect:1', + kind: 'playwright', + file: 'browser_tests/example.spec.ts', + line: 6, + title: 'conditional browser defect' + }, + { + id: 'playwright:browser_tests/example.spec.ts::1', + kind: 'playwright', + file: 'browser_tests/example.spec.ts', + line: 8, + title: '' + }, + { + id: 'playwright:browser_tests/example.spec.ts:focused browser defect:1', + kind: 'playwright', + file: 'browser_tests/example.spec.ts', + line: 9, + title: 'focused browser defect' + }, + { + id: 'playwright:browser_tests/example.spec.ts::2', + kind: 'playwright', + file: 'browser_tests/example.spec.ts', + line: 11, + title: '' + }, + { + id: 'vitest:browser_tests/example.spec.ts:skipped unit defect:1', + kind: 'vitest', + file: 'browser_tests/example.spec.ts', + line: 12, + title: 'skipped unit defect' + }, + { + id: 'vitest:browser_tests/example.spec.ts:parameterized unit defect:1', + kind: 'vitest', + file: 'browser_tests/example.spec.ts', + line: 13, + title: 'parameterized unit defect' + } + ]) + }) + + it('uses extended fixture declaration titles in inventory IDs', () => { + const inventoryId = (title: string) => + inspectSource( + ` + import { baseFixture } from '@e2e/fixtures/baseFixture' + const fixture = baseFixture.extend({}) + fixture('${title}', () => fixture.fail(true)) + `, + 'browser_tests/fixture.spec.ts' + )[0].id + + expect(inventoryId('old title')).toBe( + 'playwright:browser_tests/fixture.spec.ts:old title:1' + ) + expect(inventoryId('new title')).toBe( + 'playwright:browser_tests/fixture.spec.ts:new title:1' + ) + }) + + it('uses computed test titles instead of positional declaration IDs', () => { + const source = ` + for (const scenario of scenarios) { + test(\`rejects \${scenario.kind} collisions\`, () => { + test.fail(true) + }) + } + test(prefix + ' teardown', () => test.fail(true)) + ` + + expect( + inspectSource(source, 'browser_tests/computed-title.spec.ts').map( + ({ id }) => id + ) + ).toEqual([ + 'playwright:browser_tests/computed-title.spec.ts::1', + "playwright:browser_tests/computed-title.spec.ts::1" + ]) + }) + + it('uses the enclosing title for callback-form test.fail', () => { + const inventoryId = (title: string) => + inspectSource( + ` + test('${title}', () => { + test.fail(({ browserName }) => browserName === 'webkit') + }) + `, + 'browser_tests/callback-condition.spec.ts' + )[0].id + + expect(inventoryId('old title')).toBe( + 'playwright:browser_tests/callback-condition.spec.ts:old title:1' + ) + expect(inventoryId('new title')).toBe( + 'playwright:browser_tests/callback-condition.spec.ts:new title:1' + ) + }) + + it('recognizes aliased, extended, and merged Playwright runners', () => { + const source = ` + import { baseFixture as runner } from '@e2e/fixtures/baseFixture' + const extended = runner.extend({}) + const merged = mergeTests(extended, anotherFixture) + merged('known defect', () => merged.fail(true)) + ` + + expect(inspectSource(source, 'browser_tests/fixture.spec.ts')).toEqual([ + { + id: 'playwright:browser_tests/fixture.spec.ts:known defect:1', + kind: 'playwright', + file: 'browser_tests/fixture.spec.ts', + line: 5, + title: 'known defect' + } + ]) + }) + + it('recognizes Playwright failures in app e2e suites', () => { + const source = ` + test('known app defect', () => test.fail(true)) + test.fail('declared app defect', () => {}) + ` + + expect(inspectSource(source, 'apps/website/e2e/example.spec.ts')).toEqual([ + { + id: 'playwright:apps/website/e2e/example.spec.ts:known app defect:1', + kind: 'playwright', + file: 'apps/website/e2e/example.spec.ts', + line: 2, + title: 'known app defect' + }, + { + id: 'playwright:apps/website/e2e/example.spec.ts:declared app defect:1', + kind: 'playwright', + file: 'apps/website/e2e/example.spec.ts', + line: 3, + title: 'declared app defect' + } + ]) + }) + + it('does not classify extended Vitest runners as Playwright failures', () => { + const source = ` + const customTest = test.extend({}) + customTest.fails('known defect', () => {}) + ` + + expect(inspectSource(source, 'src/example.test.ts')).toEqual([ + { + id: 'vitest:src/example.test.ts:known defect:1', + kind: 'vitest', + file: 'src/example.test.ts', + line: 3, + title: 'known defect' + } + ]) + }) +}) + +describe('expected failure baseline comparison', () => { + const pin = (id: string) => ({ + id, + kind: 'vitest' as const, + file: 'src/example.test.ts', + line: 1, + title: id + }) + const entry = ( + id: string, + classification: 'live-defect' | 'infrastructure-assertion' = 'live-defect' + ) => ({ id, classification }) + + it.for([ + { + name: 'matches an exact baseline at the ceiling', + inventory: [pin('a'), pin('b')], + entries: [entry('a'), entry('b', 'infrastructure-assertion')], + ceiling: 1, + errors: [] + }, + { + name: 'rejects an unclassified pin', + inventory: [pin('a'), pin('new')], + entries: [entry('a')], + ceiling: 1, + errors: ['Unclassified expected failure: src/example.test.ts:1 — new'] + }, + { + name: 'rejects a resolved pin left in the baseline', + inventory: [pin('a')], + entries: [entry('a'), entry('fixed', 'infrastructure-assertion')], + ceiling: 1, + errors: ['Resolved or renamed expected failure still in baseline: fixed'] + }, + { + name: 'rejects a ceiling above the live-defect count', + inventory: [pin('a')], + entries: [entry('a')], + ceiling: 2, + errors: [ + 'Live-defect count 1 must equal LIVE_DEFECT_CEILING (2); lower it when a pin is fixed or raise it to admit a new live defect.' + ] + }, + { + name: 'rejects a ceiling below the live-defect count', + inventory: [pin('a'), pin('b')], + entries: [entry('a'), entry('b')], + ceiling: 1, + errors: [ + 'Live-defect count 2 must equal LIVE_DEFECT_CEILING (1); lower it when a pin is fixed or raise it to admit a new live defect.' + ] + } + ])('$name', ({ inventory, entries, ceiling, errors }) => { + expect(compareBaseline(inventory, entries, ceiling).errors).toEqual(errors) + }) + + it('rejects duplicate baseline IDs', () => { + expect(() => + compareBaseline([pin('a')], [entry('a'), entry('a')], 2) + ).toThrow('duplicate IDs') + }) +}) diff --git a/tools/expected-failures/src/inventory.ts b/tools/expected-failures/src/inventory.ts new file mode 100644 index 00000000000..1da2ff4f100 --- /dev/null +++ b/tools/expected-failures/src/inventory.ts @@ -0,0 +1,389 @@ +import { readFile, readdir } from 'node:fs/promises' +import path from 'node:path' + +import ts from 'typescript' +import { z } from 'zod' + +type ExpectedFailureKind = 'playwright' | 'vitest' + +interface ExpectedFailure { + id: string + kind: ExpectedFailureKind + file: string + line: number + title: string +} + +const baselineEntrySchema = z.object({ + id: z.string().min(1), + classification: z.enum(['infrastructure-assertion', 'live-defect']) +}) +const baselineSchema = z.array(baselineEntrySchema) +type BaselineEntry = z.infer + +const LIVE_DEFECT_CEILING = 14 + +const SOURCE_ROOTS = [ + 'apps', + 'browser_tests', + 'build', + 'packages', + 'scripts', + 'src', + 'tools' +] +const TEST_FILE_PATTERN = /\.(?:spec|test)\.[cm]?tsx?$/ +const READ_BATCH_SIZE = 64 + +function isTestExpression( + expression: ts.Expression, + playwrightRunners: ReadonlySet +): boolean { + if (ts.isIdentifier(expression)) return playwrightRunners.has(expression.text) + if (!ts.isPropertyAccessExpression(expression)) return false + + const { root } = callChain(expression) + return ( + root !== undefined && + playwrightRunners.has(root) && + ['fixme', 'only', 'skip'].includes(expression.name.text) + ) +} + +function titleText(node: ts.Expression | undefined): string | undefined { + if (!node) return + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) + return node.text + return `` +} + +function hasTestBody(node: ts.CallExpression): boolean { + const body = node.arguments.at(-1) + return Boolean( + body && (ts.isArrowFunction(body) || ts.isFunctionExpression(body)) + ) +} + +function testTitle( + node: ts.CallExpression, + runner: string | undefined, + playwrightRunners: ReadonlySet +): string { + if (node.arguments.length > 1 && hasTestBody(node)) { + const declaredTitle = titleText(node.arguments.at(0)) + if (declaredTitle) return declaredTitle + } + + for ( + let parent = node.parent; + parent !== node.getSourceFile(); + parent = parent.parent + ) { + if (!ts.isCallExpression(parent)) continue + + const { root } = callChain(parent.expression) + if ( + root !== runner || + !isTestExpression(parent.expression, playwrightRunners) + ) + continue + + const title = titleText(parent.arguments.at(0)) + if (title) return title + } + + return '' +} + +function callChain(expression: ts.Expression): { + root?: string + properties: string[] +} { + if (ts.isIdentifier(expression)) { + return { root: expression.text, properties: [] } + } + if (ts.isCallExpression(expression)) return callChain(expression.expression) + if (!ts.isPropertyAccessExpression(expression)) return { properties: [] } + + const chain = callChain(expression.expression) + return { ...chain, properties: [...chain.properties, expression.name.text] } +} + +function importedPlaywrightRunners(node: ts.Node): string[] { + if ( + !ts.isImportDeclaration(node) || + !ts.isStringLiteral(node.moduleSpecifier) || + !node.moduleSpecifier.text.includes('/fixtures/') + ) + return [] + const bindings = node.importClause?.namedBindings + if (!bindings || !ts.isNamedImports(bindings)) return [] + return bindings.elements + .filter(({ propertyName, name }) => + /(?:fixture|test)$/i.test((propertyName ?? name).text) + ) + .map(({ name }) => name.text) +} + +function createsRunner( + initializer: ts.CallExpression, + runners: ReadonlySet +): boolean { + const { root, properties } = callChain(initializer.expression) + if (root !== undefined && runners.has(root) && properties.includes('extend')) + return true + return ( + root === 'mergeTests' && + initializer.arguments.some( + (argument) => ts.isIdentifier(argument) && runners.has(argument.text) + ) + ) +} + +function derivedRunner( + node: ts.Node, + runners: ReadonlySet +): string | undefined { + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.initializer && + ts.isCallExpression(node.initializer) && + createsRunner(node.initializer, runners) + ) + return node.name.text +} + +function forOfRunnerAliases( + node: ts.Node, + runners: ReadonlySet +): string[] { + if (!ts.isForOfStatement(node)) return [] + const valuesExpression = ts.isAsExpression(node.expression) + ? node.expression.expression + : node.expression + if ( + !ts.isVariableDeclarationList(node.initializer) || + !ts.isArrayLiteralExpression(valuesExpression) + ) + return [] + const binding = node.initializer.declarations[0].name + if (!ts.isArrayBindingPattern(binding)) return [] + + return binding.elements.flatMap((element, index) => { + if (!ts.isBindingElement(element) || !ts.isIdentifier(element.name)) + return [] + const values = valuesExpression.elements.map((value) => + ts.isArrayLiteralExpression(value) ? value.elements[index] : null + ) + return values.length > 0 && + values.every( + (value) => value && ts.isIdentifier(value) && runners.has(value.text) + ) + ? [element.name.text] + : [] + }) +} + +function playwrightRunnerNames(sourceFile: ts.SourceFile): Set { + const runners = new Set(['test']) + + function visit(node: ts.Node) { + importedPlaywrightRunners(node).forEach((name) => runners.add(name)) + const runner = derivedRunner(node, runners) + if (runner) runners.add(runner) + forOfRunnerAliases(node, runners).forEach((name) => runners.add(name)) + ts.forEachChild(node, visit) + } + + visit(sourceFile) + return runners +} + +function vitestRunnerNames(sourceFile: ts.SourceFile): Set { + const runners = new Set(['it', 'test']) + + function visit(node: ts.Node) { + const runner = derivedRunner(node, runners) + if (runner) runners.add(runner) + ts.forEachChild(node, visit) + } + + visit(sourceFile) + return runners +} + +function isVitestFailure( + root: string | undefined, + properties: string[], + vitestRunners: ReadonlySet +) { + return ( + root !== undefined && + vitestRunners.has(root) && + properties.includes('fails') + ) +} + +function isPlaywrightFailure( + root: string | undefined, + properties: string[], + playwrightRunners: ReadonlySet +) { + return ( + root !== undefined && + properties.includes('fail') && + playwrightRunners.has(root) + ) +} + +function expectedFailureKind( + node: ts.CallExpression, + playwrightRunners: ReadonlySet, + vitestRunners: ReadonlySet +): ExpectedFailureKind | undefined { + if (ts.isCallExpression(node.parent) && node.parent.expression === node) + return + + const { root, properties } = callChain(node.expression) + if (isVitestFailure(root, properties, vitestRunners)) return 'vitest' + if (isPlaywrightFailure(root, properties, playwrightRunners)) + return 'playwright' +} + +function vitestTitle(node: ts.CallExpression): string { + return titleText(node.arguments.at(0)) ?? '' +} + +export function inspectSource(source: string, file: string): ExpectedFailure[] { + if (!TEST_FILE_PATTERN.test(file)) return [] + + const sourceFile = ts.createSourceFile( + file, + source, + ts.ScriptTarget.Latest, + true + ) + const playwrightRunners = /^(?:browser_tests|apps\/[^/]+\/e2e)\//.test(file) + ? playwrightRunnerNames(sourceFile) + : new Set() + const vitestRunners = vitestRunnerNames(sourceFile) + const found: Omit[] = [] + + function visit(node: ts.Node) { + if (!ts.isCallExpression(node)) { + ts.forEachChild(node, visit) + return + } + + const kind = expectedFailureKind(node, playwrightRunners, vitestRunners) + + if (kind) { + const position = sourceFile.getLineAndCharacterOfPosition(node.getStart()) + const { root } = callChain(node.expression) + const title = + kind === 'vitest' + ? vitestTitle(node) + : testTitle(node, root, playwrightRunners) + found.push({ kind, file, line: position.line + 1, title }) + } + + ts.forEachChild(node, visit) + } + visit(sourceFile) + + const occurrences = new Map() + return found.map((entry) => { + const fingerprint = `${entry.kind}:${entry.file}:${entry.title}` + const occurrence = (occurrences.get(fingerprint) ?? 0) + 1 + occurrences.set(fingerprint, occurrence) + return { ...entry, id: `${fingerprint}:${occurrence}` } + }) +} + +async function sourceFiles(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }) + const nested = await Promise.all( + entries.map(async (entry) => { + const entryPath = path.join(root, entry.name) + if (entry.isDirectory()) return sourceFiles(entryPath) + return TEST_FILE_PATTERN.test(entry.name) ? [entryPath] : [] + }) + ) + return nested.flat() +} + +export async function buildInventory( + repositoryRoot: string +): Promise { + const files = ( + await Promise.all( + SOURCE_ROOTS.map((root) => sourceFiles(path.join(repositoryRoot, root))) + ) + ).flat() + + const inventory: ExpectedFailure[] = [] + for (let index = 0; index < files.length; index += READ_BATCH_SIZE) { + const batch = files.slice(index, index + READ_BATCH_SIZE) + inventory.push( + ...( + await Promise.all( + batch.map(async (absoluteFile) => { + const file = path + .relative(repositoryRoot, absoluteFile) + .split(path.sep) + .join('/') + return inspectSource(await readFile(absoluteFile, 'utf8'), file) + }) + ) + ).flat() + ) + } + return inventory.sort((left, right) => left.id.localeCompare(right.id)) +} + +function validateBaseline(value: unknown): BaselineEntry[] { + const entries = baselineSchema.parse(value) + const ids = entries.map(({ id }) => id) + if (new Set(ids).size !== ids.length) { + throw new Error('Expected-failure baseline contains duplicate IDs.') + } + return entries +} + +export function compareBaseline( + inventory: ExpectedFailure[], + baselineValue: unknown, + ceiling = LIVE_DEFECT_CEILING +): { errors: string[]; liveDefects: number; infrastructure: number } { + const baseline = validateBaseline(baselineValue) + const actualIds = new Set(inventory.map(({ id }) => id)) + const baselineIds = new Set(baseline.map(({ id }) => id)) + const liveDefects = baseline.filter( + ({ classification }) => classification === 'live-defect' + ).length + const errors = [ + ...inventory + .filter(({ id }) => !baselineIds.has(id)) + .map( + (entry) => + `Unclassified expected failure: ${entry.file}:${entry.line} — ${entry.id}` + ), + ...baseline + .filter(({ id }) => !actualIds.has(id)) + .map( + ({ id }) => + `Resolved or renamed expected failure still in baseline: ${id}` + ) + ] + if (liveDefects !== ceiling) { + errors.push( + `Live-defect count ${liveDefects} must equal LIVE_DEFECT_CEILING (${ceiling}); lower it when a pin is fixed or raise it to admit a new live defect.` + ) + } + return { + errors, + liveDefects, + infrastructure: baseline.length - liveDefects + } +}