From a0bf4a859bf5f63bea0ece0bdc314140d94a7346 Mon Sep 17 00:00:00 2001 From: irparent <134547608+irparent@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:48:52 -0700 Subject: [PATCH] dev: the fast preflight runs the tests a change reaches, within a cap - The fast run executed only the test files a branch added or changed, so a change to source alone reached CI having run no test locally. It now lists every test file the change reaches through vitest's import graph (vitest list --changed) and runs them when there are at most 60; above that (a change to a module most files import reaches hundreds, which is the whole suite and CI's job) it runs the branch's own and says so. - The dashboard's checks also run when src/ changes: its tests import the server's modules. - Lint and the type checks keep their work between runs on a developer's machine (eslint --cache, tsc --incremental, both under node_modules/.cache); CI starts fresh, so nothing changes there. - The refusal on uncommitted changes no longer suggests a stash, which in a repository with several worktrees is shared by all of them. Co-Authored-By: Claude Opus 5.5 --- package.json | 6 ++-- scripts/preflight.mjs | 48 ++++++++++++++++++++++++------ tests/preflight-mirrors-ci.test.ts | 32 ++++++++++++++++---- 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 80f8d5ba..434cdcc7 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,11 @@ "verify:release": "node scripts/verify-release.mjs", "dev": "tsx src/index.ts", "start": "node dist/index.js", - "lint": "eslint src/ tests/ packages/sdk/src/ packages/sdk/test/ packages/langchain/src/ packages/langchain/test/", + "lint": "eslint --cache --cache-location node_modules/.cache/eslint/ src/ tests/ packages/sdk/src/ packages/sdk/test/ packages/langchain/src/ packages/langchain/test/", "format": "prettier --write .", "format:check": "prettier --check .", - "typecheck": "tsc --noEmit", - "typecheck:tests": "tsc -p tsconfig.tests.json && tsc -p tsconfig.e2e.json", + "typecheck": "tsc --noEmit --incremental --tsBuildInfoFile node_modules/.cache/tsc/src.tsbuildinfo", + "typecheck:tests": "tsc -p tsconfig.tests.json --incremental --tsBuildInfoFile node_modules/.cache/tsc/tests.tsbuildinfo && tsc -p tsconfig.e2e.json --incremental --tsBuildInfoFile node_modules/.cache/tsc/e2e.tsbuildinfo", "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", diff --git a/scripts/preflight.mjs b/scripts/preflight.mjs index 3cf038fa..be05d888 100644 --- a/scripts/preflight.mjs +++ b/scripts/preflight.mjs @@ -41,8 +41,17 @@ import { fileURLToPath } from 'node:url'; const root = resolve(fileURLToPath(new URL('..', import.meta.url))); /** Where the root suite's JSON report goes; replaced with a fresh temporary path when it runs. */ const ROOT_REPORT = '{root-report}'; -/** The root-suite test files the branch adds or changes; replaced when a step runs. */ +/** The root-suite test files the fast run executes; replaced when a step runs. */ const TEST_FILES = '{test-files}'; +/** + * The most test files the fast run executes. A change reaches every test + * file that imports what it touched (vitest list --changed); a narrow change + * reaches a handful, and one to a module most files import reaches hundreds, + * which is the whole suite and CI's job. Above this, the fast run executes + * the branch's own test files and says so. 60 files run in about 2 minutes + * on a desktop; 105, which one storage change reached, in about 4. + */ +export const REACHED_CAP = 60; const ACTIONLINT = 'rhysd/actionlint@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667'; /** Checks the pack list carries the dashboard, as the build job does with jq. */ @@ -79,8 +88,8 @@ export const STEPS = [ { name: 'search tokenizer table', ci: ['ci.yml#search-tokenizer-table'], run: 'npm run -s unicode61:check' }, { name: 'security exposure coverage', ci: ['ci.yml#security-exposure'], run: 'node scripts/security/check-exposure-coverage.mjs' }, { name: 'workflows lint (actionlint, in Docker)', ci: ['ci.yml#actionlint'], run: `docker run --rm -v "${root.replace(/\\/g, '/')}:/repo" -w /repo ${ACTIONLINT} -color` }, - { name: 'dashboard typecheck and lint', fast: 'dashboard/', ci: ['ci.yml#lint-and-typecheck'], cwd: 'dashboard', run: 'npm run -s typecheck && npm run -s lint' }, - { name: 'dashboard tests', fast: 'dashboard/', ci: ['ci.yml#lint-and-typecheck'], cwd: 'dashboard', run: 'npm test --silent' }, + { name: 'dashboard typecheck and lint', fast: ['dashboard/', 'src/'], ci: ['ci.yml#lint-and-typecheck'], cwd: 'dashboard', run: 'npm run -s typecheck && npm run -s lint' }, + { name: 'dashboard tests', fast: ['dashboard/', 'src/'], ci: ['ci.yml#lint-and-typecheck'], cwd: 'dashboard', run: 'npm test --silent' }, { name: 'dashboard Storybook build', ci: ['ci.yml#lint-and-typecheck'], cwd: 'dashboard', run: 'npm run -s build-storybook' }, { name: 'website lint and types', fast: 'website/', fastOnly: true, ci: ['ci.yml#website-lint-and-typecheck'], cwd: 'website', run: 'npm run -s lint && npx tsc --noEmit' }, { name: 'website lint, types and build', ci: ['ci.yml#website-lint-and-typecheck'], cwd: 'website', run: 'npm run -s lint && npx tsc --noEmit && npm run -s build' }, @@ -103,7 +112,7 @@ export const STEPS = [ }, { name: 'truthbase regenerated matches the committed one', fast: true, ci: ['claims-alignment.yml#check-truthbase-regen'], run: 'node scripts/claims/generate.mjs --check' }, { - name: 'the test files this branch adds or changes', + name: 'the tests this change reaches', fast: true, fastOnly: true, ci: ['ci.yml#test'], @@ -149,10 +158,24 @@ function baseOf() { return git(['merge-base', 'HEAD', 'origin/main']) || git(['rev-parse', 'HEAD~1']); } -/** The steps a mode runs. Fast: the marked ones, and a folder's only when the branch touched it. */ +/** The steps a mode runs. Fast: the marked ones, and a folder's only when the branch touched one of its prefixes. */ export function stepsFor(full, changed) { if (full) return STEPS.filter((s) => !s.fastOnly); - return STEPS.filter((s) => s.fast === true || (typeof s.fast === 'string' && changed.some((f) => f.startsWith(s.fast)))); + const prefixes = (s) => (typeof s.fast === 'string' ? [s.fast] : Array.isArray(s.fast) ? s.fast : []); + return STEPS.filter((s) => s.fast === true || prefixes(s).some((p) => changed.some((f) => f.startsWith(p)))); +} + +/** + * The root-suite test files the fast run executes: every file the change + * reaches when there are at most REACHED_CAP of them, else only the + * branch's own test files. Returns the files and a sentence saying which. + */ +export function testsToRun(reached, own) { + const rootSuite = (f) => /^tests\/.*\.test\.ts$/.test(f) && !/^tests\/(real-clients|mcpb|stall|upgrade)\//.test(f); + const mine = own.filter(rootSuite); + const all = [...new Set([...reached.filter(rootSuite), ...mine])].sort(); + if (all.length <= REACHED_CAP) return { files: all, why: `${all.length} test file(s) this change reaches` }; + return { files: mine, why: `this change reaches ${all.length} test files, more than ${REACHED_CAP}: running the branch's own ${mine.length}; CI runs the rest` }; } function main(scratch) { @@ -168,7 +191,7 @@ function main(scratch) { return 0; } if (git(['status', '--porcelain'])) { - process.stderr.write('preflight — the working tree has uncommitted changes. It verifies a commit: commit or stash them, then run it again.\n'); + process.stderr.write('preflight — the working tree has uncommitted changes. It verifies a commit: commit them, then run it again. (In a repository with several worktrees, a stash is shared by all of them, so it is not a safe place to put them.)\n'); return 1; } for (const dir of ['', 'dashboard', 'website']) { @@ -181,8 +204,15 @@ function main(scratch) { const commit = git(['rev-parse', 'HEAD']); const base = baseOf(); const changed = git(['diff', '--name-only', `${base}...HEAD`]).split('\n').filter(Boolean); - // Root-suite test files only: the dashboard runs its own, and the excluded folders run in their own CI jobs. - const testFiles = changed.filter((f) => /^tests\/.*\.test\.ts$/.test(f) && !/^tests\/(real-clients|mcpb|stall|upgrade)\//.test(f) && existsSync(join(root, f))); + // The tests the change reaches (vitest's own import graph), capped; the dashboard runs its own, and the excluded folders run in their own CI jobs. + let testFiles = []; + if (!full && changed.some((f) => /\.(ts|tsx|mts|mjs|js|json)$/.test(f))) { + const listed = spawnSync(`npx vitest list --filesOnly --changed ${base}`, { cwd: root, shell: true, encoding: 'utf-8' }); + const reached = listed.status === 0 ? listed.stdout.split(/\r?\n/).map((l) => l.trim()).filter(Boolean) : []; + const chosen = testsToRun(reached, changed.filter((f) => existsSync(join(root, f)))); + testFiles = chosen.files; + process.stdout.write(`preflight — tests: ${listed.status === 0 ? chosen.why : `vitest could not list what the change reaches, so the branch's own ${chosen.files.length} test file(s)`}\n`); + } const steps = stepsFor(full, changed).filter((s) => !(s.run?.includes(TEST_FILES) && testFiles.length === 0)); const started = Date.now(); for (const [i, step] of steps.entries()) { diff --git a/tests/preflight-mirrors-ci.test.ts b/tests/preflight-mirrors-ci.test.ts index 1d03002c..43b2e7b0 100644 --- a/tests/preflight-mirrors-ci.test.ts +++ b/tests/preflight-mirrors-ci.test.ts @@ -1,7 +1,7 @@ import { readdirSync, readFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { CI_ONLY, STEPS, stepsFor } from '../scripts/preflight.mjs'; +import { CI_ONLY, REACHED_CAP, STEPS, stepsFor, testsToRun } from '../scripts/preflight.mjs'; /* * `npm run preflight` is only worth running if it says what CI will say. @@ -107,16 +107,18 @@ describe('the preflight says what the pull-request workflows say', () => { describe('the default preflight is the fast one', () => { const names = (full: boolean, changed: string[]) => stepsFor(full, changed).map((s: { name: string }) => s.name); - it('runs the test files the branch changes, never the whole suite or the builds', () => { + it('runs the tests a change reaches, never the whole suite or the builds', () => { const fast = names(false, ['src/tools/get-traces.ts']); - expect(fast).toContain('the test files this branch adds or changes'); + expect(fast).toContain('the tests this change reaches'); expect(fast).toContain('typecheck'); expect(fast).not.toContain('every test, with the coverage floors'); expect(fast).not.toContain('build'); }); - it("runs the dashboard's and the website's checks only when the branch touches them", () => { - expect(names(false, ['src/x.ts'])).not.toContain('dashboard tests'); + it("runs the dashboard's checks when the branch touches it or the server source its tests import, and the website's only when it touches the website", () => { + expect(names(false, ['docs/x.md'])).not.toContain('dashboard tests'); + expect(names(false, ['src/x.ts'])).toContain('dashboard tests'); + expect(names(false, ['src/x.ts'])).not.toContain('website lint and types'); expect(names(false, ['dashboard/src/App.tsx'])).toContain('dashboard tests'); expect(names(false, ['website/src/app/page.tsx'])).toContain('website lint and types'); }); @@ -124,7 +126,25 @@ describe('the default preflight is the fast one', () => { it('--full runs every mirrored step and none of the fast-only ones', () => { const full = names(true, []); expect(full).toContain('every test, with the coverage floors'); - expect(full).not.toContain('the test files this branch adds or changes'); + expect(full).not.toContain('the tests this change reaches'); expect(full).not.toContain('website lint and types'); }); }); + +describe('the fast run picks the tests a change reaches, within a cap', () => { + it('runs every reached root-suite file, the branch\'s own included, when there are few', () => { + const r = testsToRun(['tests/a.test.ts', 'tests/b.test.ts'], ['src/x.ts', 'tests/c.test.ts']); + expect(r.files).toEqual(['tests/a.test.ts', 'tests/b.test.ts', 'tests/c.test.ts']); + }); + + it('above the cap, runs only the branch\'s own test files and says CI runs the rest', () => { + const many = Array.from({ length: REACHED_CAP + 1 }, (_, i) => `tests/t${i}.test.ts`); + const r = testsToRun(many, ['tests/mine.test.ts']); + expect(r.files).toEqual(['tests/mine.test.ts']); + expect(r.why).toMatch(/CI runs the rest/); + }); + + it('leaves out the folders that run under their own CI jobs', () => { + expect(testsToRun(['tests/upgrade/x.test.ts', 'tests/stall/y.test.ts', 'tests/z.test.ts'], []).files).toEqual(['tests/z.test.ts']); + }); +});