perf(scripts): check:changed on Windows no longer spends minutes walking node_modules (2,500 s → 5.0 s)

scripts/check-changed.test.ts

@@ -20,6 +20,7 @@ import {
   defaultConcurrency,
   docReaderTests,
   gitLines,
+  ignoredCheckInputs,
   runSteps,
   splitTests,
   type Step,
@@ -271,16 +272,17 @@ function fixture(remotes: Record<string, "A" | "B">) {
   };
 }
 
-const compare = (root: string, ref: Option.Option<string>) =>
+const inFixture = <A, E>(effect: Effect.Effect<A, E, ChildProcessSpawner.ChildProcessSpawner>) =>
   Effect.gen(function* () {
     const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
-    return yield* comparisonBase(root, ref).pipe(
+    return yield* effect.pipe(
       Effect.provideService(
         ChildProcessSpawner.ChildProcessSpawner,
         ChildProcessSpawner.make((command) => spawner.spawn(ChildProcess.setEnv(command, env))),
       ),
     );
   }).pipe(Effect.provide(NodeServices.layer), Effect.runPromise);
+const compare = (root: string, ref: Option.Option<string>) => inFixture(comparisonBase(root, ref));
 
 it.each(["upstream", "team/upstream"])(
   "compares against %s/main when the fork's origin/main lags it",
@@ -299,6 +301,44 @@ it("compares against an explicit ref instead of the closest remote main", async
   assert.strictEqual(result.ref, "origin/main");
 });
 
+it("lists the ignored inputs that main's pathspecs list, without entering node_modules", async () => {
+  using project = fixture({});
+  const git = (...args: Array<string>) =>
+    NodeChildProcess.execFileSync("git", args, { cwd: project.root, encoding: "utf8", env });
+  const write = (file: string, content = "") => {
+    NodeFS.mkdirSync(NodePath.dirname(NodePath.join(project.root, file)), { recursive: true });
+    NodeFS.writeFileSync(NodePath.join(project.root, file), content);
+  };
+  write(".gitignore", "node_modules/\ndist/\n.generated/\n.env*\n*.gen.d.ts\nreplaced/\nskipped/");
+  for (const file of ["replaced", "skipped"]) write(file);
+  git("add", ".");
+  git("commit", "-m", "files that directories replace");
+  git("update-index", "--skip-worktree", "skipped");
+  for (const file of ["replaced", "skipped"]) NodeFS.rmSync(NodePath.join(project.root, file));
+  for (const file of [
+    ".env.local",
+    "src/types.gen.d.ts",
+    "src/.generated/routes.ts",
+    "dist/index.d.ts",
+    "dist/node_modules/pkg/index.d.ts",
+    "node_modules/pkg/index.d.ts",
+    // A package folder left behind by a branch switch.
+    "old/dist/a.d.ts",
+    "old/node_modules/pkg/index.d.ts",
+    "replaced/sub/a.d.ts",
+    "skipped/a.d.ts",
+  ])
+    write(file);
+  const command =
+    "ls-files -o -i --exclude-standard -- :(glob)**/.generated/** :(glob)**/.astro/** :(glob)**/*.d.ts :(glob)**/.env* :(exclude,glob)**/node_modules/**";
+  const old = git(...command.split(" "))
+    .split("\n")
+    .filter(Boolean)
+    .sort();
+  assert.includeMembers(old, ["old/dist/a.d.ts", "replaced/sub/a.d.ts", "skipped/a.d.ts"]);
+  assert.deepStrictEqual(await inFixture(ignoredCheckInputs(project.root)), old);
+});
+
 function launcherFixture() {
   const root = NodeFS.realpathSync(
     NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3 checks & (fixture) ")),

scripts/check-changed.ts

@@ -150,6 +150,47 @@ export const git = Effect.fn("checkChanged.git")(function* (
 export const gitLines = (cwd: string, args: ReadonlyArray<string>) =>
   git(cwd, args).pipe(Effect.map((stdout) => stdout.toString().split("\n").filter(Boolean)));
 
+// Ignored files that checks read: generated sources, declarations, and env files.
+const checkInput = /(^|\/)\.(generated|astro)\/|\.d\.ts$|(^|\/)\.env[^/]*$/;
+const inNodeModules = /(^|\/)node_modules\//;
+
+// Pathspecs don't stop git from walking an ignored directory, and on Windows it follows pnpm's
+// junctions in node_modules until MAX_PATH. `--directory` lists one without entering it.
+export const ignoredCheckInputs = Effect.fn("checkChanged.ignoredCheckInputs")(function* (
+  root: string,
+) {
+  const list = (args: ReadonlyArray<string>) =>
+    git(root, ["ls-files", "-z", ...args]).pipe(
+      Effect.map((stdout) => stdout.toString().split("\0").filter(Boolean)),
+    );
+  const ignored = ["--others", "--ignored", "--exclude-standard"];
+  // `--directory` hides a directory where the index has a file; `-k` lists those.
+  const [listed, replaced] = yield* Effect.all(
+    [list([...ignored, "--directory"]), list(["-k", "--directory"])],
+    { concurrency: 2 },
+  );
+  const entries = [...new Set([...listed, ...replaced])].sort();
+  // git lists an untracked directory whose contents are all ignored together with them;
+  // listing it again would walk its node_modules.
+  const directories = entries.filter(
+    (entry, index) =>
+      entry.endsWith("/") && !inNodeModules.test(entry) && !entries[index + 1]?.startsWith(entry),
+  );
+  const files = entries.filter((entry) => !entry.endsWith("/"));
+  // Calls fit Windows' 32,767-character command line: 16 covers `:(literal)`, quotes, a space.
+  // `.git/` matches nothing; it stops git starting its walk inside a nested repository.
+  const longest = directories.reduce((max, dir) => Math.max(max, dir.length), 0);
+  const size = Math.max(1, Math.floor(30_000 / (longest + 16)));
+  for (let start = 0; start < directories.length; start += size) {
+    const batch = directories.slice(start, start + size).map((dir) => `:(literal)${dir}`);
+    files.push(...(yield* list([...ignored, "--", ":(literal).git/", ...batch])));
+  }
+  // A nested repository is listed by its directory, as the old pathspecs listed it.
+  return files
+    .filter((file) => !inNodeModules.test(file) && (file.endsWith("/") || checkInput.test(file)))
+    .sort();
+});
+
 export const comparisonBase = Effect.fn("checkChanged.comparisonBase")(function* (
   root: string,
   explicitRef: Option.Option<string>,
@@ -274,18 +315,7 @@ const checkChanged = Effect.fn("checkChanged")(function* (input: {
         "--",
       ]),
     );
-    const ignored = yield* lines(
-      "ls-files",
-      "--others",
-      "--ignored",
-      "--exclude-standard",
-      "--",
-      ":(glob)**/.generated/**",
-      ":(glob)**/.astro/**",
-      ":(glob)**/*.d.ts",
-      ":(glob)**/.env*",
-      ":(exclude,glob)**/node_modules/**",
-    );
+    const ignored = yield* ignoredCheckInputs(root);
     for (const file of [
       // Raw bytes of changed files too: .gitattributes normalizes line endings in
       // the diff, but the formatter reads the file as it is on disk.