From 4ea36a37c3826c854f468a3964cc1053029f05b1 Mon Sep 17 00:00:00 2001 From: Guanying Chen Date: Sun, 1 Nov 2026 16:36:43 -0800 Subject: [PATCH 1/2] perf(server): batch Git ignore checks for wide directories --- .../src/workspace/WorkspaceEntries.test.ts | 57 ++++++++++++++++++- apps/server/src/workspace/WorkspaceEntries.ts | 40 +++++++++++-- 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index d861935fe17cc..cf57bb1082d1b 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -15,6 +15,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspaceEntries from "./WorkspaceEntries.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; +import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); @@ -248,10 +249,11 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const platform = yield* HostProcess.Platform; + const longDirectory = Array.from({ length: 6 }, () => "é".repeat(95)).join("/"); yield* writeTextFile( cwd, ".gitignore", - "node_modules/\ndist/\n\\[id\\]/\n*.map\nfoo\n!foo/\n", + "node_modules/\ndist/\n\\[id\\]/\n*.map\n!src/keep.map\nfoo\n!foo/\n", ); yield* Effect.forEach( [ @@ -259,6 +261,7 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { "dist", "[id]", "src", + longDirectory, ...(platform !== "win32" ? [":foo"] : []), ].flatMap((directory) => Array.from({ length: 1001 }, (_, index) => `${directory}/pkg-${index}.js`), @@ -272,6 +275,8 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { "i/page.tsx", "[id]/page.tsx", "src/bundle.map", + "src/keep.map", + `${longDirectory}/bundle.map`, ]) { yield* writeTextFile(cwd, relativePath); } @@ -280,13 +285,27 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { yield* fileSystem.symlink(path.join(cwd, "src"), path.join(cwd, "node_modules/link")); } - const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const vcsProcess = yield* VcsProcess.VcsProcess; + const ignoreInputs: string[] = []; + const workspaceEntries = yield* WorkspaceEntries.make.pipe( + Effect.provide(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layer), + Effect.provideService( + VcsProcess.VcsProcess, + VcsProcess.VcsProcess.of({ + run: (input) => { + if (input.args.includes("check-ignore")) ignoreInputs.push(input.stdin ?? ""); + return vcsProcess.run(input); + }, + }), + ), + ); for (const { directoryPath, ignored, except } of [ { directoryPath: "node_modules", ignored: true, except: [] }, { directoryPath: "dist", ignored: true, except: ["dist/kept.js", "dist/vendored"] }, // git reads `[id]/page.tsx` as a pathspec, which matches the tracked `i/page.tsx`. { directoryPath: "[id]", ignored: true, except: ["[id]/page.tsx"] }, { directoryPath: "src", ignored: false, except: ["src/bundle.map"] }, + { directoryPath: longDirectory, ignored: false, except: [`${longDirectory}/bundle.map`] }, ...(platform !== "win32" ? [ // git refuses to classify paths beyond a symlink, so nothing there is labelled. @@ -296,6 +315,7 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { ] : []), ]) { + ignoreInputs.length = 0; const { entries } = yield* workspaceEntries.list({ cwd, directoryPath }); expect(entries.length, directoryPath).toBeGreaterThan(1000); expect( @@ -305,10 +325,43 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { .toSorted(), directoryPath, ).toEqual(except); + expect(ignoreInputs.length, directoryPath).toBeLessThanOrEqual( + directoryPath === longDirectory ? 3 : 2, + ); + expect(ignoreInputs.every((input) => Buffer.byteLength(input) <= 1024 * 1024)).toBe(true); + if (directoryPath === "node_modules") expect(ignoreInputs).toEqual(["node_modules\0"]); } }), ); + it.effect("retains earlier ignore labels when a later path makes git fail", () => + Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + if (platform === "win32") return; + const cwd = yield* makeTempDir({ git: true }); + const paths = Array.from( + { length: 6000 }, + (_, index) => `${index.toString().padStart(4, "0")}-${"x".repeat(240)}.map`, + ); + yield* writeTextFile(cwd, ".gitignore", "*.map\n"); + yield* Effect.forEach(paths, (relativePath) => writeTextFile(cwd, relativePath), { + concurrency: 32, + discard: true, + }); + yield* writeTextFile(cwd, ":(attr:x)bad.map"); + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const { entries } = yield* workspaceEntries.list({ cwd, directoryPath: "" }); + // The first chunk includes .gitignore; Git rejects the pathspec in the next one. + expect( + entries + .filter((entry) => entry.ignored) + .map((entry) => entry.path) + .toSorted(), + ).toEqual(paths.slice(0, 5999)); + }), + ); + it.effect("lists a host folder and its parent, each from its own root", () => Effect.gen(function* () { const parent = yield* makeTempDir(); diff --git a/apps/server/src/workspace/WorkspaceEntries.ts b/apps/server/src/workspace/WorkspaceEntries.ts index 0b853ccdaa437..144920c8bfc5d 100644 --- a/apps/server/src/workspace/WorkspaceEntries.ts +++ b/apps/server/src/workspace/WorkspaceEntries.ts @@ -115,7 +115,11 @@ export class WorkspaceEntries extends Context.Service< const IGNORED_FILES_LIMIT = 200; /** Paths sent to one `git check-ignore` call when labelling a folder's children. */ -const IGNORE_CHECK_CHUNK_SIZE = 1000; +const IGNORE_CHECK_CHUNK_SIZE = 10_000; +const IGNORE_CHECK_FALLBACK_CHUNK_SIZE = 1000; +// NUL-delimited stdout echoes input paths; stay well below the 16 MiB output limit. +const IGNORE_CHECK_MAX_INPUT_BYTES = 1024 * 1024; +const PARENT_IGNORE_CHECK_THRESHOLD = 1000; /** Children classified at once when listing a folder; each symlink costs a realpath and a stat. */ const SYMLINK_RESOLVE_BATCH_SIZE = 64; @@ -412,7 +416,7 @@ export const make = Effect.gen(function* () { // Check ignored folders once to avoid child index lookups; exit 0 guarantees no tracked descendants. // Symlinks and pathspec magic still need per-child checks. if ( - entries.length > IGNORE_CHECK_CHUNK_SIZE && + entries.length > PARENT_IGNORE_CHECK_THRESHOLD && target.relativePath !== "" && !throughSymlink && !/^:|[*?[\\]/.test(target.relativePath) && @@ -424,12 +428,36 @@ export const make = Effect.gen(function* () { }; } const ignored = new Set(); - for (let offset = 0; offset < entries.length; offset += IGNORE_CHECK_CHUNK_SIZE) { - const result = yield* checkIgnore( - entries.slice(offset, offset + IGNORE_CHECK_CHUNK_SIZE).map((entry) => entry.path), - ); + let chunkSize = IGNORE_CHECK_CHUNK_SIZE; + for (let offset = 0; offset < entries.length;) { + const paths: string[] = []; + let inputBytes = 0; + for (const entry of entries.slice(offset, offset + chunkSize)) { + const entryBytes = Buffer.byteLength(entry.path) + 1; + if (paths.length > 0 && inputBytes + entryBytes > IGNORE_CHECK_MAX_INPUT_BYTES) { + // Keep the error fallback aligned with the original groups when they fit the byte cap. + if (paths.length > IGNORE_CHECK_FALLBACK_CHUNK_SIZE) { + paths.length -= paths.length % IGNORE_CHECK_FALLBACK_CHUNK_SIZE; + } + break; + } + paths.push(entry.path); + inputBytes += entryBytes; + } + const result = yield* checkIgnore(paths); + // Preserve earlier chunks' labels if a later path makes Git reject the batch. + if ( + result && + result.exitCode !== 0 && + result.exitCode !== 1 && + paths.length > IGNORE_CHECK_FALLBACK_CHUNK_SIZE + ) { + chunkSize = IGNORE_CHECK_FALLBACK_CHUNK_SIZE; + continue; + } if (!result || (result.exitCode !== 0 && result.exitCode !== 1)) break; for (const ignoredPath of result.stdout.split("\0")) ignored.add(ignoredPath); + offset += paths.length; } return { entries: entries.map((entry) => From e985131e8e05e616b0437b44bb7bf4447ebef527 Mon Sep 17 00:00:00 2001 From: Guanying Chen Date: Sun, 1 Nov 2026 18:15:58 -0800 Subject: [PATCH 2/2] test(server): trim wide-directory ignore fixtures --- .../src/workspace/WorkspaceEntries.test.ts | 50 ++++++++----------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index cf57bb1082d1b..5bbb0859341e0 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -249,19 +249,18 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const platform = yield* HostProcess.Platform; - const longDirectory = Array.from({ length: 6 }, () => "é".repeat(95)).join("/"); + const srcDirectory = Array.from({ length: 6 }, () => "é".repeat(95)).join("/"); yield* writeTextFile( cwd, ".gitignore", - "node_modules/\ndist/\n\\[id\\]/\n*.map\n!src/keep.map\nfoo\n!foo/\n", + "node_modules/\ndist/\n\\[id\\]/\n*.map\nfoo\n!foo/\n", ); yield* Effect.forEach( [ "node_modules", "dist", "[id]", - "src", - longDirectory, + srcDirectory, ...(platform !== "win32" ? [":foo"] : []), ].flatMap((directory) => Array.from({ length: 1001 }, (_, index) => `${directory}/pkg-${index}.js`), @@ -274,38 +273,35 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { "dist/vendored/index.js", "i/page.tsx", "[id]/page.tsx", - "src/bundle.map", - "src/keep.map", - `${longDirectory}/bundle.map`, + `${srcDirectory}/bundle.map`, ]) { yield* writeTextFile(cwd, relativePath); } yield* git(cwd, ["add", "-f", "dist/kept.js", "dist/vendored/index.js", "i/page.tsx"]); if (platform !== "win32") { - yield* fileSystem.symlink(path.join(cwd, "src"), path.join(cwd, "node_modules/link")); + yield* fileSystem.symlink( + path.join(cwd, srcDirectory), + path.join(cwd, "node_modules/link"), + ); } const vcsProcess = yield* VcsProcess.VcsProcess; const ignoreInputs: string[] = []; const workspaceEntries = yield* WorkspaceEntries.make.pipe( Effect.provide(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layer), - Effect.provideService( - VcsProcess.VcsProcess, - VcsProcess.VcsProcess.of({ - run: (input) => { - if (input.args.includes("check-ignore")) ignoreInputs.push(input.stdin ?? ""); - return vcsProcess.run(input); - }, - }), - ), + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => { + if (input.args.includes("check-ignore")) ignoreInputs.push(input.stdin ?? ""); + return vcsProcess.run(input); + }, + }), ); for (const { directoryPath, ignored, except } of [ { directoryPath: "node_modules", ignored: true, except: [] }, { directoryPath: "dist", ignored: true, except: ["dist/kept.js", "dist/vendored"] }, // git reads `[id]/page.tsx` as a pathspec, which matches the tracked `i/page.tsx`. { directoryPath: "[id]", ignored: true, except: ["[id]/page.tsx"] }, - { directoryPath: "src", ignored: false, except: ["src/bundle.map"] }, - { directoryPath: longDirectory, ignored: false, except: [`${longDirectory}/bundle.map`] }, + { directoryPath: srcDirectory, ignored: false, except: [`${srcDirectory}/bundle.map`] }, ...(platform !== "win32" ? [ // git refuses to classify paths beyond a symlink, so nothing there is labelled. @@ -326,7 +322,7 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { directoryPath, ).toEqual(except); expect(ignoreInputs.length, directoryPath).toBeLessThanOrEqual( - directoryPath === longDirectory ? 3 : 2, + directoryPath === srcDirectory ? 3 : 2, ); expect(ignoreInputs.every((input) => Buffer.byteLength(input) <= 1024 * 1024)).toBe(true); if (directoryPath === "node_modules") expect(ignoreInputs).toEqual(["node_modules\0"]); @@ -336,11 +332,11 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { it.effect("retains earlier ignore labels when a later path makes git fail", () => Effect.gen(function* () { - const platform = yield* HostProcess.Platform; - if (platform === "win32") return; + if ((yield* HostProcess.Platform) === "win32") return; const cwd = yield* makeTempDir({ git: true }); + // With .gitignore, these fill five 1,000-path groups before Git rejects the pathspec. const paths = Array.from( - { length: 6000 }, + { length: 4999 }, (_, index) => `${index.toString().padStart(4, "0")}-${"x".repeat(240)}.map`, ); yield* writeTextFile(cwd, ".gitignore", "*.map\n"); @@ -352,13 +348,7 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const { entries } = yield* workspaceEntries.list({ cwd, directoryPath: "" }); - // The first chunk includes .gitignore; Git rejects the pathspec in the next one. - expect( - entries - .filter((entry) => entry.ignored) - .map((entry) => entry.path) - .toSorted(), - ).toEqual(paths.slice(0, 5999)); + expect(entries.filter((entry) => entry.ignored)).toHaveLength(paths.length); }), );