From 7e131b1d69d6b441cafc32cc85698350d5635d4f Mon Sep 17 00:00:00 2001 From: yashranaway Date: Sat, 12 Sep 2026 05:43:17 +0000 Subject: [PATCH 1/9] fix(server): isolate native workspace indexing from the event loop --- apps/server/scripts/workspace-search-mock.ts | 73 +++ .../NativeWorkspaceSearchIndex.test.ts | 348 +++++++++++ .../workspace/NativeWorkspaceSearchIndex.ts | 439 +++++++++++++ .../src/workspace/WorkspaceEntries.test.ts | 31 +- .../src/workspace/WorkspaceSearchHost.ts | 93 +++ .../workspace/WorkspaceSearchIndex.test.ts | 509 ++++++---------- .../src/workspace/WorkspaceSearchIndex.ts | 576 +++--------------- .../workspace/WorkspaceSearchIndexService.ts | 104 ++++ .../src/workspace/WorkspaceSearchProcess.ts | 120 ++++ .../src/workspace/workspaceSearchProtocol.ts | 44 ++ apps/server/src/workspaceSearchWorker.ts | 56 ++ apps/server/vite.config.ts | 2 +- 12 files changed, 1559 insertions(+), 836 deletions(-) create mode 100644 apps/server/scripts/workspace-search-mock.ts create mode 100644 apps/server/src/workspace/NativeWorkspaceSearchIndex.test.ts create mode 100644 apps/server/src/workspace/NativeWorkspaceSearchIndex.ts create mode 100644 apps/server/src/workspace/WorkspaceSearchHost.ts create mode 100644 apps/server/src/workspace/WorkspaceSearchIndexService.ts create mode 100644 apps/server/src/workspace/WorkspaceSearchProcess.ts create mode 100644 apps/server/src/workspace/workspaceSearchProtocol.ts create mode 100644 apps/server/src/workspaceSearchWorker.ts diff --git a/apps/server/scripts/workspace-search-mock.ts b/apps/server/scripts/workspace-search-mock.ts new file mode 100644 index 0000000000000..e3727fbdc6451 --- /dev/null +++ b/apps/server/scripts/workspace-search-mock.ts @@ -0,0 +1,73 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeNet from "node:net"; +import * as Exit from "effect/Exit"; +import * as Schema from "effect/Schema"; +import { SearchRequest, SearchResponse } from "../src/workspace/workspaceSearchProtocol.ts"; +import { + WorkspaceSearchIndexRefreshFailed, + WorkspaceSearchIndexScanTimedOut, +} from "../src/workspace/WorkspaceSearchIndexService.ts"; + +const decodeSearchRequest = Schema.decodeUnknownSync(SearchRequest); +const encodeSearchResponse = Schema.encodeSync(SearchResponse); + +const directories = new Map(); +const block = () => { + const socket = NodeNet.connect( + Number(process.env.T3_SEARCH_TEST_RECEIPT_PORT), + "127.0.0.1", + () => { + socket.write("blocked"); + // Model a synchronous FFI call which cannot service IPC or JS timers. + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0); + }, + ); +}; +process.on("disconnect", () => process.exit(0)); +process.on("message", (message) => { + const { id, operation: input } = decodeSearchRequest(message); + if (input.method === "initialize") directories.set(id, input.cwd); + const cwd = directories.get(id) ?? ""; + if (input.method === "dispose") { + if (cwd === "block-dispose") { + block(); + return; + } + directories.delete(id); + process.send?.(encodeSearchResponse(Exit.succeed(null))); + return; + } + if (cwd === "block-initialize" || (input.method === "search" && input.query === "block")) { + block(); + return; + } + if (input.method === "search" && input.query === "crash") process.exit(1); + if (input.method === "refresh") { + process.send?.( + encodeSearchResponse( + Exit.fail(new WorkspaceSearchIndexRefreshFailed({ cwd, reason: "scan failed" })), + ), + ); + return; + } + if (cwd === "scan-timeout") { + process.send?.( + encodeSearchResponse( + Exit.fail(new WorkspaceSearchIndexScanTimedOut({ cwd, timeout: "15 seconds" })), + ), + ); + return; + } + process.send?.( + encodeSearchResponse( + Exit.succeed( + input.method === "initialize" + ? null + : { + entries: [{ path: String(process.pid), kind: "file" }], + truncated: false, + }, + ), + ), + ); +}); diff --git a/apps/server/src/workspace/NativeWorkspaceSearchIndex.test.ts b/apps/server/src/workspace/NativeWorkspaceSearchIndex.test.ts new file mode 100644 index 0000000000000..f3b6f2a24b22e --- /dev/null +++ b/apps/server/src/workspace/NativeWorkspaceSearchIndex.test.ts @@ -0,0 +1,348 @@ +import { + FileFinder, + type FileItem, + type GrepCursor, + type GrepOptions, + type GrepResult, +} from "@ff-labs/fff-node"; +import { afterEach, expect, it } from "@effect/vitest"; +import * as Cause from "effect/Cause"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import { vi } from "vite-plus/test"; + +import * as WorkspaceSearchIndex from "./NativeWorkspaceSearchIndex.ts"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function fileItem(relativePath: string): FileItem { + return { + relativePath, + fileName: relativePath.slice(relativePath.lastIndexOf("/") + 1), + size: 1, + modified: 0, + accessFrecencyScore: 0, + modificationFrecencyScore: 0, + totalFrecencyScore: 0, + gitStatus: "clean", + }; +} + +it.effect("filters image searches before applying the result limit", () => + Effect.scoped( + Effect.gen(function* () { + const items = [ + ...Array.from({ length: 200 }, (_, index) => fileItem(`src/file-${index}.ts`)), + fileItem("public/icon.svg"), + ]; + const fileSearch = vi.fn(() => ({ + ok: true as const, + value: { + items, + scores: [], + totalMatched: items.length, + totalFiles: items.length, + }, + })); + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + fileSearch, + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); + const resultWithoutKind = yield* searchIndex.search("", 200, undefined, true); + const resultWithDirectoryKind = yield* searchIndex.search("", 200, "directory", true); + + expect(resultWithoutKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); + expect(resultWithDirectoryKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); + expect(fileSearch).toHaveBeenCalledTimes(2); + expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 }); + }), + ), +); + +it.effect("preserves unexpected FileFinder creation failures", () => + Effect.gen(function* () { + const cause = new Error("native initialization failed"); + vi.spyOn(FileFinder, "create").mockImplementationOnce(() => { + throw cause; + }); + + const error = yield* Effect.flip( + Effect.scoped(WorkspaceSearchIndex.make("/workspace/project")), + ); + + expect(error).toMatchObject({ + _tag: "WorkspaceSearchIndexCreateFailed", + cwd: "/workspace/project", + reason: "FileFinder.create threw unexpectedly.", + cause, + }); + }), +); + +it.effect("keeps returned FileFinder creation diagnostics out of the cause chain", () => + Effect.gen(function* () { + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ + ok: false, + error: "native index rejected the directory", + }); + + const error = yield* Effect.flip( + Effect.scoped(WorkspaceSearchIndex.make("/workspace/project")), + ); + + expect(error).toMatchObject({ + _tag: "WorkspaceSearchIndexCreateFailed", + cwd: "/workspace/project", + reason: "native index rejected the directory", + }); + expect(error.cause).toBeUndefined(); + }), +); + +it.effect("waits for the full content index warmup before returning", () => + Effect.gen(function* () { + const waitForIndexReady = vi.fn(async () => ({ ok: true as const, value: true })); + const finder = { + destroy: vi.fn(), + waitForIndexReady, + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + yield* Effect.scoped(WorkspaceSearchIndex.make("/workspace/project", "content")); + + expect(waitForIndexReady).toHaveBeenCalledWith(15_000); + }), +); + +it.effect("preserves a full-index warmup timeout as a structured error", () => + Effect.gen(function* () { + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: false })), + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const error = yield* Effect.flip( + Effect.scoped(WorkspaceSearchIndex.make("/workspace/project", "content")), + ); + + expect(error).toMatchObject({ + _tag: "WorkspaceSearchIndexScanTimedOut", + cwd: "/workspace/project", + timeout: "15 seconds", + }); + }), +); + +it.effect("preserves FileFinder destroy failures as structured defects", () => + Effect.gen(function* () { + const cause = new Error("native destroy failed"); + const finder = { + destroy: vi.fn(() => { + throw cause; + }), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const exit = yield* Effect.scoped(WorkspaceSearchIndex.make("/workspace/project")).pipe( + Effect.exit, + ); + + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Cause.hasDies(exit.cause)).toBe(true); + const error = Cause.squash(exit.cause); + expect(error).toBeInstanceOf(WorkspaceSearchIndex.WorkspaceSearchIndexDestroyFailed); + expect(error).toMatchObject({ + _tag: "WorkspaceSearchIndexDestroyFailed", + cwd: "/workspace/project", + cause, + }); + } + }), +); + +it.effect("preserves search and refresh failures with operation context", () => + Effect.scoped( + Effect.gen(function* () { + const searchCause = new Error("native search failed"); + const refreshCause = new Error("native scan failed"); + const contentSearchCause = new Error("native grep failed"); + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + mixedSearch: vi.fn(() => { + throw searchCause; + }), + grep: vi.fn(() => { + throw contentSearchCause; + }), + scanFiles: vi.fn(() => { + throw refreshCause; + }), + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); + const query = "authorization: Bearer secret-token"; + const searchError = yield* Effect.flip(searchIndex.search(query, 3)); + const contentSearchError = yield* Effect.flip( + searchIndex.searchContents({ + query, + limit: 3, + caseSensitive: false, + wholeWord: false, + useRegex: false, + }), + ); + const refreshError = yield* Effect.flip(searchIndex.refresh()); + + expect(searchError).toMatchObject({ + _tag: "WorkspaceSearchIndexSearchFailed", + cwd: "/workspace/project", + queryLength: query.length, + pageSize: 4, + reason: "FileFinder.mixedSearch threw unexpectedly.", + cause: searchCause, + }); + expect(searchError).not.toHaveProperty("query"); + expect(searchError.message).not.toMatch(/Bearer|secret-token/); + expect(contentSearchError).toMatchObject({ + _tag: "WorkspaceSearchIndexSearchFailed", + cwd: "/workspace/project", + queryLength: query.length, + pageSize: 3, + reason: "FileFinder.grep threw unexpectedly.", + cause: contentSearchCause, + }); + expect(contentSearchError).not.toHaveProperty("query"); + expect(contentSearchError.message).not.toMatch(/Bearer|secret-token/); + expect(refreshError).toMatchObject({ + _tag: "WorkspaceSearchIndexRefreshFailed", + cwd: "/workspace/project", + reason: "FileFinder.scanFiles threw unexpectedly.", + cause: refreshCause, + }); + }), + ), +); + +it.effect("keeps returned search diagnostics out of the cause chain", () => + Effect.scoped( + Effect.gen(function* () { + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + mixedSearch: vi.fn(() => ({ ok: false, error: "native query rejected" })), + scanFiles: vi.fn(() => ({ ok: false, error: "native refresh rejected" })), + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); + const query = "authorization: Bearer secret-token"; + const searchError = yield* Effect.flip(searchIndex.search(query, 3)); + const refreshError = yield* Effect.flip(searchIndex.refresh()); + + expect(searchError).toMatchObject({ + _tag: "WorkspaceSearchIndexSearchFailed", + cwd: "/workspace/project", + queryLength: query.length, + pageSize: 4, + reason: "native query rejected", + }); + expect(searchError).not.toHaveProperty("query"); + expect(searchError.message).not.toMatch(/Bearer|secret-token/); + expect(searchError.cause).toBeUndefined(); + expect(refreshError).toMatchObject({ + _tag: "WorkspaceSearchIndexRefreshFailed", + cwd: "/workspace/project", + reason: "native refresh rejected", + }); + expect(refreshError.cause).toBeUndefined(); + }), + ), +); + +it.effect("continues whole-word searches after a filtered grep page", () => + Effect.scoped( + Effect.gen(function* () { + const nextCursor = { + __brand: "GrepCursor", + _offset: 1, + } as GrepCursor; + const grepResult = ( + lineContent: string, + matchRanges: Array<[number, number]>, + cursor: GrepCursor | null, + ): GrepResult => ({ + items: [ + { + relativePath: "src/words.ts", + fileName: "words.ts", + gitStatus: "unmodified", + size: lineContent.length, + modified: 0, + isBinary: false, + totalFrecencyScore: 0, + accessFrecencyScore: 0, + modificationFrecencyScore: 0, + lineNumber: 1, + col: 0, + byteOffset: 0, + lineContent, + matchRanges, + }, + ], + totalMatched: 1, + totalFilesSearched: 1, + totalFiles: 1, + filteredFileCount: 1, + nextCursor: cursor, + }); + const grep = vi.fn((_query: string, options?: GrepOptions) => + options?.cursor + ? { ok: true as const, value: grepResult("needle", [[0, 6]], null) } + : { + ok: true as const, + value: grepResult("needleSuffix", [[0, 6]], nextCursor), + }, + ); + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + grep, + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); + const result = yield* searchIndex.searchContents({ + query: "needle", + limit: 1, + caseSensitive: true, + wholeWord: true, + useRegex: false, + }); + + expect(result).toEqual({ + matches: [ + { + path: "src/words.ts", + lineNumber: 1, + lineContent: "needle", + matchRanges: [{ start: 0, end: 6 }], + }, + ], + truncated: false, + }); + expect(grep).toHaveBeenCalledTimes(2); + expect(grep.mock.calls[1]?.[1]?.cursor).toBe(nextCursor); + }), + ), +); diff --git a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts new file mode 100644 index 0000000000000..e6e51eaa4c5f3 --- /dev/null +++ b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts @@ -0,0 +1,439 @@ +import { + type DirItem, + type DirSearchResult, + type FileItem, + FileFinder, + type GrepCursor, + type MixedItem, + type MixedSearchResult, + type Result, + type SearchResult, +} from "@ff-labs/fff-node"; +import * as Effect from "effect/Effect"; + +import type { + ProjectEntry, + ProjectSearchContentsInput, + ProjectSearchContentsResult, + ProjectSearchEntriesResult, +} from "@t3tools/contracts"; +import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; + +import { + WorkspaceSearchIndex, + WorkspaceSearchIndexCreateFailed, + WorkspaceSearchIndexDestroyFailed, + WorkspaceSearchIndexRefreshFailed, + WorkspaceSearchIndexScanTimedOut, + WorkspaceSearchIndexSearchFailed, + type WorkspaceSearchIndexVariant, +} from "./WorkspaceSearchIndexService.ts"; + +export * from "./WorkspaceSearchIndexService.ts"; + +const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; +const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; +const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; +const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000; +const CONTENT_SEARCH_TIME_BUDGET_MS = 250; +const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100; + +function toPosixPath(input: string): string { + return input.replaceAll("\\", "/"); +} + +function trimDirectorySeparator(input: string): string { + return input.endsWith("/") ? input.slice(0, -1) : input; +} + +function parentPathOf(input: string): string | undefined { + const separatorIndex = input.lastIndexOf("/"); + return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex); +} + +function toProjectEntry(item: MixedItem): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath)); + if (!normalizedPath) { + return null; + } + + return { + path: normalizedPath, + kind: item.type, + }; +} + +function toFileEntry(item: FileItem): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath)); + return normalizedPath ? { path: normalizedPath, kind: "file" } : null; +} + +function toDirectoryEntry(item: DirItem): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath)); + return normalizedPath ? { path: normalizedPath, kind: "directory" } : null; +} + +function mapFileSearchResult( + result: SearchResult, + limit: number, + imageOnly = false, +): ProjectSearchEntriesResult { + const entries = result.items.flatMap((item) => { + const entry = toFileEntry(item); + return entry && (!imageOnly || isWorkspaceImagePreviewPath(entry.path)) ? [entry] : []; + }); + return { + entries: entries.slice(0, limit), + truncated: entries.length > limit || result.totalMatched > result.items.length, + }; +} + +function mapDirectorySearchResult( + result: DirSearchResult, + limit: number, +): ProjectSearchEntriesResult { + const entries = result.items.flatMap((item) => { + const entry = toDirectoryEntry(item); + return entry ? [entry] : []; + }); + const rootDirectoryCount = result.items.some((item) => item.relativePath.length === 0) ? 1 : 0; + return { + entries: entries.slice(0, limit), + truncated: result.totalMatched - rootDirectoryCount > limit, + }; +} + +function mapMixedSearchResult( + result: MixedSearchResult, + limit: number, +): { readonly entries: ProjectEntry[]; readonly truncated: boolean } { + const entries: ProjectEntry[] = []; + for (const item of result.items) { + const entry = toProjectEntry(item); + if (entry) { + entries.push(entry); + } + if (entries.length >= limit) { + break; + } + } + + const rootDirectoryCount = result.items.some( + (item) => item.type === "directory" && item.item.relativePath.length === 0, + ) + ? 1 + : 0; + return { + entries, + truncated: result.totalMatched - rootDirectoryCount > limit, + }; +} + +const WORD_CHARACTER = /[\p{Letter}\p{Mark}\p{Number}_]/u; + +function codePointAt(line: string, index: number): string | undefined { + const codePoint = line.codePointAt(index); + return codePoint === undefined ? undefined : String.fromCodePoint(codePoint); +} + +function codePointBefore(line: string, index: number): string | undefined { + if (index <= 0) return undefined; + const previousCodeUnit = line.charCodeAt(index - 1); + const previousIndex = + previousCodeUnit >= 0xdc00 && previousCodeUnit <= 0xdfff ? index - 2 : index - 1; + return codePointAt(line, previousIndex); +} + +function buildContentSearchQuery(input: Omit): { + readonly searchQuery: string; + readonly regexMode: boolean; +} { + if (input.caseSensitive) { + return { searchQuery: input.query, regexMode: input.useRegex }; + } + // Plain mode relies on smart case: an all-lowercase needle matches + // case-insensitively. Regex mode needs an explicit inline flag instead. + return input.useRegex + ? { searchQuery: `(?i)${input.query}`, regexMode: true } + : { searchQuery: input.query.toLowerCase(), regexMode: false }; +} + +function mapContentMatchRanges( + line: string, + byteRanges: ReadonlyArray, +): Array<{ readonly start: number; readonly end: number }> { + const lineBytes = Buffer.from(line); + const toStringIndex = (byteOffset: number) => lineBytes.subarray(0, byteOffset).toString().length; + return byteRanges.map(([startByte, endByte]) => ({ + start: toStringIndex(startByte), + end: toStringIndex(endByte), + })); +} + +/** + * Whole-word filtering happens after the grep rather than by wrapping the + * pattern in boundary regex: consuming boundaries such as `(?:^|\W)` swallow + * the separator between adjacent matches and widen the reported ranges, and + * `\b` cannot match punctuation-edged queries at all. Matching VS Code, a + * match edge is a word boundary when it touches the line edge, the + * neighbouring character is not a word character, or the match's own edge + * character is not a word character. + */ +function isWholeWordRange( + line: string, + range: { readonly start: number; readonly end: number }, +): boolean { + if (range.end <= range.start) return false; + const isWord = (character: string | undefined) => + character !== undefined && WORD_CHARACTER.test(character); + const leftIsBoundary = + range.start === 0 || + !isWord(codePointBefore(line, range.start)) || + !isWord(codePointAt(line, range.start)); + const rightIsBoundary = + range.end >= line.length || + !isWord(codePointAt(line, range.end)) || + !isWord(codePointBefore(line, range.end)); + return leftIsBoundary && rightIsBoundary; +} + +function withDirectoryAncestors(entries: ReadonlyArray): ProjectEntry[] { + const entryByPath = new Map(entries.map((entry) => [entry.path, entry])); + for (const entry of entries) { + let parentPath = parentPathOf(entry.path); + while (parentPath) { + if (!entryByPath.has(parentPath)) { + entryByPath.set(parentPath, { path: parentPath, kind: "directory" }); + } + parentPath = parentPathOf(parentPath); + } + } + return [...entryByPath.values()]; +} + +const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* ( + cwd: string, + variant: WorkspaceSearchIndexVariant, +) { + const result = yield* Effect.try({ + try: () => + FileFinder.create({ + basePath: cwd, + disableMmapCache: true, + // Content indexing costs scan CPU and memory, so only the on-demand + // content-search index pays for it; path-only consumers (file tree, + // composer path search, file picker) keep the lightweight index. + disableContentIndexing: variant !== "content", + aiMode: false, + enableFsRootScanning: true, + enableHomeDirScanning: true, + }), + catch: (cause) => + new WorkspaceSearchIndexCreateFailed({ + cwd, + reason: "FileFinder.create threw unexpectedly.", + cause, + }), + }); + if (result.ok) return result.value; + return yield* new WorkspaceSearchIndexCreateFailed({ + cwd, + reason: result.error, + }); +}); + +const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(function* ( + cwd: string, + finder: FileFinder, + onFailure: (input: { readonly reason: string; readonly cause?: unknown }) => E, +): Effect.fn.Return { + const result = yield* Effect.tryPromise({ + try: () => finder.waitForIndexReady(WORKSPACE_INDEX_SCAN_TIMEOUT_MS), + catch: (cause) => + onFailure({ + reason: "FileFinder.waitForIndexReady rejected unexpectedly.", + cause, + }), + }); + if (!result.ok) { + return yield* Effect.fail(onFailure({ reason: result.error })); + } + if (!result.value) { + return yield* new WorkspaceSearchIndexScanTimedOut({ + cwd, + timeout: WORKSPACE_INDEX_SCAN_TIMEOUT, + }); + } +}); + +export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( + cwd: string, + variant: WorkspaceSearchIndexVariant = "paths", +) { + const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) => + Effect.try({ + try: () => finder.destroy(), + catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }), + }).pipe(Effect.orDie), + ); + yield* waitForIndexReady( + cwd, + finder, + ({ reason, cause }) => + new WorkspaceSearchIndexCreateFailed({ + cwd, + reason, + cause, + }), + ); + + const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* ( + query: string, + pageSize: number, + operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch", + execute: () => Result, + ): Effect.fn.Return { + const result = yield* Effect.try({ + try: execute, + catch: (cause) => + new WorkspaceSearchIndexSearchFailed({ + cwd, + queryLength: query.length, + pageSize, + reason: `FileFinder.${operation} threw unexpectedly.`, + cause, + }), + }); + if (!result.ok) { + return yield* new WorkspaceSearchIndexSearchFailed({ + cwd, + queryLength: query.length, + pageSize, + reason: result.error, + }); + } + return result.value; + }); + + const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( + "WorkspaceSearchIndex.refresh", + )(function* () { + const result = yield* Effect.try({ + try: () => finder.scanFiles(), + catch: (cause) => + new WorkspaceSearchIndexRefreshFailed({ + cwd, + reason: "FileFinder.scanFiles threw unexpectedly.", + cause, + }), + }); + if (!result.ok) { + return yield* new WorkspaceSearchIndexRefreshFailed({ + cwd, + reason: result.error, + }); + } + yield* waitForIndexReady( + cwd, + finder, + ({ reason, cause }) => + new WorkspaceSearchIndexRefreshFailed({ + cwd, + reason, + cause, + }), + ); + }); + + const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")( + function* () { + const result = yield* runSearch("", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", () => + finder.mixedSearch("", { pageSize: WORKSPACE_INDEX_PAGE_SIZE }), + ); + const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES); + const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => + left.path.localeCompare(right.path), + ); + const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES); + return { + entries, + truncated: mapped.truncated || entries.length < sortedEntries.length, + }; + }, + ); + + const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( + "WorkspaceSearchIndex.search", + )(function* (query, limit, kind, imageOnly) { + const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); + if (kind === "file" || imageOnly) { + const result = yield* runSearch(query, pageSize, "fileSearch", () => + finder.fileSearch(query, { pageSize }), + ); + return mapFileSearchResult(result, limit, imageOnly); + } + if (kind === "directory") { + const result = yield* runSearch(query, pageSize, "directorySearch", () => + finder.directorySearch(query, { pageSize }), + ); + return mapDirectorySearchResult(result, limit); + } + const result = yield* runSearch(query, pageSize, "mixedSearch", () => + finder.mixedSearch(query, { pageSize }), + ); + return mapMixedSearchResult(result, limit); + }); + + const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn( + "WorkspaceSearchIndex.searchContents", + )(function* (input) { + const { searchQuery, regexMode } = buildContentSearchQuery(input); + const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; + // Grep cursors advance by file, so whole-word post-filtering needs enough + // raw candidates from the current file before moving to the next one. + const rawPageSize = input.wholeWord + ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE) + : input.limit; + const matches: Array = []; + let nextCursor: GrepCursor | null = null; + let regexFallbackError: string | undefined; + + do { + const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); + const result = yield* runSearch(input.query, input.limit, "grep", () => + finder.grep(searchQuery, { + mode: regexMode ? "regex" : "plain", + smartCase: !input.caseSensitive && !regexMode, + // A single dense file must not consume the whole result page. + maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), + pageSize: rawPageSize, + cursor: nextCursor, + timeBudgetMs: remainingTimeBudgetMs, + }), + ); + + for (const match of result.items) { + const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( + (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), + ); + if (matchRanges.length === 0) continue; + matches.push({ + path: toPosixPath(match.relativePath), + lineNumber: match.lineNumber, + lineContent: match.lineContent, + matchRanges, + }); + } + nextCursor = result.nextCursor; + regexFallbackError ??= result.regexFallbackError; + } while (matches.length < input.limit && nextCursor !== null && performance.now() < deadline); + + return { + matches: matches.slice(0, input.limit), + truncated: matches.length > input.limit || nextCursor !== null, + ...(regexFallbackError !== undefined ? { regexFallbackError } : {}), + }; + }); + + return WorkspaceSearchIndex.of({ list, refresh, search, searchContents }); +}); diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index ff3343c371385..66568aa1ac18e 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -1,7 +1,6 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeFSP from "node:fs/promises"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { FileFinder } from "@ff-labs/fff-node"; import { it, afterEach, describe, expect } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -15,6 +14,8 @@ import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspaceEntries from "./WorkspaceEntries.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; +import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; +import { WorkspaceSearchWorkerPath } from "./WorkspaceSearchProcess.ts"; vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); @@ -339,19 +340,21 @@ it.layer(TestLayer, { excludeTestServices: true })("WorkspaceEntries", (it) => { const cwd = yield* makeTempDir({ prefix: "t3code-workspace-refresh-failure-" }); yield* writeTextFile(cwd, "src/index.ts", "export {};\n"); - const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; - const createSpy = vi.spyOn(FileFinder, "create"); - yield* workspaceEntries.list({ cwd }); - expect(createSpy).toHaveBeenCalledTimes(1); - - vi.spyOn(FileFinder.prototype, "scanFiles").mockReturnValueOnce({ - ok: false, - error: "scan failed", - }); - yield* workspaceEntries.refresh(cwd); - - yield* workspaceEntries.list({ cwd }); - expect(createSpy).toHaveBeenCalledTimes(2); + yield* Effect.gen(function* () { + const workspaceEntries = yield* WorkspaceEntries.make; + const before = yield* workspaceEntries.list({ cwd }); + yield* workspaceEntries.refresh(cwd); + const after = yield* workspaceEntries.list({ cwd }); + // The fixture returns its PID as the file path and rejects refresh. + expect(after.entries).not.toEqual(before.entries); + expect(after.entries).toHaveLength(1); + }).pipe( + Effect.provide(Layer.fresh(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layer)), + Effect.provideService( + WorkspaceSearchWorkerPath, + new URL("../../scripts/workspace-search-mock.ts", import.meta.url), + ), + ); }), ); }); diff --git a/apps/server/src/workspace/WorkspaceSearchHost.ts b/apps/server/src/workspace/WorkspaceSearchHost.ts new file mode 100644 index 0000000000000..4a9de2bbb2f43 --- /dev/null +++ b/apps/server/src/workspace/WorkspaceSearchHost.ts @@ -0,0 +1,93 @@ +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Semaphore from "effect/Semaphore"; +import { + startSearchProcess, + WorkspaceSearchProcessFailed, + type SearchProcess, +} from "./WorkspaceSearchProcess.ts"; +import type { WorkspaceSearchIndexVariant } from "./WorkspaceSearchIndexService.ts"; +import type { SearchOperation } from "./workspaceSearchProtocol.ts"; + +const make = Effect.gen(function* () { + const semaphore = yield* Semaphore.make(1); + let current: { process: SearchProcess; indexes: Set } | undefined; + let nextId = 0; + let closed = false; + const stop = Effect.fn("WorkspaceSearchHost.stop")(function* () { + const previous = current; + current = undefined; + if (previous) yield* previous.process.stop; + }); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + closed = true; + yield* stop(); + }), + ); + + const open = Effect.fn("WorkspaceSearchHost.open")(function* ( + cwd: string, + variant: WorkspaceSearchIndexVariant, + ) { + const id = ++nextId; + let released = false; + const initialize: SearchOperation = { method: "initialize", cwd, variant }; + const run = Effect.fn("WorkspaceSearchHost.request")( + function* (operation: SearchOperation) { + yield* Effect.annotateCurrentSpan({ cwd, variant, operation: operation.method }); + if (closed || released) + return yield* new WorkspaceSearchProcessFailed({ + cause: new Error("Workspace search index is closed."), + }); + return yield* Effect.gen(function* () { + const active = yield* Effect.gen(function* () { + if (current) return current; + const process = yield* startSearchProcess(); + current = { process, indexes: new Set() }; + return current; + }).pipe(Effect.uninterruptible); + if (!active.indexes.has(id)) { + yield* active.process.request({ id, operation: initialize }); + active.indexes.add(id); + } + return operation.method === "initialize" + ? null + : yield* active.process.request({ id, operation }); + }).pipe(Effect.onError(stop)); + }, + semaphore.withPermits(1), + Effect.timeout("20 seconds"), + ); + + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + released = true; + // Disposal must not wait behind a stuck search or native destructor. + // Other handles rebuild lazily if this retires the shared process. + yield* Effect.acquireUseRelease( + semaphore.take(1).pipe(Effect.timeout("1 second"), Effect.onError(stop)), + () => + Effect.gen(function* () { + if (!current?.indexes.has(id)) return; + current.indexes.delete(id); + if (current.indexes.size === 0) return yield* stop(); + yield* current.process.request({ id, operation: { method: "dispose" } }); + }).pipe(Effect.timeout("1 second"), Effect.onError(stop)), + () => semaphore.release(1), + ).pipe(Effect.catchCause(() => Effect.void)); + }), + ); + yield* run(initialize); + return { request: run }; + }); + return { open }; +}); + +export class WorkspaceSearchHost extends Context.Service< + WorkspaceSearchHost, + Effect.Success +>()("t3/workspace/WorkspaceSearchHost") { + static readonly layer = Layer.effect(WorkspaceSearchHost, make); +} diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 1fdf956447d45..8b3fb682e795a 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -1,348 +1,227 @@ -import { - FileFinder, - type FileItem, - type GrepCursor, - type GrepOptions, - type GrepResult, -} from "@ff-labs/fff-node"; -import { afterEach, expect, it } from "@effect/vitest"; -import * as Cause from "effect/Cause"; +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeNet from "node:net"; +import * as NodeHttp from "node:http"; +import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import { expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as Scope from "effect/Scope"; import * as Exit from "effect/Exit"; -import { vi } from "vite-plus/test"; - -import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; - -afterEach(() => { - vi.restoreAllMocks(); -}); - -function fileItem(relativePath: string): FileItem { +import * as TestClock from "effect/testing/TestClock"; +import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; +import * as Index from "./WorkspaceSearchIndex.ts"; +import { WorkspaceSearchWorkerPath } from "./WorkspaceSearchProcess.ts"; +import { WorkspaceSearchHost } from "./WorkspaceSearchHost.ts"; + +const mockWorker = new URL("../../scripts/workspace-search-mock.ts", import.meta.url); +const withWorker = (effect: Effect.Effect) => + effect.pipe( + Effect.provide(WorkspaceSearchHost.layer), + Effect.provideService(WorkspaceSearchWorkerPath, mockWorker), + ); + +const receipts = Effect.fn(function* () { + const blocked = yield* Deferred.make(); + const exited = yield* Deferred.make(); + const server = NodeNet.createServer((socket) => { + socket.once("data", () => Deferred.doneUnsafe(blocked, Effect.void)); + socket.once("close", () => Deferred.doneUnsafe(exited, Effect.void)); + }); + yield* Effect.acquireRelease( + Effect.promise(() => new Promise((resolve) => server.listen(0, "127.0.0.1", resolve))), + () => Effect.promise(() => new Promise((resolve) => server.close(() => resolve()))), + ); + const address = server.address(); + if (!address || typeof address === "string") return yield* Effect.die("Missing receipt port"); return { - relativePath, - fileName: relativePath.slice(relativePath.lastIndexOf("/") + 1), - size: 1, - modified: 0, - accessFrecencyScore: 0, - modificationFrecencyScore: 0, - totalFrecencyScore: 0, - gitStatus: "clean", + blocked, + exited, + environment: { ...process.env, T3_SEARCH_TEST_RECEIPT_PORT: String(address.port) }, }; -} +}); + +it.effect( + "keeps HTTP responsive during a blocked search and rebuilds shared indexes after timeout", + () => + Effect.scoped( + Effect.gen(function* () { + const control = yield* receipts(); + const server = NodeHttp.createServer((_request, response) => response.end("healthy")); + yield* Effect.acquireRelease( + Effect.promise( + () => new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)), + ), + () => Effect.promise(() => new Promise((resolve) => server.close(() => resolve()))), + ); + const address = server.address(); + if (!address || typeof address === "string") return yield* Effect.die("Missing HTTP port"); + yield* Effect.gen(function* () { + const index = yield* Index.make("workspace"); + const initial = yield* index.list(); + const other = yield* Index.make("another-workspace"); + expect((yield* other.list()).entries).toEqual(initial.entries); + const blocked = yield* index.search("block", 1).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(control.blocked); + + const health = yield* HttpClient.get(`http://127.0.0.1:${address.port}`).pipe( + Effect.flatMap((response) => response.text), + Effect.provide(NodeHttpClient.layerNodeHttp), + ); + expect(health).toBe("healthy"); + yield* TestClock.adjust("1 second"); + const queued = yield* other.list().pipe(Effect.forkChild); + + yield* TestClock.adjust("19 seconds"); + const result = yield* Fiber.join(blocked); + expect(Exit.isFailure(result)).toBe(true); + yield* Deferred.await(control.exited); + const recovered = yield* Fiber.join(queued); + expect(recovered.entries).not.toEqual(initial.entries); + expect((yield* index.list()).entries).toEqual(recovered.entries); + }).pipe(Effect.provideService(HostProcessEnvironment, control.environment)); + }).pipe(withWorker), + ), +); -it.effect("filters image searches before applying the result limit", () => +it.effect("cancels a blocked request before allowing the queued request to rebuild", () => Effect.scoped( Effect.gen(function* () { - const items = [ - ...Array.from({ length: 200 }, (_, index) => fileItem(`src/file-${index}.ts`)), - fileItem("public/icon.svg"), - ]; - const fileSearch = vi.fn(() => ({ - ok: true as const, - value: { - items, - scores: [], - totalMatched: items.length, - totalFiles: items.length, - }, - })); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); - const resultWithoutKind = yield* searchIndex.search("", 200, undefined, true); - const resultWithDirectoryKind = yield* searchIndex.search("", 200, "directory", true); - - expect(resultWithoutKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(resultWithDirectoryKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(fileSearch).toHaveBeenCalledTimes(2); - expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 }); - }), + const control = yield* receipts(); + yield* Effect.gen(function* () { + const index = yield* Index.make("workspace"); + const original = yield* index.list(); + const blocked = yield* index.search("block", 1).pipe(Effect.forkChild); + yield* Deferred.await(control.blocked); + const queued = yield* index.list().pipe(Effect.forkChild); + yield* Fiber.interrupt(blocked); + yield* Deferred.await(control.exited); + expect((yield* Fiber.join(queued)).entries).not.toEqual(original.entries); + }).pipe(Effect.provideService(HostProcessEnvironment, control.environment)); + }).pipe(withWorker), ), ); -it.effect("preserves unexpected FileFinder creation failures", () => - Effect.gen(function* () { - const cause = new Error("native initialization failed"); - vi.spyOn(FileFinder, "create").mockImplementationOnce(() => { - throw cause; - }); - - const error = yield* Effect.flip( - Effect.scoped(WorkspaceSearchIndex.make("/workspace/project")), - ); - - expect(error).toMatchObject({ - _tag: "WorkspaceSearchIndexCreateFailed", - cwd: "/workspace/project", - reason: "FileFinder.create threw unexpectedly.", - cause, - }); - }), -); - -it.effect("keeps returned FileFinder creation diagnostics out of the cause chain", () => - Effect.gen(function* () { - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ - ok: false, - error: "native index rejected the directory", - }); - - const error = yield* Effect.flip( - Effect.scoped(WorkspaceSearchIndex.make("/workspace/project")), - ); - - expect(error).toMatchObject({ - _tag: "WorkspaceSearchIndexCreateFailed", - cwd: "/workspace/project", - reason: "native index rejected the directory", - }); - expect(error.cause).toBeUndefined(); - }), -); - -it.effect("waits for the full content index warmup before returning", () => - Effect.gen(function* () { - const waitForIndexReady = vi.fn(async () => ({ ok: true as const, value: true })); - const finder = { - destroy: vi.fn(), - waitForIndexReady, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - yield* Effect.scoped(WorkspaceSearchIndex.make("/workspace/project", "content")); - - expect(waitForIndexReady).toHaveBeenCalledWith(15_000); - }), -); - -it.effect("preserves a full-index warmup timeout as a structured error", () => - Effect.gen(function* () { - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: false })), - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const error = yield* Effect.flip( - Effect.scoped(WorkspaceSearchIndex.make("/workspace/project", "content")), - ); - - expect(error).toMatchObject({ - _tag: "WorkspaceSearchIndexScanTimedOut", - cwd: "/workspace/project", - timeout: "15 seconds", - }); - }), +it.effect("bounds initialization even when the native scan never returns", () => + Effect.scoped( + Effect.gen(function* () { + const control = yield* receipts(); + const startup = yield* Index.make("block-initialize").pipe( + Effect.provideService(HostProcessEnvironment, control.environment), + Effect.exit, + Effect.forkChild, + ); + yield* Deferred.await(control.blocked); + yield* TestClock.adjust("20 seconds"); + expect(Exit.isFailure(yield* Fiber.join(startup))).toBe(true); + yield* Deferred.await(control.exited); + }).pipe(withWorker), + ), ); -it.effect("preserves FileFinder destroy failures as structured defects", () => - Effect.gen(function* () { - const cause = new Error("native destroy failed"); - const finder = { - destroy: vi.fn(() => { - throw cause; - }), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const exit = yield* Effect.scoped(WorkspaceSearchIndex.make("/workspace/project")).pipe( - Effect.exit, - ); - - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - expect(Cause.hasDies(exit.cause)).toBe(true); - const error = Cause.squash(exit.cause); - expect(error).toBeInstanceOf(WorkspaceSearchIndex.WorkspaceSearchIndexDestroyFailed); - expect(error).toMatchObject({ - _tag: "WorkspaceSearchIndexDestroyFailed", - cwd: "/workspace/project", - cause, - }); - } - }), +it.effect("kills a blocked native call when the owning index scope closes", () => + Effect.scoped( + Effect.gen(function* () { + const control = yield* receipts(); + const scope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); + const index = yield* Index.make("workspace").pipe( + Scope.provide(scope), + Effect.provideService(HostProcessEnvironment, control.environment), + ); + const blocked = yield* index.search("block", 1).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(control.blocked); + const closing = yield* Scope.close(scope, Exit.void).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(closing); + yield* Deferred.await(control.exited); + expect(Exit.isFailure(yield* Fiber.join(blocked))).toBe(true); + }).pipe(withWorker), + ), ); -it.effect("preserves search and refresh failures with operation context", () => +it.effect("preserves native timeout and refresh errors across IPC and recovers after a crash", () => Effect.scoped( Effect.gen(function* () { - const searchCause = new Error("native search failed"); - const refreshCause = new Error("native scan failed"); - const contentSearchCause = new Error("native grep failed"); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - mixedSearch: vi.fn(() => { - throw searchCause; - }), - grep: vi.fn(() => { - throw contentSearchCause; - }), - scanFiles: vi.fn(() => { - throw refreshCause; - }), - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); - const query = "authorization: Bearer secret-token"; - const searchError = yield* Effect.flip(searchIndex.search(query, 3)); - const contentSearchError = yield* Effect.flip( - searchIndex.searchContents({ - query, - limit: 3, - caseSensitive: false, - wholeWord: false, - useRegex: false, - }), + expect(yield* Effect.flip(Index.make("scan-timeout"))).toBeInstanceOf( + Index.WorkspaceSearchIndexScanTimedOut, ); - const refreshError = yield* Effect.flip(searchIndex.refresh()); - - expect(searchError).toMatchObject({ - _tag: "WorkspaceSearchIndexSearchFailed", - cwd: "/workspace/project", - queryLength: query.length, - pageSize: 4, - reason: "FileFinder.mixedSearch threw unexpectedly.", - cause: searchCause, - }); - expect(searchError).not.toHaveProperty("query"); - expect(searchError.message).not.toMatch(/Bearer|secret-token/); - expect(contentSearchError).toMatchObject({ - _tag: "WorkspaceSearchIndexSearchFailed", - cwd: "/workspace/project", - queryLength: query.length, - pageSize: 3, - reason: "FileFinder.grep threw unexpectedly.", - cause: contentSearchCause, - }); - expect(contentSearchError).not.toHaveProperty("query"); - expect(contentSearchError.message).not.toMatch(/Bearer|secret-token/); - expect(refreshError).toMatchObject({ + const index = yield* Index.make("workspace"); + const original = yield* index.list(); + expect(yield* Effect.flip(index.search("crash", 1))).toBeInstanceOf( + Index.WorkspaceSearchIndexSearchFailed, + ); + expect((yield* index.list()).entries).not.toEqual(original.entries); + expect(yield* Effect.flip(index.refresh())).toMatchObject({ _tag: "WorkspaceSearchIndexRefreshFailed", - cwd: "/workspace/project", - reason: "FileFinder.scanFiles threw unexpectedly.", - cause: refreshCause, + reason: "scan failed", }); - }), + expect((yield* index.list()).entries).toHaveLength(1); + }).pipe(withWorker), ), ); -it.effect("keeps returned search diagnostics out of the cause chain", () => +it.effect("bounds native disposal and lazily rebuilds the surviving workspace", () => Effect.scoped( Effect.gen(function* () { - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - mixedSearch: vi.fn(() => ({ ok: false, error: "native query rejected" })), - scanFiles: vi.fn(() => ({ ok: false, error: "native refresh rejected" })), - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); - const query = "authorization: Bearer secret-token"; - const searchError = yield* Effect.flip(searchIndex.search(query, 3)); - const refreshError = yield* Effect.flip(searchIndex.refresh()); - - expect(searchError).toMatchObject({ - _tag: "WorkspaceSearchIndexSearchFailed", - cwd: "/workspace/project", - queryLength: query.length, - pageSize: 4, - reason: "native query rejected", - }); - expect(searchError).not.toHaveProperty("query"); - expect(searchError.message).not.toMatch(/Bearer|secret-token/); - expect(searchError.cause).toBeUndefined(); - expect(refreshError).toMatchObject({ - _tag: "WorkspaceSearchIndexRefreshFailed", - cwd: "/workspace/project", - reason: "native refresh rejected", - }); - expect(refreshError.cause).toBeUndefined(); - }), + const control = yield* receipts(); + yield* Effect.gen(function* () { + const scope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); + const retiring = yield* Index.make("block-dispose").pipe(Scope.provide(scope)); + const surviving = yield* Index.make("surviving"); + const before = yield* surviving.list(); + expect((yield* retiring.list()).entries).toEqual(before.entries); + const closing = yield* Scope.close(scope, Exit.void).pipe(Effect.forkChild); + yield* Deferred.await(control.blocked); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(closing); + yield* Deferred.await(control.exited); + expect((yield* surviving.list()).entries).not.toEqual(before.entries); + expect(yield* Effect.flip(retiring.list())).toBeInstanceOf( + Index.WorkspaceSearchIndexSearchFailed, + ); + }).pipe(Effect.provideService(HostProcessEnvironment, control.environment)); + }).pipe(withWorker), ), ); -it.effect("continues whole-word searches after a filtered grep page", () => +it.effect("releases one index without restarting the process used by another index", () => Effect.scoped( Effect.gen(function* () { - const nextCursor = { - __brand: "GrepCursor", - _offset: 1, - } as GrepCursor; - const grepResult = ( - lineContent: string, - matchRanges: Array<[number, number]>, - cursor: GrepCursor | null, - ): GrepResult => ({ - items: [ - { - relativePath: "src/words.ts", - fileName: "words.ts", - gitStatus: "unmodified", - size: lineContent.length, - modified: 0, - isBinary: false, - totalFrecencyScore: 0, - accessFrecencyScore: 0, - modificationFrecencyScore: 0, - lineNumber: 1, - col: 0, - byteOffset: 0, - lineContent, - matchRanges, - }, - ], - totalMatched: 1, - totalFilesSearched: 1, - totalFiles: 1, - filteredFileCount: 1, - nextCursor: cursor, - }); - const grep = vi.fn((_query: string, options?: GrepOptions) => - options?.cursor - ? { ok: true as const, value: grepResult("needle", [[0, 6]], null) } - : { - ok: true as const, - value: grepResult("needleSuffix", [[0, 6]], nextCursor), - }, + const scope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); + const retiring = yield* Index.make("retiring").pipe(Scope.provide(scope)); + const surviving = yield* Index.make("surviving"); + const before = yield* surviving.list(); + expect((yield* retiring.list()).entries).toEqual(before.entries); + yield* Scope.close(scope, Exit.void); + expect((yield* surviving.list()).entries).toEqual(before.entries); + expect(yield* Effect.flip(retiring.list())).toBeInstanceOf( + Index.WorkspaceSearchIndexSearchFailed, ); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - grep, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); - const result = yield* searchIndex.searchContents({ - query: "needle", - limit: 1, - caseSensitive: true, - wholeWord: true, - useRegex: false, - }); + expect((yield* surviving.list()).entries).toEqual(before.entries); + }).pipe(withWorker), + ), +); - expect(result).toEqual({ - matches: [ - { - path: "src/words.ts", - lineNumber: 1, - lineContent: "needle", - matchRanges: [{ start: 0, end: 6 }], - }, - ], - truncated: false, - }); - expect(grep).toHaveBeenCalledTimes(2); - expect(grep.mock.calls[1]?.[1]?.cursor).toBe(nextCursor); - }), +it.effect("cancels a queued request without terminating the active search process", () => + Effect.scoped( + Effect.gen(function* () { + const control = yield* receipts(); + yield* Effect.gen(function* () { + const index = yield* Index.make("workspace"); + const other = yield* Index.make("other"); + const blocked = yield* index.search("block", 1).pipe(Effect.forkChild); + yield* Deferred.await(control.blocked); + const queued = yield* other.list().pipe(Effect.forkChild); + yield* Fiber.interrupt(queued); + expect(yield* Deferred.isDone(control.exited)).toBe(false); + yield* Fiber.interrupt(blocked); + yield* Deferred.await(control.exited); + expect((yield* other.list()).entries).toHaveLength(1); + }).pipe(Effect.provideService(HostProcessEnvironment, control.environment)); + }).pipe(withWorker), ), ); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 44a9c3397c6ff..993cec0cef99b 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -1,532 +1,95 @@ -import { - type DirItem, - type DirSearchResult, - type FileItem, - FileFinder, - type GrepCursor, - type MixedItem, - type MixedSearchResult, - type Result, - type SearchResult, -} from "@ff-labs/fff-node"; -import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as LayerMap from "effect/LayerMap"; import * as Schema from "effect/Schema"; - -import type { - ProjectEntry, - ProjectEntryKind, +import { ProjectListEntriesResult, - ProjectSearchContentsInput, - ProjectSearchContentsResult, ProjectSearchEntriesResult, + ProjectSearchContentsResult, } from "@t3tools/contracts"; -import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; - -const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; -const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; -const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; -const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000; -const WORKSPACE_INDEX_IDLE_TTL = "15 minutes"; -const CONTENT_SEARCH_TIME_BUDGET_MS = 250; -const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100; - -export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError()( - "WorkspaceSearchIndexCreateFailed", - { - cwd: Schema.String, - reason: Schema.String, - cause: Schema.optional(Schema.Defect()), - }, -) { - override get message(): string { - return `Failed to create the workspace search index for '${this.cwd}'.`; - } -} - -export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedError()( - "WorkspaceSearchIndexScanTimedOut", - { - cwd: Schema.String, - timeout: Schema.String, - }, -) { - override get message(): string { - return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`; - } -} - -export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedError()( - "WorkspaceSearchIndexSearchFailed", - { - cwd: Schema.String, - queryLength: Schema.Number, - pageSize: Schema.Number, - reason: Schema.String, - cause: Schema.optional(Schema.Defect()), - }, -) { - override get message(): string { - return `Workspace search failed for '${this.cwd}'.`; - } -} - -export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedError()( - "WorkspaceSearchIndexRefreshFailed", - { - cwd: Schema.String, - reason: Schema.String, - cause: Schema.optional(Schema.Defect()), - }, -) { - override get message(): string { - return `Failed to refresh the workspace search index for '${this.cwd}'.`; - } -} - -export class WorkspaceSearchIndexDestroyFailed extends Schema.TaggedError()( - "WorkspaceSearchIndexDestroyFailed", - { - cwd: Schema.String, - cause: Schema.Defect(), - }, -) { - override get message(): string { - return `Failed to destroy the workspace search index for '${this.cwd}'.`; - } -} - -export type WorkspaceSearchIndexError = - | WorkspaceSearchIndexCreateFailed - | WorkspaceSearchIndexScanTimedOut - | WorkspaceSearchIndexSearchFailed - | WorkspaceSearchIndexRefreshFailed; - -export class WorkspaceSearchIndex extends Context.Service< +import { WorkspaceSearchIndex, - { - readonly list: () => Effect.Effect; - readonly search: ( - query: string, - limit: number, - kind?: ProjectEntryKind, - imageOnly?: boolean, - ) => Effect.Effect; - readonly searchContents: ( - input: Omit, - ) => Effect.Effect; - readonly refresh: () => Effect.Effect< - void, - WorkspaceSearchIndexRefreshFailed | WorkspaceSearchIndexScanTimedOut - >; - } ->()("t3/workspace/WorkspaceSearchIndex") {} - -function toPosixPath(input: string): string { - return input.replaceAll("\\", "/"); -} - -function trimDirectorySeparator(input: string): string { - return input.endsWith("/") ? input.slice(0, -1) : input; -} - -function parentPathOf(input: string): string | undefined { - const separatorIndex = input.lastIndexOf("/"); - return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex); -} - -function toProjectEntry(item: MixedItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath)); - if (!normalizedPath) { - return null; - } - - return { - path: normalizedPath, - kind: item.type, - }; -} - -function toFileEntry(item: FileItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath)); - return normalizedPath ? { path: normalizedPath, kind: "file" } : null; -} - -function toDirectoryEntry(item: DirItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath)); - return normalizedPath ? { path: normalizedPath, kind: "directory" } : null; -} - -function mapFileSearchResult( - result: SearchResult, - limit: number, - imageOnly = false, -): ProjectSearchEntriesResult { - const entries = result.items.flatMap((item) => { - const entry = toFileEntry(item); - return entry && (!imageOnly || isWorkspaceImagePreviewPath(entry.path)) ? [entry] : []; - }); - return { - entries: entries.slice(0, limit), - truncated: entries.length > limit || result.totalMatched > result.items.length, - }; -} - -function mapDirectorySearchResult( - result: DirSearchResult, - limit: number, -): ProjectSearchEntriesResult { - const entries = result.items.flatMap((item) => { - const entry = toDirectoryEntry(item); - return entry ? [entry] : []; - }); - const rootDirectoryCount = result.items.some((item) => item.relativePath.length === 0) ? 1 : 0; - return { - entries: entries.slice(0, limit), - truncated: result.totalMatched - rootDirectoryCount > limit, - }; -} - -function mapMixedSearchResult( - result: MixedSearchResult, - limit: number, -): { readonly entries: ProjectEntry[]; readonly truncated: boolean } { - const entries: ProjectEntry[] = []; - for (const item of result.items) { - const entry = toProjectEntry(item); - if (entry) { - entries.push(entry); - } - if (entries.length >= limit) { - break; - } - } - - const rootDirectoryCount = result.items.some( - (item) => item.type === "directory" && item.item.relativePath.length === 0, - ) - ? 1 - : 0; - return { - entries, - truncated: result.totalMatched - rootDirectoryCount > limit, - }; -} - -const WORD_CHARACTER = /[\p{Letter}\p{Mark}\p{Number}_]/u; - -function codePointAt(line: string, index: number): string | undefined { - const codePoint = line.codePointAt(index); - return codePoint === undefined ? undefined : String.fromCodePoint(codePoint); -} - -function codePointBefore(line: string, index: number): string | undefined { - if (index <= 0) return undefined; - const previousCodeUnit = line.charCodeAt(index - 1); - const previousIndex = - previousCodeUnit >= 0xdc00 && previousCodeUnit <= 0xdfff ? index - 2 : index - 1; - return codePointAt(line, previousIndex); -} - -function buildContentSearchQuery(input: Omit): { - readonly searchQuery: string; - readonly regexMode: boolean; -} { - if (input.caseSensitive) { - return { searchQuery: input.query, regexMode: input.useRegex }; - } - // Plain mode relies on smart case: an all-lowercase needle matches - // case-insensitively. Regex mode needs an explicit inline flag instead. - return input.useRegex - ? { searchQuery: `(?i)${input.query}`, regexMode: true } - : { searchQuery: input.query.toLowerCase(), regexMode: false }; -} - -function mapContentMatchRanges( - line: string, - byteRanges: ReadonlyArray, -): Array<{ readonly start: number; readonly end: number }> { - const lineBytes = Buffer.from(line); - const toStringIndex = (byteOffset: number) => lineBytes.subarray(0, byteOffset).toString().length; - return byteRanges.map(([startByte, endByte]) => ({ - start: toStringIndex(startByte), - end: toStringIndex(endByte), - })); -} - -/** - * Whole-word filtering happens after the grep rather than by wrapping the - * pattern in boundary regex: consuming boundaries such as `(?:^|\W)` swallow - * the separator between adjacent matches and widen the reported ranges, and - * `\b` cannot match punctuation-edged queries at all. Matching VS Code, a - * match edge is a word boundary when it touches the line edge, the - * neighbouring character is not a word character, or the match's own edge - * character is not a word character. - */ -function isWholeWordRange( - line: string, - range: { readonly start: number; readonly end: number }, -): boolean { - if (range.end <= range.start) return false; - const isWord = (character: string | undefined) => - character !== undefined && WORD_CHARACTER.test(character); - const leftIsBoundary = - range.start === 0 || - !isWord(codePointBefore(line, range.start)) || - !isWord(codePointAt(line, range.start)); - const rightIsBoundary = - range.end >= line.length || - !isWord(codePointAt(line, range.end)) || - !isWord(codePointBefore(line, range.end)); - return leftIsBoundary && rightIsBoundary; -} - -function withDirectoryAncestors(entries: ReadonlyArray): ProjectEntry[] { - const entryByPath = new Map(entries.map((entry) => [entry.path, entry])); - for (const entry of entries) { - let parentPath = parentPathOf(entry.path); - while (parentPath) { - if (!entryByPath.has(parentPath)) { - entryByPath.set(parentPath, { path: parentPath, kind: "directory" }); - } - parentPath = parentPathOf(parentPath); - } - } - return [...entryByPath.values()]; -} - -const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* ( - cwd: string, - variant: WorkspaceSearchIndexVariant, -) { - const result = yield* Effect.try({ - try: () => - FileFinder.create({ - basePath: cwd, - disableMmapCache: true, - // Content indexing costs scan CPU and memory, so only the on-demand - // content-search index pays for it; path-only consumers (file tree, - // composer path search, file picker) keep the lightweight index. - disableContentIndexing: variant !== "content", - aiMode: false, - enableFsRootScanning: true, - enableHomeDirScanning: true, - }), - catch: (cause) => - new WorkspaceSearchIndexCreateFailed({ - cwd, - reason: "FileFinder.create threw unexpectedly.", - cause, - }), - }); - if (result.ok) return result.value; - return yield* new WorkspaceSearchIndexCreateFailed({ - cwd, - reason: result.error, - }); -}); - -const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(function* ( - cwd: string, - finder: FileFinder, - onFailure: (input: { readonly reason: string; readonly cause?: unknown }) => E, -): Effect.fn.Return { - const result = yield* Effect.tryPromise({ - try: () => finder.waitForIndexReady(WORKSPACE_INDEX_SCAN_TIMEOUT_MS), - catch: (cause) => - onFailure({ - reason: "FileFinder.waitForIndexReady rejected unexpectedly.", - cause, - }), - }); - if (!result.ok) { - return yield* Effect.fail(onFailure({ reason: result.error })); - } - if (!result.value) { - return yield* new WorkspaceSearchIndexScanTimedOut({ - cwd, - timeout: WORKSPACE_INDEX_SCAN_TIMEOUT, - }); - } -}); + WorkspaceSearchIndexCreateFailed, + WorkspaceSearchIndexRefreshFailed, + WorkspaceSearchIndexScanTimedOut, + WorkspaceSearchIndexSearchFailed, + type WorkspaceSearchIndexVariant, +} from "./WorkspaceSearchIndexService.ts"; +import { WorkspaceSearchHost } from "./WorkspaceSearchHost.ts"; + +export * from "./WorkspaceSearchIndexService.ts"; +const WORKSPACE_INDEX_IDLE_TTL = "15 minutes"; +const isCreateFailed = Schema.is(WorkspaceSearchIndexCreateFailed); +const isScanTimedOut = Schema.is(WorkspaceSearchIndexScanTimedOut); +const isSearchFailed = Schema.is(WorkspaceSearchIndexSearchFailed); +const isRefreshFailed = Schema.is(WorkspaceSearchIndexRefreshFailed); +const decodeList = Schema.decodeUnknownEffect(ProjectListEntriesResult); +const decodeSearch = Schema.decodeUnknownEffect(ProjectSearchEntriesResult); +const decodeContents = Schema.decodeUnknownEffect(ProjectSearchContentsResult); export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cwd: string, variant: WorkspaceSearchIndexVariant = "paths", ) { - const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) => - Effect.try({ - try: () => finder.destroy(), - catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }), - }).pipe(Effect.orDie), - ); - yield* waitForIndexReady( - cwd, - finder, - ({ reason, cause }) => - new WorkspaceSearchIndexCreateFailed({ - cwd, - reason, - cause, - }), + const host = yield* WorkspaceSearchHost; + const remote = yield* host.open(cwd, variant).pipe( + Effect.mapError((cause) => + isCreateFailed(cause) || isScanTimedOut(cause) + ? cause + : new WorkspaceSearchIndexCreateFailed({ + cwd, + reason: "Workspace search process could not initialize.", + cause, + }), + ), ); - const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* ( - query: string, - pageSize: number, - operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch", - execute: () => Result, - ): Effect.fn.Return { - const result = yield* Effect.try({ - try: execute, - catch: (cause) => - new WorkspaceSearchIndexSearchFailed({ + const searchFailure = (queryLength: number, pageSize: number) => (cause: unknown) => + isSearchFailed(cause) + ? cause + : new WorkspaceSearchIndexSearchFailed({ cwd, - queryLength: query.length, + queryLength, pageSize, - reason: `FileFinder.${operation} threw unexpectedly.`, + reason: "Workspace search process failed.", cause, - }), - }); - if (!result.ok) { - return yield* new WorkspaceSearchIndexSearchFailed({ - cwd, - queryLength: query.length, - pageSize, - reason: result.error, - }); - } - return result.value; - }); - - const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( - "WorkspaceSearchIndex.refresh", - )(function* () { - const result = yield* Effect.try({ - try: () => finder.scanFiles(), - catch: (cause) => - new WorkspaceSearchIndexRefreshFailed({ - cwd, - reason: "FileFinder.scanFiles threw unexpectedly.", - cause, - }), - }); - if (!result.ok) { - return yield* new WorkspaceSearchIndexRefreshFailed({ - cwd, - reason: result.error, - }); - } - yield* waitForIndexReady( - cwd, - finder, - ({ reason, cause }) => - new WorkspaceSearchIndexRefreshFailed({ - cwd, - reason, - cause, - }), - ); - }); - - const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")( - function* () { - const result = yield* runSearch("", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", () => - finder.mixedSearch("", { pageSize: WORKSPACE_INDEX_PAGE_SIZE }), - ); - const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES); - const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => - left.path.localeCompare(right.path), - ); - const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES); - return { - entries, - truncated: mapped.truncated || entries.length < sortedEntries.length, - }; - }, - ); - - const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( - "WorkspaceSearchIndex.search", - )(function* (query, limit, kind, imageOnly) { - const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); - if (kind === "file" || imageOnly) { - const result = yield* runSearch(query, pageSize, "fileSearch", () => - finder.fileSearch(query, { pageSize }), - ); - return mapFileSearchResult(result, limit, imageOnly); - } - if (kind === "directory") { - const result = yield* runSearch(query, pageSize, "directorySearch", () => - finder.directorySearch(query, { pageSize }), - ); - return mapDirectorySearchResult(result, limit); - } - const result = yield* runSearch(query, pageSize, "mixedSearch", () => - finder.mixedSearch(query, { pageSize }), - ); - return mapMixedSearchResult(result, limit); - }); - - const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn( - "WorkspaceSearchIndex.searchContents", - )(function* (input) { - const { searchQuery, regexMode } = buildContentSearchQuery(input); - const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; - // Grep cursors advance by file, so whole-word post-filtering needs enough - // raw candidates from the current file before moving to the next one. - const rawPageSize = input.wholeWord - ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE) - : input.limit; - const matches: Array = []; - let nextCursor: GrepCursor | null = null; - let regexFallbackError: string | undefined; - - do { - const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); - const result = yield* runSearch(input.query, input.limit, "grep", () => - finder.grep(searchQuery, { - mode: regexMode ? "regex" : "plain", - smartCase: !input.caseSensitive && !regexMode, - // A single dense file must not consume the whole result page. - maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), - pageSize: rawPageSize, - cursor: nextCursor, - timeBudgetMs: remainingTimeBudgetMs, - }), - ); - - for (const match of result.items) { - const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( - (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), - ); - if (matchRanges.length === 0) continue; - matches.push({ - path: toPosixPath(match.relativePath), - lineNumber: match.lineNumber, - lineContent: match.lineContent, - matchRanges, }); - } - nextCursor = result.nextCursor; - regexFallbackError ??= result.regexFallbackError; - } while (matches.length < input.limit && nextCursor !== null && performance.now() < deadline); - return { - matches: matches.slice(0, input.limit), - truncated: matches.length > input.limit || nextCursor !== null, - ...(regexFallbackError !== undefined ? { regexFallbackError } : {}), - }; + return WorkspaceSearchIndex.of({ + list: () => + remote + .request({ method: "list" }) + .pipe(Effect.flatMap(decodeList), Effect.mapError(searchFailure(0, 25_002))), + search: (query, limit, kind, imageOnly) => + remote + .request({ method: "search", query, limit, kind, imageOnly }) + .pipe( + Effect.flatMap(decodeSearch), + Effect.mapError(searchFailure(query.length, limit + 1)), + ), + searchContents: (input) => + remote + .request({ method: "searchContents", input }) + .pipe( + Effect.flatMap(decodeContents), + Effect.mapError(searchFailure(input.query.length, input.limit)), + ), + refresh: () => + remote.request({ method: "refresh" }).pipe( + Effect.asVoid, + Effect.mapError((cause) => + isRefreshFailed(cause) || isScanTimedOut(cause) + ? cause + : new WorkspaceSearchIndexRefreshFailed({ + cwd, + reason: "Workspace search process failed.", + cause, + }), + ), + ), }); - - return WorkspaceSearchIndex.of({ list, refresh, search, searchContents }); }); -export const WORKSPACE_SEARCH_INDEX_VARIANTS = ["paths", "content"] as const; -export type WorkspaceSearchIndexVariant = (typeof WORKSPACE_SEARCH_INDEX_VARIANTS)[number]; - /** * Composite LayerMap key so the lightweight path index and the on-demand * content-search index of the same workspace are separate resources with @@ -563,6 +126,7 @@ export class WorkspaceSearchIndexMap extends LayerMap.Service()( + "WorkspaceSearchIndexCreateFailed", + { + cwd: Schema.String, + reason: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return `Failed to create the workspace search index for '${this.cwd}'.`; + } +} + +export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedError()( + "WorkspaceSearchIndexScanTimedOut", + { + cwd: Schema.String, + timeout: Schema.String, + }, +) { + override get message(): string { + return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`; + } +} + +export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedError()( + "WorkspaceSearchIndexSearchFailed", + { + cwd: Schema.String, + queryLength: Schema.Number, + pageSize: Schema.Number, + reason: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return `Workspace search failed for '${this.cwd}'.`; + } +} + +export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedError()( + "WorkspaceSearchIndexRefreshFailed", + { + cwd: Schema.String, + reason: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return `Failed to refresh the workspace search index for '${this.cwd}'.`; + } +} + +export class WorkspaceSearchIndexDestroyFailed extends Schema.TaggedError()( + "WorkspaceSearchIndexDestroyFailed", + { + cwd: Schema.String, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return `Failed to destroy the workspace search index for '${this.cwd}'.`; + } +} + +export type WorkspaceSearchIndexError = + | WorkspaceSearchIndexCreateFailed + | WorkspaceSearchIndexScanTimedOut + | WorkspaceSearchIndexSearchFailed + | WorkspaceSearchIndexRefreshFailed; + +export class WorkspaceSearchIndex extends Context.Service< + WorkspaceSearchIndex, + { + readonly list: () => Effect.Effect; + readonly search: ( + query: string, + limit: number, + kind?: ProjectEntryKind, + imageOnly?: boolean, + ) => Effect.Effect; + readonly searchContents: ( + input: Omit, + ) => Effect.Effect; + readonly refresh: () => Effect.Effect< + void, + WorkspaceSearchIndexRefreshFailed | WorkspaceSearchIndexScanTimedOut + >; + } +>()("t3/workspace/WorkspaceSearchIndexService/WorkspaceSearchIndex") {} + +export const WORKSPACE_SEARCH_INDEX_VARIANTS = ["paths", "content"] as const; +export type WorkspaceSearchIndexVariant = (typeof WORKSPACE_SEARCH_INDEX_VARIANTS)[number]; diff --git a/apps/server/src/workspace/WorkspaceSearchProcess.ts b/apps/server/src/workspace/WorkspaceSearchProcess.ts new file mode 100644 index 0000000000000..d8192856d075d --- /dev/null +++ b/apps/server/src/workspace/WorkspaceSearchProcess.ts @@ -0,0 +1,120 @@ +// @effect-diagnostics nodeBuiltinImport:off +// Node IPC belongs at this boundary; the index and its callers remain Effects. +import * as NodeChildProcess from "node:child_process"; +import * as NodeURL from "node:url"; +import { HostProcessEnvironment, HostProcessExecutablePath } from "@t3tools/shared/hostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Schema from "effect/Schema"; +import { SearchResponse, type SearchRequest } from "./workspaceSearchProtocol.ts"; + +const decodeSearchResponse = Schema.decodeUnknownEffect(SearchResponse); + +export const WorkspaceSearchWorkerPath = Context.Reference("t3/workspace/SearchWorkerPath", { + defaultValue: () => + new URL( + import.meta.url.endsWith(".ts") + ? "../workspaceSearchWorker.ts" + : "./workspaceSearchWorker.mjs", + import.meta.url, + ), +}); + +export class WorkspaceSearchProcessFailed extends Schema.TaggedError()( + "WorkspaceSearchProcessFailed", + { cause: Schema.Defect() }, +) {} + +export const startSearchProcess = Effect.fn("WorkspaceSearchProcess.start")(function* () { + const workerPath = yield* WorkspaceSearchWorkerPath; + const environment = yield* HostProcessEnvironment; + const executable = yield* HostProcessExecutablePath; + const child = yield* Effect.try(() => + NodeChildProcess.spawn(executable, [NodeURL.fileURLToPath(workerPath)], { + env: { ...environment, ELECTRON_RUN_AS_NODE: "1" }, + stdio: ["ignore", "ignore", "pipe", "ipc"], + windowsHide: true, + }), + ); + let failure: Error | undefined; + let exited = false; + let stderr = ""; + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => { + stderr = (stderr + chunk).slice(-8_192); + }); + const exitFailure = () => + new WorkspaceSearchProcessFailed({ + cause: + failure ?? + new Error(`Workspace search process exited${stderr ? `: ${stderr.trim()}` : "."}`), + }); + child.on("error", (error) => { + failure = error; + }); + child.once("exit", () => { + exited = true; + }); + + // These indexes are disposable and mmap persistence is disabled. Do not ask + // the native destructor to run: it can wait on the same lock as search. + const stop = Effect.callback((resume) => { + if (exited || child.pid === undefined) { + resume(Effect.void); + return; + } + const onExit = () => resume(Effect.void); + child.once("exit", onExit); + child.kill("SIGKILL"); + return Effect.sync(() => child.removeListener("exit", onExit)); + }).pipe( + // SIGKILL normally reaps immediately. A process stuck in kernel I/O must + // not keep server shutdown waiting indefinitely either. + Effect.timeout("2 seconds"), + Effect.catchTag("TimeoutError", () => + Effect.sync(() => { + child.unref(); + child.channel?.unref(); + child.stderr?.destroy(); + }), + ), + ); + + const request = Effect.fn("WorkspaceSearchProcess.request")(function* (input: SearchRequest) { + const response = yield* Effect.callback((resume) => { + if (failure || exited || !child.connected) { + resume(Effect.fail(exitFailure())); + return; + } + const onMessage = (message: unknown) => { + cleanup(); + resume(Effect.succeed(message)); + }; + const onError = (error: Error) => { + cleanup(); + resume(Effect.fail(new WorkspaceSearchProcessFailed({ cause: error }))); + }; + const onExit = () => { + cleanup(); + resume(Effect.fail(exitFailure())); + }; + const cleanup = () => { + child.removeListener("message", onMessage); + child.removeListener("error", onError); + child.removeListener("exit", onExit); + }; + child.once("message", onMessage); + child.once("error", onError); + child.once("exit", onExit); + child.send(input, (error) => { + if (error) onError(error); + }); + return Effect.sync(cleanup); + }).pipe(Effect.timeout("20 seconds")); + const exit = yield* decodeSearchResponse(response); + return yield* Exit.isFailure(exit) ? Effect.failCause(exit.cause) : Effect.succeed(exit.value); + }); + return { request, stop }; +}); +export type SearchProcess = Effect.Success>; diff --git a/apps/server/src/workspace/workspaceSearchProtocol.ts b/apps/server/src/workspace/workspaceSearchProtocol.ts new file mode 100644 index 0000000000000..cc8941a2bfdaf --- /dev/null +++ b/apps/server/src/workspace/workspaceSearchProtocol.ts @@ -0,0 +1,44 @@ +import { ProjectSearchContentsInput, ProjectSearchEntriesInput } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import * as Struct from "effect/Struct"; +import { + WorkspaceSearchIndexCreateFailed, + WorkspaceSearchIndexRefreshFailed, + WorkspaceSearchIndexScanTimedOut, + WorkspaceSearchIndexSearchFailed, +} from "./WorkspaceSearchIndexService.ts"; + +export const SearchOperation = Schema.Union([ + Schema.Struct({ + method: Schema.Literal("initialize"), + cwd: Schema.String, + variant: Schema.Literals(["paths", "content"]), + }), + Schema.Struct({ method: Schema.Literal("list") }), + Schema.Struct({ method: Schema.Literal("refresh") }), + Schema.Struct({ method: Schema.Literal("dispose") }), + Schema.Struct({ + method: Schema.Literal("search"), + ...Struct.omit(ProjectSearchEntriesInput.fields, ["cwd"]), + }), + Schema.Struct({ + method: Schema.Literal("searchContents"), + input: Schema.Struct(Struct.omit(ProjectSearchContentsInput.fields, ["cwd"])), + }), +]); +export type SearchOperation = typeof SearchOperation.Type; +export const SearchRequest = Schema.Struct({ id: Schema.Int, operation: SearchOperation }); +export type SearchRequest = typeof SearchRequest.Type; + +export const SearchResponse = Schema.toCodecJson( + Schema.Exit( + Schema.Unknown, + Schema.Union([ + WorkspaceSearchIndexCreateFailed, + WorkspaceSearchIndexScanTimedOut, + WorkspaceSearchIndexSearchFailed, + WorkspaceSearchIndexRefreshFailed, + ]), + Schema.Defect(), + ), +); diff --git a/apps/server/src/workspaceSearchWorker.ts b/apps/server/src/workspaceSearchWorker.ts new file mode 100644 index 0000000000000..b6b441d83a38f --- /dev/null +++ b/apps/server/src/workspaceSearchWorker.ts @@ -0,0 +1,56 @@ +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as NativeIndex from "./workspace/NativeWorkspaceSearchIndex.ts"; +import { SearchRequest, SearchResponse } from "./workspace/workspaceSearchProtocol.ts"; + +const decodeSearchRequest = Schema.decodeUnknownEffect(SearchRequest); +const encodeSearchResponse = Schema.encodeSync(SearchResponse); + +// All indexes share this process. Native locks can stall workspace searches, +// but never the server's WebSocket, HTTP, or provider event loops. +const indexes = new Map< + number, + { scope: Scope.Closeable; index: NativeIndex.WorkspaceSearchIndex["Service"] } +>(); +const execute = Effect.fn("WorkspaceSearchWorker.execute")(function* (request: SearchRequest) { + const { id, operation: input } = request; + if (input.method === "initialize") { + const scope = Scope.makeUnsafe(); + const index = yield* NativeIndex.make(input.cwd, input.variant).pipe(Scope.provide(scope)); + indexes.set(id, { scope, index }); + return null; + } + const entry = indexes.get(id); + if (!entry) return yield* Effect.die(new Error("Workspace index has not initialized.")); + switch (input.method) { + case "list": + return yield* entry.index.list(); + case "search": + return yield* entry.index.search(input.query, input.limit, input.kind, input.imageOnly); + case "searchContents": + return yield* entry.index.searchContents(input.input); + case "refresh": + yield* entry.index.refresh(); + return null; + case "dispose": + indexes.delete(id); + yield* Scope.close(entry.scope, Exit.void); + return null; + } +}); + +process.on("disconnect", () => process.exit(0)); +process.on("message", (message) => { + // The parent serializes requests, so a response belongs to the sole caller. + void Effect.runPromiseExit( + decodeSearchRequest(message).pipe(Effect.orDie, Effect.flatMap(execute)), + ).then((exit) => { + try { + process.send?.(encodeSearchResponse(exit)); + } catch (cause) { + process.send?.(encodeSearchResponse(Exit.die(cause))); + } + }); +}); diff --git a/apps/server/vite.config.ts b/apps/server/vite.config.ts index 621a1f7bf66fb..2dc3f4407a451 100644 --- a/apps/server/vite.config.ts +++ b/apps/server/vite.config.ts @@ -36,7 +36,7 @@ export default mergeConfig( }, }, pack: { - entry: ["src/bin.ts", "src/claudeHistoryWorker.ts"], + entry: ["src/bin.ts", "src/claudeHistoryWorker.ts", "src/workspaceSearchWorker.ts"], outDir: "dist", sourcemap: true, clean: true, From 1db36c6d8539451057cdfeacb52134c639190826 Mon Sep 17 00:00:00 2001 From: yashranaway Date: Sat, 12 Sep 2026 05:49:58 +0000 Subject: [PATCH 2/9] fix(server): preserve active searches during index cleanup --- apps/server/scripts/workspace-search-mock.ts | 19 ++++++++++ .../src/workspace/WorkspaceSearchHost.ts | 38 ++++++++++++------- .../workspace/WorkspaceSearchIndex.test.ts | 30 +++++++++++++++ knip.jsonc | 1 + 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/apps/server/scripts/workspace-search-mock.ts b/apps/server/scripts/workspace-search-mock.ts index e3727fbdc6451..4b19c813b1888 100644 --- a/apps/server/scripts/workspace-search-mock.ts +++ b/apps/server/scripts/workspace-search-mock.ts @@ -42,6 +42,25 @@ process.on("message", (message) => { return; } if (input.method === "search" && input.query === "crash") process.exit(1); + if (input.method === "search" && input.query === "hold") { + const socket = NodeNet.connect( + Number(process.env.T3_SEARCH_TEST_RECEIPT_PORT), + "127.0.0.1", + () => socket.write("blocked"), + ); + socket.once("data", () => { + socket.end(); + process.send?.( + encodeSearchResponse( + Exit.succeed({ + entries: [{ path: String(process.pid), kind: "file" }], + truncated: false, + }), + ), + ); + }); + return; + } if (input.method === "refresh") { process.send?.( encodeSearchResponse( diff --git a/apps/server/src/workspace/WorkspaceSearchHost.ts b/apps/server/src/workspace/WorkspaceSearchHost.ts index 4a9de2bbb2f43..eca1881193ef1 100644 --- a/apps/server/src/workspace/WorkspaceSearchHost.ts +++ b/apps/server/src/workspace/WorkspaceSearchHost.ts @@ -13,6 +13,7 @@ import type { SearchOperation } from "./workspaceSearchProtocol.ts"; const make = Effect.gen(function* () { const semaphore = yield* Semaphore.make(1); let current: { process: SearchProcess; indexes: Set } | undefined; + let activeIndex: number | undefined; let nextId = 0; let closed = false; const stop = Effect.fn("WorkspaceSearchHost.stop")(function* () { @@ -41,6 +42,7 @@ const make = Effect.gen(function* () { return yield* new WorkspaceSearchProcessFailed({ cause: new Error("Workspace search index is closed."), }); + activeIndex = id; return yield* Effect.gen(function* () { const active = yield* Effect.gen(function* () { if (current) return current; @@ -55,7 +57,14 @@ const make = Effect.gen(function* () { return operation.method === "initialize" ? null : yield* active.process.request({ id, operation }); - }).pipe(Effect.onError(stop)); + }).pipe( + Effect.onError(stop), + Effect.ensuring( + Effect.sync(() => { + activeIndex = undefined; + }), + ), + ); }, semaphore.withPermits(1), Effect.timeout("20 seconds"), @@ -64,19 +73,20 @@ const make = Effect.gen(function* () { yield* Effect.addFinalizer(() => Effect.gen(function* () { released = true; - // Disposal must not wait behind a stuck search or native destructor. - // Other handles rebuild lazily if this retires the shared process. - yield* Effect.acquireUseRelease( - semaphore.take(1).pipe(Effect.timeout("1 second"), Effect.onError(stop)), - () => - Effect.gen(function* () { - if (!current?.indexes.has(id)) return; - current.indexes.delete(id); - if (current.indexes.size === 0) return yield* stop(); - yield* current.process.request({ id, operation: { method: "dispose" } }); - }).pipe(Effect.timeout("1 second"), Effect.onError(stop)), - () => semaphore.release(1), - ).pipe(Effect.catchCause(() => Effect.void)); + if (activeIndex === id) yield* stop(); + // Give unrelated requests their own deadline. The shorter disposal + // deadline bounds the native destructor, not another workspace's scan. + yield* Effect.gen(function* () { + if (!current?.indexes.has(id)) return; + current.indexes.delete(id); + if (current.indexes.size === 0) return yield* stop(); + yield* current.process.request({ id, operation: { method: "dispose" } }); + }).pipe( + Effect.timeout("1 second"), + Effect.onError(stop), + semaphore.withPermits(1), + Effect.catchCause(() => Effect.void), + ); }), ); yield* run(initialize); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 8b3fb682e795a..ce144a8e9ea6c 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -25,7 +25,11 @@ const withWorker = (effect: Effect.Effect) => const receipts = Effect.fn(function* () { const blocked = yield* Deferred.make(); const exited = yield* Deferred.make(); + const connection = yield* Deferred.make(); const server = NodeNet.createServer((socket) => { + // The fixture can be killed while its receipt socket has a write in flight. + socket.on("error", () => {}); + Deferred.doneUnsafe(connection, Effect.succeed(socket)); socket.once("data", () => Deferred.doneUnsafe(blocked, Effect.void)); socket.once("close", () => Deferred.doneUnsafe(exited, Effect.void)); }); @@ -38,6 +42,9 @@ const receipts = Effect.fn(function* () { return { blocked, exited, + release: Deferred.await(connection).pipe( + Effect.flatMap((socket) => Effect.sync(() => socket.write("release"))), + ), environment: { ...process.env, T3_SEARCH_TEST_RECEIPT_PORT: String(address.port) }, }; }); @@ -225,3 +232,26 @@ it.effect("cancels a queued request without terminating the active search proces }).pipe(withWorker), ), ); + +it.effect("lets an active workspace finish while an unrelated index expires", () => + Effect.scoped( + Effect.gen(function* () { + const control = yield* receipts(); + yield* Effect.gen(function* () { + const scope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); + yield* Index.make("retiring").pipe(Scope.provide(scope)); + const surviving = yield* Index.make("surviving"); + const before = yield* surviving.list(); + const active = yield* surviving.search("hold", 1).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(control.blocked); + const closing = yield* Scope.close(scope, Exit.void).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* control.release; + expect(yield* Fiber.join(active)).toEqual(Exit.succeed(before)); + yield* Fiber.join(closing); + expect((yield* surviving.list()).entries).toEqual(before.entries); + }).pipe(Effect.provideService(HostProcessEnvironment, control.environment)); + }).pipe(withWorker), + ), +); diff --git a/knip.jsonc b/knip.jsonc index 9b8712a078723..7ab7ddbf21cb3 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -22,6 +22,7 @@ "src/bin.ts!", "src/service-launcher.ts!", "src/claudeHistoryWorker.ts!", + "src/workspaceSearchWorker.ts!", "scripts/cli.ts", "src/provider/testFixtures/*.mjs", ], From 394f707122e9c03fc00a6ca14c1646b9d38f84a7 Mon Sep 17 00:00:00 2001 From: yashranaway Date: Sat, 12 Sep 2026 06:07:19 +0000 Subject: [PATCH 3/9] fix(server): retain search workers after recoverable errors --- apps/server/scripts/workspace-search-mock.ts | 16 ++++++++++ .../workspace/NativeWorkspaceSearchIndex.ts | 4 +-- .../src/workspace/WorkspaceSearchHost.ts | 21 +++++++++++-- .../workspace/WorkspaceSearchIndex.test.ts | 31 ++++++++++++++++--- .../src/workspace/WorkspaceSearchIndex.ts | 13 ++++++-- .../workspace/WorkspaceSearchIndexService.ts | 3 ++ 6 files changed, 77 insertions(+), 11 deletions(-) diff --git a/apps/server/scripts/workspace-search-mock.ts b/apps/server/scripts/workspace-search-mock.ts index 4b19c813b1888..c71f361dc62d1 100644 --- a/apps/server/scripts/workspace-search-mock.ts +++ b/apps/server/scripts/workspace-search-mock.ts @@ -6,6 +6,7 @@ import { SearchRequest, SearchResponse } from "../src/workspace/workspaceSearchP import { WorkspaceSearchIndexRefreshFailed, WorkspaceSearchIndexScanTimedOut, + WorkspaceSearchIndexSearchFailed, } from "../src/workspace/WorkspaceSearchIndexService.ts"; const decodeSearchRequest = Schema.decodeUnknownSync(SearchRequest); @@ -42,6 +43,21 @@ process.on("message", (message) => { return; } if (input.method === "search" && input.query === "crash") process.exit(1); + if (input.method === "search" && input.query === "search-error") { + process.send?.( + encodeSearchResponse( + Exit.fail( + new WorkspaceSearchIndexSearchFailed({ + cwd, + queryLength: input.query.length, + pageSize: input.limit + 1, + reason: "search rejected", + }), + ), + ), + ); + return; + } if (input.method === "search" && input.query === "hold") { const socket = NodeNet.connect( Number(process.env.T3_SEARCH_TEST_RECEIPT_PORT), diff --git a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts index e6e51eaa4c5f3..2281798c2419f 100644 --- a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts @@ -21,6 +21,8 @@ import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; import { WorkspaceSearchIndex, + WORKSPACE_INDEX_MAX_ENTRIES, + WORKSPACE_INDEX_PAGE_SIZE, WorkspaceSearchIndexCreateFailed, WorkspaceSearchIndexDestroyFailed, WorkspaceSearchIndexRefreshFailed, @@ -31,8 +33,6 @@ import { export * from "./WorkspaceSearchIndexService.ts"; -const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; -const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000; const CONTENT_SEARCH_TIME_BUDGET_MS = 250; diff --git a/apps/server/src/workspace/WorkspaceSearchHost.ts b/apps/server/src/workspace/WorkspaceSearchHost.ts index eca1881193ef1..73b89213e1866 100644 --- a/apps/server/src/workspace/WorkspaceSearchHost.ts +++ b/apps/server/src/workspace/WorkspaceSearchHost.ts @@ -1,15 +1,25 @@ import * as Context from "effect/Context"; +import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import { startSearchProcess, WorkspaceSearchProcessFailed, type SearchProcess, } from "./WorkspaceSearchProcess.ts"; -import type { WorkspaceSearchIndexVariant } from "./WorkspaceSearchIndexService.ts"; +import { + WorkspaceSearchIndexSearchFailed, + WorkspaceSearchIndexRefreshFailed, + type WorkspaceSearchIndexVariant, +} from "./WorkspaceSearchIndexService.ts"; import type { SearchOperation } from "./workspaceSearchProtocol.ts"; +const isRecoverable = Schema.is( + Schema.Union([WorkspaceSearchIndexSearchFailed, WorkspaceSearchIndexRefreshFailed]), +); + const make = Effect.gen(function* () { const semaphore = yield* Semaphore.make(1); let current: { process: SearchProcess; indexes: Set } | undefined; @@ -58,7 +68,14 @@ const make = Effect.gen(function* () { ? null : yield* active.process.request({ id, operation }); }).pipe( - Effect.onError(stop), + Effect.onError((cause) => + cause.reasons.length > 0 && + cause.reasons.every( + (reason) => Cause.isFailReason(reason) && isRecoverable(reason.error), + ) + ? Effect.void + : stop(), + ), Effect.ensuring( Effect.sync(() => { activeIndex = undefined; diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index ce144a8e9ea6c..c33819f7b680f 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -220,14 +220,14 @@ it.effect("cancels a queued request without terminating the active search proces yield* Effect.gen(function* () { const index = yield* Index.make("workspace"); const other = yield* Index.make("other"); - const blocked = yield* index.search("block", 1).pipe(Effect.forkChild); + const before = yield* index.list(); + const active = yield* index.search("hold", 1).pipe(Effect.forkChild); yield* Deferred.await(control.blocked); const queued = yield* other.list().pipe(Effect.forkChild); yield* Fiber.interrupt(queued); - expect(yield* Deferred.isDone(control.exited)).toBe(false); - yield* Fiber.interrupt(blocked); - yield* Deferred.await(control.exited); - expect((yield* other.list()).entries).toHaveLength(1); + yield* control.release; + expect(yield* Fiber.join(active)).toEqual(before); + expect((yield* other.list()).entries).toEqual(before.entries); }).pipe(Effect.provideService(HostProcessEnvironment, control.environment)); }).pipe(withWorker), ), @@ -255,3 +255,24 @@ it.effect("lets an active workspace finish while an unrelated index expires", () }).pipe(withWorker), ), ); + +it.effect("preserves the shared process after ordinary search and refresh errors", () => + Effect.scoped( + Effect.gen(function* () { + const index = yield* Index.make("workspace"); + const other = yield* Index.make("other"); + const before = yield* other.list(); + expect(yield* Effect.flip(index.search("search-error", 1))).toMatchObject({ + _tag: "WorkspaceSearchIndexSearchFailed", + reason: "search rejected", + }); + expect((yield* other.list()).entries).toEqual(before.entries); + expect(yield* Effect.flip(index.refresh())).toMatchObject({ + _tag: "WorkspaceSearchIndexRefreshFailed", + reason: "scan failed", + }); + expect((yield* other.list()).entries).toEqual(before.entries); + expect((yield* index.list()).entries).toEqual(before.entries); + }).pipe(withWorker), + ), +); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 993cec0cef99b..944af059aba7f 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -9,6 +9,7 @@ import { } from "@t3tools/contracts"; import { WorkspaceSearchIndex, + WORKSPACE_INDEX_PAGE_SIZE, WorkspaceSearchIndexCreateFailed, WorkspaceSearchIndexRefreshFailed, WorkspaceSearchIndexScanTimedOut, @@ -59,13 +60,21 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( list: () => remote .request({ method: "list" }) - .pipe(Effect.flatMap(decodeList), Effect.mapError(searchFailure(0, 25_002))), + .pipe( + Effect.flatMap(decodeList), + Effect.mapError(searchFailure(0, WORKSPACE_INDEX_PAGE_SIZE)), + ), search: (query, limit, kind, imageOnly) => remote .request({ method: "search", query, limit, kind, imageOnly }) .pipe( Effect.flatMap(decodeSearch), - Effect.mapError(searchFailure(query.length, limit + 1)), + Effect.mapError( + searchFailure( + query.length, + imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1), + ), + ), ), searchContents: (input) => remote diff --git a/apps/server/src/workspace/WorkspaceSearchIndexService.ts b/apps/server/src/workspace/WorkspaceSearchIndexService.ts index b75015cdff624..41b75e64f2255 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndexService.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndexService.ts @@ -9,6 +9,9 @@ import type { ProjectSearchEntriesResult, } from "@t3tools/contracts"; +export const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; +export const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; + export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError()( "WorkspaceSearchIndexCreateFailed", { From c5b9ac4ba44cd19f3a3ca2b655d863204c89f1b4 Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:13:44 +0000 Subject: [PATCH 4/9] fix(server): launch search workers on supported Node runtimes --- apps/server/scripts/workspace-search-mock.ts | 1 + .../src/workspace/WorkspaceSearchHost.ts | 24 ++++++++---- .../workspace/WorkspaceSearchIndex.test.ts | 2 +- .../src/workspace/WorkspaceSearchIndex.ts | 4 +- .../src/workspace/WorkspaceSearchProcess.ts | 38 +++++++++++-------- 5 files changed, 44 insertions(+), 25 deletions(-) diff --git a/apps/server/scripts/workspace-search-mock.ts b/apps/server/scripts/workspace-search-mock.ts index c71f361dc62d1..c2e6cd3f55da8 100644 --- a/apps/server/scripts/workspace-search-mock.ts +++ b/apps/server/scripts/workspace-search-mock.ts @@ -1,4 +1,5 @@ // @effect-diagnostics nodeBuiltinImport:off +// Native IPC tests use Node sockets to signal blocked and retired workers. import * as NodeNet from "node:net"; import * as Exit from "effect/Exit"; import * as Schema from "effect/Schema"; diff --git a/apps/server/src/workspace/WorkspaceSearchHost.ts b/apps/server/src/workspace/WorkspaceSearchHost.ts index 73b89213e1866..9af1d3cf5e7c2 100644 --- a/apps/server/src/workspace/WorkspaceSearchHost.ts +++ b/apps/server/src/workspace/WorkspaceSearchHost.ts @@ -4,6 +4,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; +import type * as Scope from "effect/Scope"; import { startSearchProcess, WorkspaceSearchProcessFailed, @@ -20,7 +21,21 @@ const isRecoverable = Schema.is( Schema.Union([WorkspaceSearchIndexSearchFailed, WorkspaceSearchIndexRefreshFailed]), ); -const make = Effect.gen(function* () { +export class WorkspaceSearchHost extends Context.Service< + WorkspaceSearchHost, + { + readonly open: ( + cwd: string, + variant: WorkspaceSearchIndexVariant, + ) => Effect.Effect< + { readonly request: (operation: SearchOperation) => ReturnType }, + Effect.Error>, + Scope.Scope + >; + } +>()("t3/workspace/WorkspaceSearchHost") {} + +export const make = Effect.gen(function* () { const semaphore = yield* Semaphore.make(1); let current: { process: SearchProcess; indexes: Set } | undefined; let activeIndex: number | undefined; @@ -112,9 +127,4 @@ const make = Effect.gen(function* () { return { open }; }); -export class WorkspaceSearchHost extends Context.Service< - WorkspaceSearchHost, - Effect.Success ->()("t3/workspace/WorkspaceSearchHost") { - static readonly layer = Layer.effect(WorkspaceSearchHost, make); -} +export const layer = Layer.effect(WorkspaceSearchHost, make); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index c33819f7b680f..7b66315da8917 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -13,7 +13,7 @@ import * as TestClock from "effect/testing/TestClock"; import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; import * as Index from "./WorkspaceSearchIndex.ts"; import { WorkspaceSearchWorkerPath } from "./WorkspaceSearchProcess.ts"; -import { WorkspaceSearchHost } from "./WorkspaceSearchHost.ts"; +import * as WorkspaceSearchHost from "./WorkspaceSearchHost.ts"; const mockWorker = new URL("../../scripts/workspace-search-mock.ts", import.meta.url); const withWorker = (effect: Effect.Effect) => diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 944af059aba7f..7fe4cde6384be 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -16,7 +16,7 @@ import { WorkspaceSearchIndexSearchFailed, type WorkspaceSearchIndexVariant, } from "./WorkspaceSearchIndexService.ts"; -import { WorkspaceSearchHost } from "./WorkspaceSearchHost.ts"; +import * as WorkspaceSearchHost from "./WorkspaceSearchHost.ts"; export * from "./WorkspaceSearchIndexService.ts"; const WORKSPACE_INDEX_IDLE_TTL = "15 minutes"; @@ -32,7 +32,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cwd: string, variant: WorkspaceSearchIndexVariant = "paths", ) { - const host = yield* WorkspaceSearchHost; + const host = yield* WorkspaceSearchHost.WorkspaceSearchHost; const remote = yield* host.open(cwd, variant).pipe( Effect.mapError((cause) => isCreateFailed(cause) || isScanTimedOut(cause) diff --git a/apps/server/src/workspace/WorkspaceSearchProcess.ts b/apps/server/src/workspace/WorkspaceSearchProcess.ts index d16858b6b0d82..4f442feedb283 100644 --- a/apps/server/src/workspace/WorkspaceSearchProcess.ts +++ b/apps/server/src/workspace/WorkspaceSearchProcess.ts @@ -35,14 +35,21 @@ export const startSearchProcess = Effect.fn("WorkspaceSearchProcess.start")(func const environment = yield* HostProcessEnvironment; const executable = yield* HostProcessExecutablePath; const isExecutable = yield* HostProcessIsExecutable; - const args = isExecutable ? ["__workspace-search"] : [NodeURL.fileURLToPath(workerPath)]; - const child = yield* Effect.try(() => - NodeChildProcess.spawn(executable, args, { - env: { ...environment, ELECTRON_RUN_AS_NODE: "1" }, - stdio: ["ignore", "ignore", "pipe", "ipc"], - windowsHide: true, - }), - ); + const args = isExecutable + ? ["__workspace-search"] + : [ + ...(workerPath.pathname.endsWith(".ts") ? ["--experimental-strip-types"] : []), + NodeURL.fileURLToPath(workerPath), + ]; + const child = yield* Effect.try({ + try: () => + NodeChildProcess.spawn(executable, args, { + env: { ...environment, ELECTRON_RUN_AS_NODE: "1" }, + stdio: ["ignore", "ignore", "pipe", "ipc"], + windowsHide: true, + }), + catch: (cause) => new WorkspaceSearchProcessFailed({ cause }), + }); let failure: Error | undefined; let exited = false; let stderr = ""; @@ -78,13 +85,14 @@ export const startSearchProcess = Effect.fn("WorkspaceSearchProcess.start")(func // SIGKILL normally reaps immediately. A process stuck in kernel I/O must // not keep server shutdown waiting indefinitely either. Effect.timeout("2 seconds"), - Effect.catchTag("TimeoutError", () => - Effect.sync(() => { - child.unref(); - child.channel?.unref(); - child.stderr?.destroy(); - }), - ), + Effect.catchTags({ + TimeoutError: () => + Effect.sync(() => { + child.unref(); + child.channel?.unref(); + child.stderr?.destroy(); + }), + }), ); const request = Effect.fn("WorkspaceSearchProcess.request")(function* (input: SearchRequest) { From bad24566f595cf4c8e6a0140f07b720b0225e772 Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:17:43 +0000 Subject: [PATCH 5/9] fix(server): retain public workspace search construction API --- apps/server/src/workspace/WorkspaceSearchHost.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/workspace/WorkspaceSearchHost.ts b/apps/server/src/workspace/WorkspaceSearchHost.ts index 9af1d3cf5e7c2..1539faf3caa83 100644 --- a/apps/server/src/workspace/WorkspaceSearchHost.ts +++ b/apps/server/src/workspace/WorkspaceSearchHost.ts @@ -35,6 +35,7 @@ export class WorkspaceSearchHost extends Context.Service< } >()("t3/workspace/WorkspaceSearchHost") {} +/** @public Service construction is part of the canonical Effect module API. */ export const make = Effect.gen(function* () { const semaphore = yield* Semaphore.make(1); let current: { process: SearchProcess; indexes: Set } | undefined; From 75adfc4a309a0d74e2bcbed83aed1df34faa261b Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:27:27 +0000 Subject: [PATCH 6/9] fix(server): retry bounded candidates for whole-word searches --- .../NativeWorkspaceSearchIndex.test.ts | 21 +++++++ .../workspace/NativeWorkspaceSearchIndex.ts | 43 +++++++++++--- .../src/workspace/WorkspaceEntries.test.ts | 58 ++++++++++++++++++- 3 files changed, 112 insertions(+), 10 deletions(-) diff --git a/apps/server/src/workspace/NativeWorkspaceSearchIndex.test.ts b/apps/server/src/workspace/NativeWorkspaceSearchIndex.test.ts index f3b6f2a24b22e..0768333c906a5 100644 --- a/apps/server/src/workspace/NativeWorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/NativeWorkspaceSearchIndex.test.ts @@ -61,6 +61,27 @@ it.effect("filters image searches before applying the result limit", () => expect(resultWithDirectoryKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); expect(fileSearch).toHaveBeenCalledTimes(2); expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 }); + fileSearch.mockReturnValueOnce({ + ok: true, + value: { items: [fileItem("src/source.ts")], scores: [], totalMatched: 2, totalFiles: 2 }, + }); + expect(yield* searchIndex.search("", 1, "file", true)).toEqual({ + entries: [], + truncated: true, + }); + fileSearch.mockReturnValueOnce({ + ok: true, + value: { + items: [fileItem("public/one.png"), fileItem("public/two.png")], + scores: [], + totalMatched: 2, + totalFiles: 2, + }, + }); + expect(yield* searchIndex.search("", 1, "file", true)).toEqual({ + entries: [{ kind: "file", path: "public/one.png" }], + truncated: true, + }); }), ), ); diff --git a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts index 1ee4de3ebf4d9..729fb5afa8994 100644 --- a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts @@ -43,6 +43,7 @@ const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000; const CONTENT_SEARCH_TIME_BUDGET_MS = 250; const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100; +const CONTENT_SEARCH_MAX_CANDIDATES = 25_000; function toPosixPath(input: string): string { return input.replaceAll("\\", "/"); @@ -397,46 +398,72 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; // Grep cursors advance by file, so whole-word post-filtering needs enough // raw candidates from the current file before moving to the next one. - const rawPageSize = input.wholeWord + let rawPageSize = input.wholeWord ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE) : input.limit; const matches: Array = []; let nextCursor: GrepCursor | null = null; let regexFallbackError: string | undefined; + let candidateLimitReached = false; - do { + while (true) { const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); const result = yield* runSearch(input.query, input.limit, "grep", () => finder.grep(searchQuery, { mode: regexMode ? "regex" : "plain", smartCase: !input.caseSensitive && !regexMode, - // A single dense file must not consume the whole result page. - maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), + // Whole-word filtering needs the full candidate page from a dense file. + maxMatchesPerFile: input.wholeWord + ? rawPageSize + : Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), pageSize: rawPageSize, cursor: nextCursor, timeBudgetMs: remainingTimeBudgetMs, }), ); + regexFallbackError ??= result.regexFallbackError; + const pageMatches: Array = []; for (const match of result.items) { const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), ); if (matchRanges.length === 0) continue; - matches.push({ + pageMatches.push({ path: toPosixPath(match.relativePath), lineNumber: match.lineNumber, lineContent: match.lineContent, matchRanges, }); } + // Cursors advance by file. Retry a full raw page before skipping a file + // whose later lines may contain the first whole-word match. + if (input.wholeWord && result.items.length >= rawPageSize) { + if ( + matches.length + pageMatches.length < input.limit && + rawPageSize < CONTENT_SEARCH_MAX_CANDIDATES && + performance.now() < deadline + ) { + rawPageSize = Math.min(CONTENT_SEARCH_MAX_CANDIDATES, rawPageSize * 2); + continue; + } + candidateLimitReached = true; + } + matches.push(...pageMatches); nextCursor = result.nextCursor; - regexFallbackError ??= result.regexFallbackError; - } while (matches.length < input.limit && nextCursor !== null && performance.now() < deadline); + if ( + candidateLimitReached || + matches.length >= input.limit || + nextCursor === null || + performance.now() >= deadline + ) { + break; + } + } return { matches: matches.slice(0, input.limit), - truncated: matches.length > input.limit || nextCursor !== null, + truncated: candidateLimitReached || matches.length > input.limit || nextCursor !== null, ...(regexFallbackError !== undefined ? { regexFallbackError } : {}), }; }); diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index a3f9f008aa86e..6067ca8af9e34 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -542,7 +542,7 @@ it.layer(TestLayer, { excludeTestServices: true })("WorkspaceEntries", (it) => { it.effect("finds later whole-word matches in a file after rejected raw matches", () => Effect.gen(function* () { const cwd = yield* makeTempDir({ prefix: "t3code-workspace-content-late-whole-word-" }); - yield* writeTextFile(cwd, "src/words.ts", `${"afoo\n".repeat(10)}foo\n`); + yield* writeTextFile(cwd, "src/words.ts", `${"afoo\n".repeat(150)}foo\n`); const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const result = yield* workspaceEntries.searchContents({ @@ -557,13 +557,67 @@ it.layer(TestLayer, { excludeTestServices: true })("WorkspaceEntries", (it) => { expect(result.matches).toEqual([ expect.objectContaining({ path: "src/words.ts", - lineNumber: 11, + lineNumber: 151, matchRanges: [{ start: 0, end: 3 }], }), ]); }), ); + it.effect("retries raw pages without duplicating earlier whole-word matches", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-content-retry-" }); + yield* writeTextFile(cwd, "src/words.ts", `foo\n${"afoo\n".repeat(150)}foo\n`); + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const result = yield* workspaceEntries.searchContents({ + cwd, + query: "foo", + limit: 2, + caseSensitive: true, + wholeWord: true, + useRegex: false, + }); + expect(result.matches.map((match) => match.lineNumber)).toEqual([1, 152]); + expect(result.truncated).toBe(false); + }), + ); + + it.effect("reports an incomplete raw page when the whole-word result limit is reached", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-content-full-page-" }); + yield* writeTextFile(cwd, "src/words.ts", `foo\n${"afoo\n".repeat(150)}foo\n`); + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const result = yield* workspaceEntries.searchContents({ + cwd, + query: "foo", + limit: 1, + caseSensitive: true, + wholeWord: true, + useRegex: false, + }); + expect(result.matches.map((match) => match.lineNumber)).toEqual([1]); + expect(result.truncated).toBe(true); + }), + ); + + it.effect("reports incomplete whole-word results at the candidate bound", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-content-bounded-" }); + yield* writeTextFile(cwd, "src/words.ts", `${"afoo\n".repeat(26_000)}foo\n`); + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const result = yield* workspaceEntries.searchContents({ + cwd, + query: "foo", + limit: 1, + caseSensitive: true, + wholeWord: true, + useRegex: false, + }); + expect(result.matches).toEqual([]); + expect(result.truncated).toBe(true); + }), + ); + it.effect("treats astral-plane letters as whole word characters", () => Effect.gen(function* () { const cwd = yield* makeTempDir({ prefix: "t3code-workspace-content-astral-word-" }); From 182851ba16a15b9f68c635d2c2d7b1b18440346e Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:30:01 +0000 Subject: [PATCH 7/9] refactor(server): consume workspace index service through its namespace --- .../workspace/NativeWorkspaceSearchIndex.ts | 234 +++++++++--------- .../src/workspace/WorkspaceSearchIndex.ts | 49 ++-- 2 files changed, 143 insertions(+), 140 deletions(-) diff --git a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts index 729fb5afa8994..7e54146d86751 100644 --- a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts @@ -21,17 +21,7 @@ import type { } from "@t3tools/contracts"; import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; -import { - WorkspaceSearchIndex, - WORKSPACE_INDEX_MAX_ENTRIES, - WORKSPACE_INDEX_PAGE_SIZE, - WorkspaceSearchIndexCreateFailed, - WorkspaceSearchIndexDestroyFailed, - WorkspaceSearchIndexRefreshFailed, - WorkspaceSearchIndexScanTimedOut, - WorkspaceSearchIndexSearchFailed, - type WorkspaceSearchIndexVariant, -} from "./WorkspaceSearchIndexService.ts"; +import * as WorkspaceSearchIndexService from "./WorkspaceSearchIndexService.ts"; export * from "./WorkspaceSearchIndexService.ts"; @@ -220,7 +210,7 @@ function withDirectoryAncestors(entries: ReadonlyArray): ProjectEn const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* ( cwd: string, - variant: WorkspaceSearchIndexVariant, + variant: WorkspaceSearchIndexService.WorkspaceSearchIndexVariant, ) { const result = yield* Effect.try({ try: () => @@ -236,14 +226,14 @@ const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* ( enableHomeDirScanning: true, }), catch: (cause) => - new WorkspaceSearchIndexCreateFailed({ + new WorkspaceSearchIndexService.WorkspaceSearchIndexCreateFailed({ cwd, reason: "FileFinder.create threw unexpectedly.", cause, }), }); if (result.ok) return result.value; - return yield* new WorkspaceSearchIndexCreateFailed({ + return yield* new WorkspaceSearchIndexService.WorkspaceSearchIndexCreateFailed({ cwd, reason: result.error, }); @@ -253,7 +243,7 @@ const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(fu cwd: string, finder: FileFinderType, onFailure: (input: { readonly reason: string; readonly cause?: unknown }) => E, -): Effect.fn.Return { +): Effect.fn.Return { const result = yield* Effect.tryPromise({ try: () => finder.waitForIndexReady(WORKSPACE_INDEX_SCAN_TIMEOUT_MS), catch: (cause) => @@ -266,7 +256,7 @@ const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(fu return yield* Effect.fail(onFailure({ reason: result.error })); } if (!result.value) { - return yield* new WorkspaceSearchIndexScanTimedOut({ + return yield* new WorkspaceSearchIndexService.WorkspaceSearchIndexScanTimedOut({ cwd, timeout: WORKSPACE_INDEX_SCAN_TIMEOUT, }); @@ -275,19 +265,20 @@ const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(fu export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cwd: string, - variant: WorkspaceSearchIndexVariant = "paths", + variant: WorkspaceSearchIndexService.WorkspaceSearchIndexVariant = "paths", ) { const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) => Effect.try({ try: () => finder.destroy(), - catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }), + catch: (cause) => + new WorkspaceSearchIndexService.WorkspaceSearchIndexDestroyFailed({ cwd, cause }), }).pipe(Effect.orDie), ); yield* waitForIndexReady( cwd, finder, ({ reason, cause }) => - new WorkspaceSearchIndexCreateFailed({ + new WorkspaceSearchIndexService.WorkspaceSearchIndexCreateFailed({ cwd, reason, cause, @@ -299,11 +290,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( pageSize: number, operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch", execute: () => Result, - ): Effect.fn.Return { + ): Effect.fn.Return { const result = yield* Effect.try({ try: execute, catch: (cause) => - new WorkspaceSearchIndexSearchFailed({ + new WorkspaceSearchIndexService.WorkspaceSearchIndexSearchFailed({ cwd, queryLength: query.length, pageSize, @@ -312,7 +303,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( }), }); if (!result.ok) { - return yield* new WorkspaceSearchIndexSearchFailed({ + return yield* new WorkspaceSearchIndexService.WorkspaceSearchIndexSearchFailed({ cwd, queryLength: query.length, pageSize, @@ -322,20 +313,20 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( return result.value; }); - const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( + const refresh: WorkspaceSearchIndexService.WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( "WorkspaceSearchIndex.refresh", )(function* () { const result = yield* Effect.try({ try: () => finder.scanFiles(), catch: (cause) => - new WorkspaceSearchIndexRefreshFailed({ + new WorkspaceSearchIndexService.WorkspaceSearchIndexRefreshFailed({ cwd, reason: "FileFinder.scanFiles threw unexpectedly.", cause, }), }); if (!result.ok) { - return yield* new WorkspaceSearchIndexRefreshFailed({ + return yield* new WorkspaceSearchIndexService.WorkspaceSearchIndexRefreshFailed({ cwd, reason: result.error, }); @@ -344,7 +335,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cwd, finder, ({ reason, cause }) => - new WorkspaceSearchIndexRefreshFailed({ + new WorkspaceSearchIndexService.WorkspaceSearchIndexRefreshFailed({ cwd, reason, cause, @@ -352,27 +343,36 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( ); }); - const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")( - function* () { - const result = yield* runSearch("", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", () => - finder.mixedSearch("", { pageSize: WORKSPACE_INDEX_PAGE_SIZE }), - ); - const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES); - const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => - left.path.localeCompare(right.path), - ); - const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES); - return { - entries, - truncated: mapped.truncated || entries.length < sortedEntries.length, - }; - }, - ); + const list: WorkspaceSearchIndexService.WorkspaceSearchIndex["Service"]["list"] = Effect.fn( + "WorkspaceSearchIndex.list", + )(function* () { + const result = yield* runSearch( + "", + WorkspaceSearchIndexService.WORKSPACE_INDEX_PAGE_SIZE, + "mixedSearch", + () => + finder.mixedSearch("", { pageSize: WorkspaceSearchIndexService.WORKSPACE_INDEX_PAGE_SIZE }), + ); + const mapped = mapMixedSearchResult( + result, + WorkspaceSearchIndexService.WORKSPACE_INDEX_MAX_ENTRIES, + ); + const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => + left.path.localeCompare(right.path), + ); + const entries = sortedEntries.slice(0, WorkspaceSearchIndexService.WORKSPACE_INDEX_MAX_ENTRIES); + return { + entries, + truncated: mapped.truncated || entries.length < sortedEntries.length, + }; + }); - const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( + const search: WorkspaceSearchIndexService.WorkspaceSearchIndex["Service"]["search"] = Effect.fn( "WorkspaceSearchIndex.search", )(function* (query, limit, kind, imageOnly) { - const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); + const pageSize = imageOnly + ? WorkspaceSearchIndexService.WORKSPACE_INDEX_PAGE_SIZE + : Math.max(1, limit + 1); if (kind === "file" || imageOnly) { const result = yield* runSearch(query, pageSize, "fileSearch", () => finder.fileSearch(query, { pageSize }), @@ -391,82 +391,86 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( return mapMixedSearchResult(result, limit); }); - const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn( - "WorkspaceSearchIndex.searchContents", - )(function* (input) { - const { searchQuery, regexMode } = buildContentSearchQuery(input); - const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; - // Grep cursors advance by file, so whole-word post-filtering needs enough - // raw candidates from the current file before moving to the next one. - let rawPageSize = input.wholeWord - ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE) - : input.limit; - const matches: Array = []; - let nextCursor: GrepCursor | null = null; - let regexFallbackError: string | undefined; - let candidateLimitReached = false; - - while (true) { - const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); - const result = yield* runSearch(input.query, input.limit, "grep", () => - finder.grep(searchQuery, { - mode: regexMode ? "regex" : "plain", - smartCase: !input.caseSensitive && !regexMode, - // Whole-word filtering needs the full candidate page from a dense file. - maxMatchesPerFile: input.wholeWord - ? rawPageSize - : Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), - pageSize: rawPageSize, - cursor: nextCursor, - timeBudgetMs: remainingTimeBudgetMs, - }), - ); - - regexFallbackError ??= result.regexFallbackError; - const pageMatches: Array = []; - for (const match of result.items) { - const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( - (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), + const searchContents: WorkspaceSearchIndexService.WorkspaceSearchIndex["Service"]["searchContents"] = + Effect.fn("WorkspaceSearchIndex.searchContents")(function* (input) { + const { searchQuery, regexMode } = buildContentSearchQuery(input); + const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; + // Grep cursors advance by file, so whole-word post-filtering needs enough + // raw candidates from the current file before moving to the next one. + let rawPageSize = input.wholeWord + ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE) + : input.limit; + const matches: Array = []; + let nextCursor: GrepCursor | null = null; + let regexFallbackError: string | undefined; + let candidateLimitReached = false; + + while (true) { + const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); + const result = yield* runSearch(input.query, input.limit, "grep", () => + finder.grep(searchQuery, { + mode: regexMode ? "regex" : "plain", + smartCase: !input.caseSensitive && !regexMode, + // Whole-word filtering needs the full candidate page from a dense file. + maxMatchesPerFile: input.wholeWord + ? rawPageSize + : Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), + pageSize: rawPageSize, + cursor: nextCursor, + timeBudgetMs: remainingTimeBudgetMs, + }), ); - if (matchRanges.length === 0) continue; - pageMatches.push({ - path: toPosixPath(match.relativePath), - lineNumber: match.lineNumber, - lineContent: match.lineContent, - matchRanges, - }); - } - // Cursors advance by file. Retry a full raw page before skipping a file - // whose later lines may contain the first whole-word match. - if (input.wholeWord && result.items.length >= rawPageSize) { + + regexFallbackError ??= result.regexFallbackError; + const pageMatches: Array = []; + for (const match of result.items) { + const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( + (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), + ); + if (matchRanges.length === 0) continue; + pageMatches.push({ + path: toPosixPath(match.relativePath), + lineNumber: match.lineNumber, + lineContent: match.lineContent, + matchRanges, + }); + } + // Cursors advance by file. Retry a full raw page before skipping a file + // whose later lines may contain the first whole-word match. + if (input.wholeWord && result.items.length >= rawPageSize) { + if ( + matches.length + pageMatches.length < input.limit && + rawPageSize < CONTENT_SEARCH_MAX_CANDIDATES && + performance.now() < deadline + ) { + rawPageSize = Math.min(CONTENT_SEARCH_MAX_CANDIDATES, rawPageSize * 2); + continue; + } + candidateLimitReached = true; + } + matches.push(...pageMatches); + nextCursor = result.nextCursor; if ( - matches.length + pageMatches.length < input.limit && - rawPageSize < CONTENT_SEARCH_MAX_CANDIDATES && - performance.now() < deadline + candidateLimitReached || + matches.length >= input.limit || + nextCursor === null || + performance.now() >= deadline ) { - rawPageSize = Math.min(CONTENT_SEARCH_MAX_CANDIDATES, rawPageSize * 2); - continue; + break; } - candidateLimitReached = true; - } - matches.push(...pageMatches); - nextCursor = result.nextCursor; - if ( - candidateLimitReached || - matches.length >= input.limit || - nextCursor === null || - performance.now() >= deadline - ) { - break; } - } - return { - matches: matches.slice(0, input.limit), - truncated: candidateLimitReached || matches.length > input.limit || nextCursor !== null, - ...(regexFallbackError !== undefined ? { regexFallbackError } : {}), - }; - }); + return { + matches: matches.slice(0, input.limit), + truncated: candidateLimitReached || matches.length > input.limit || nextCursor !== null, + ...(regexFallbackError !== undefined ? { regexFallbackError } : {}), + }; + }); - return WorkspaceSearchIndex.of({ list, refresh, search, searchContents }); + return WorkspaceSearchIndexService.WorkspaceSearchIndex.of({ + list, + refresh, + search, + searchContents, + }); }); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 7fe4cde6384be..995ae977d7297 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -7,37 +7,29 @@ import { ProjectSearchEntriesResult, ProjectSearchContentsResult, } from "@t3tools/contracts"; -import { - WorkspaceSearchIndex, - WORKSPACE_INDEX_PAGE_SIZE, - WorkspaceSearchIndexCreateFailed, - WorkspaceSearchIndexRefreshFailed, - WorkspaceSearchIndexScanTimedOut, - WorkspaceSearchIndexSearchFailed, - type WorkspaceSearchIndexVariant, -} from "./WorkspaceSearchIndexService.ts"; +import * as WorkspaceSearchIndexService from "./WorkspaceSearchIndexService.ts"; import * as WorkspaceSearchHost from "./WorkspaceSearchHost.ts"; export * from "./WorkspaceSearchIndexService.ts"; const WORKSPACE_INDEX_IDLE_TTL = "15 minutes"; -const isCreateFailed = Schema.is(WorkspaceSearchIndexCreateFailed); -const isScanTimedOut = Schema.is(WorkspaceSearchIndexScanTimedOut); -const isSearchFailed = Schema.is(WorkspaceSearchIndexSearchFailed); -const isRefreshFailed = Schema.is(WorkspaceSearchIndexRefreshFailed); +const isCreateFailed = Schema.is(WorkspaceSearchIndexService.WorkspaceSearchIndexCreateFailed); +const isScanTimedOut = Schema.is(WorkspaceSearchIndexService.WorkspaceSearchIndexScanTimedOut); +const isSearchFailed = Schema.is(WorkspaceSearchIndexService.WorkspaceSearchIndexSearchFailed); +const isRefreshFailed = Schema.is(WorkspaceSearchIndexService.WorkspaceSearchIndexRefreshFailed); const decodeList = Schema.decodeUnknownEffect(ProjectListEntriesResult); const decodeSearch = Schema.decodeUnknownEffect(ProjectSearchEntriesResult); const decodeContents = Schema.decodeUnknownEffect(ProjectSearchContentsResult); export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cwd: string, - variant: WorkspaceSearchIndexVariant = "paths", + variant: WorkspaceSearchIndexService.WorkspaceSearchIndexVariant = "paths", ) { const host = yield* WorkspaceSearchHost.WorkspaceSearchHost; const remote = yield* host.open(cwd, variant).pipe( Effect.mapError((cause) => isCreateFailed(cause) || isScanTimedOut(cause) ? cause - : new WorkspaceSearchIndexCreateFailed({ + : new WorkspaceSearchIndexService.WorkspaceSearchIndexCreateFailed({ cwd, reason: "Workspace search process could not initialize.", cause, @@ -48,7 +40,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const searchFailure = (queryLength: number, pageSize: number) => (cause: unknown) => isSearchFailed(cause) ? cause - : new WorkspaceSearchIndexSearchFailed({ + : new WorkspaceSearchIndexService.WorkspaceSearchIndexSearchFailed({ cwd, queryLength, pageSize, @@ -56,13 +48,13 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cause, }); - return WorkspaceSearchIndex.of({ + return WorkspaceSearchIndexService.WorkspaceSearchIndex.of({ list: () => remote .request({ method: "list" }) .pipe( Effect.flatMap(decodeList), - Effect.mapError(searchFailure(0, WORKSPACE_INDEX_PAGE_SIZE)), + Effect.mapError(searchFailure(0, WorkspaceSearchIndexService.WORKSPACE_INDEX_PAGE_SIZE)), ), search: (query, limit, kind, imageOnly) => remote @@ -72,7 +64,9 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( Effect.mapError( searchFailure( query.length, - imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1), + imageOnly + ? WorkspaceSearchIndexService.WORKSPACE_INDEX_PAGE_SIZE + : Math.max(1, limit + 1), ), ), ), @@ -89,7 +83,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( Effect.mapError((cause) => isRefreshFailed(cause) || isScanTimedOut(cause) ? cause - : new WorkspaceSearchIndexRefreshFailed({ + : new WorkspaceSearchIndexService.WorkspaceSearchIndexRefreshFailed({ cwd, reason: "Workspace search process failed.", cause, @@ -104,16 +98,21 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( * content-search index of the same workspace are separate resources with * independent lifecycles. "\n" cannot appear in a filesystem path. */ -export const workspaceSearchIndexKey = (cwd: string, variant: WorkspaceSearchIndexVariant) => - `${variant}\n${cwd}`; +export const workspaceSearchIndexKey = ( + cwd: string, + variant: WorkspaceSearchIndexService.WorkspaceSearchIndexVariant, +) => `${variant}\n${cwd}`; function parseWorkspaceSearchIndexKey(key: string): { readonly cwd: string; - readonly variant: WorkspaceSearchIndexVariant; + readonly variant: WorkspaceSearchIndexService.WorkspaceSearchIndexVariant; } { const separatorIndex = key.indexOf("\n"); return { - variant: key.slice(0, separatorIndex) as WorkspaceSearchIndexVariant, + variant: key.slice( + 0, + separatorIndex, + ) as WorkspaceSearchIndexService.WorkspaceSearchIndexVariant, cwd: key.slice(separatorIndex + 1), }; } @@ -128,7 +127,7 @@ function parseWorkspaceSearchIndexKey(key: string): { */ export const layer = (key: string) => { const { cwd, variant } = parseWorkspaceSearchIndexKey(key); - return Layer.effect(WorkspaceSearchIndex, make(cwd, variant)); + return Layer.effect(WorkspaceSearchIndexService.WorkspaceSearchIndex, make(cwd, variant)); }; export class WorkspaceSearchIndexMap extends LayerMap.Service()( From c7e5f9656a2327736913162ba6c4346f0820dd96 Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:39:12 +0000 Subject: [PATCH 8/9] fix(server): preserve POSIX filenames in native search results --- .../workspace/NativeWorkspaceSearchIndex.ts | 36 +++++++++++-------- .../src/workspace/WorkspaceEntries.test.ts | 24 +++++++++++++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts index 7e54146d86751..80d13e85a78e6 100644 --- a/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/NativeWorkspaceSearchIndex.ts @@ -20,6 +20,7 @@ import type { ProjectSearchEntriesResult, } from "@t3tools/contracts"; import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as WorkspaceSearchIndexService from "./WorkspaceSearchIndexService.ts"; @@ -35,8 +36,8 @@ const CONTENT_SEARCH_TIME_BUDGET_MS = 250; const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100; const CONTENT_SEARCH_MAX_CANDIDATES = 25_000; -function toPosixPath(input: string): string { - return input.replaceAll("\\", "/"); +function toPosixPath(input: string, platform: NodeJS.Platform): string { + return platform === "win32" ? input.replaceAll("\\", "/") : input; } function trimDirectorySeparator(input: string): string { @@ -48,8 +49,8 @@ function parentPathOf(input: string): string | undefined { return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex); } -function toProjectEntry(item: MixedItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath)); +function toProjectEntry(item: MixedItem, platform: NodeJS.Platform): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath, platform)); if (!normalizedPath) { return null; } @@ -60,23 +61,24 @@ function toProjectEntry(item: MixedItem): ProjectEntry | null { }; } -function toFileEntry(item: FileItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath)); +function toFileEntry(item: FileItem, platform: NodeJS.Platform): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath, platform)); return normalizedPath ? { path: normalizedPath, kind: "file" } : null; } -function toDirectoryEntry(item: DirItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath)); +function toDirectoryEntry(item: DirItem, platform: NodeJS.Platform): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath, platform)); return normalizedPath ? { path: normalizedPath, kind: "directory" } : null; } function mapFileSearchResult( result: SearchResult, limit: number, + platform: NodeJS.Platform, imageOnly = false, ): ProjectSearchEntriesResult { const entries = result.items.flatMap((item) => { - const entry = toFileEntry(item); + const entry = toFileEntry(item, platform); return entry && (!imageOnly || isWorkspaceImagePreviewPath(entry.path)) ? [entry] : []; }); return { @@ -88,9 +90,10 @@ function mapFileSearchResult( function mapDirectorySearchResult( result: DirSearchResult, limit: number, + platform: NodeJS.Platform, ): ProjectSearchEntriesResult { const entries = result.items.flatMap((item) => { - const entry = toDirectoryEntry(item); + const entry = toDirectoryEntry(item, platform); return entry ? [entry] : []; }); const rootDirectoryCount = result.items.some((item) => item.relativePath.length === 0) ? 1 : 0; @@ -103,10 +106,11 @@ function mapDirectorySearchResult( function mapMixedSearchResult( result: MixedSearchResult, limit: number, + platform: NodeJS.Platform, ): { readonly entries: ProjectEntry[]; readonly truncated: boolean } { const entries: ProjectEntry[] = []; for (const item of result.items) { - const entry = toProjectEntry(item); + const entry = toProjectEntry(item, platform); if (entry) { entries.push(entry); } @@ -267,6 +271,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cwd: string, variant: WorkspaceSearchIndexService.WorkspaceSearchIndexVariant = "paths", ) { + const platform = yield* HostProcessPlatform; const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) => Effect.try({ try: () => finder.destroy(), @@ -356,6 +361,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const mapped = mapMixedSearchResult( result, WorkspaceSearchIndexService.WORKSPACE_INDEX_MAX_ENTRIES, + platform, ); const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => left.path.localeCompare(right.path), @@ -377,18 +383,18 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const result = yield* runSearch(query, pageSize, "fileSearch", () => finder.fileSearch(query, { pageSize }), ); - return mapFileSearchResult(result, limit, imageOnly); + return mapFileSearchResult(result, limit, platform, imageOnly); } if (kind === "directory") { const result = yield* runSearch(query, pageSize, "directorySearch", () => finder.directorySearch(query, { pageSize }), ); - return mapDirectorySearchResult(result, limit); + return mapDirectorySearchResult(result, limit, platform); } const result = yield* runSearch(query, pageSize, "mixedSearch", () => finder.mixedSearch(query, { pageSize }), ); - return mapMixedSearchResult(result, limit); + return mapMixedSearchResult(result, limit, platform); }); const searchContents: WorkspaceSearchIndexService.WorkspaceSearchIndex["Service"]["searchContents"] = @@ -429,7 +435,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( ); if (matchRanges.length === 0) continue; pageMatches.push({ - path: toPosixPath(match.relativePath), + path: toPosixPath(match.relativePath, platform), lineNumber: match.lineNumber, lineContent: match.lineContent, matchRanges, diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index 6067ca8af9e34..36bfe526570db 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -193,6 +193,30 @@ it.layer(TestLayer, { excludeTestServices: true })("WorkspaceEntries", (it) => { 60_000, ); + it.effect.skipIf(HostProcessPlatform.defaultValue() === "win32")( + "preserves literal backslashes in POSIX search results", + () => + Effect.gen(function* () { + const cwd = yield* makeTempDir(); + const relativePath = "src/foo\\bar.ts"; + yield* writeTextFile(cwd, relativePath, "synthetic content\n"); + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + expect( + (yield* workspaceEntries.search({ cwd, query: "foo", limit: 10 })).entries, + ).toContainEqual({ path: relativePath, kind: "file" }); + expect( + (yield* workspaceEntries.searchContents({ + cwd, + query: "synthetic", + limit: 10, + caseSensitive: true, + wholeWord: false, + useRegex: false, + })).matches, + ).toContainEqual(expect.objectContaining({ path: relativePath })); + }), + ); + it.effect("returns the complete cached workspace index", () => Effect.gen(function* () { const cwd = yield* makeTempDir(); From bcd24076e739be2fa3eb5c1cc9e6cdbd2a54ec57 Mon Sep 17 00:00:00 2001 From: Aditya Garud <153842990+yashranaway@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:54:48 +0000 Subject: [PATCH 9/9] fix(server): give queued searches a fresh active deadline --- apps/server/scripts/workspace-search-mock.ts | 35 +++++---- .../src/workspace/WorkspaceSearchHost.ts | 77 +++++++++---------- .../workspace/WorkspaceSearchIndex.test.ts | 33 ++++++++ 3 files changed, 89 insertions(+), 56 deletions(-) diff --git a/apps/server/scripts/workspace-search-mock.ts b/apps/server/scripts/workspace-search-mock.ts index c2e6cd3f55da8..6a602acaec707 100644 --- a/apps/server/scripts/workspace-search-mock.ts +++ b/apps/server/scripts/workspace-search-mock.ts @@ -14,6 +14,15 @@ const decodeSearchRequest = Schema.decodeUnknownSync(SearchRequest); const encodeSearchResponse = Schema.encodeSync(SearchResponse); const directories = new Map(); +const hold = (value: unknown) => { + const socket = NodeNet.connect(Number(process.env.T3_SEARCH_TEST_RECEIPT_PORT), "127.0.0.1", () => + socket.write("blocked"), + ); + socket.once("data", () => { + socket.end(); + process.send?.(encodeSearchResponse(Exit.succeed(value))); + }); +}; const block = () => { const socket = NodeNet.connect( Number(process.env.T3_SEARCH_TEST_RECEIPT_PORT), @@ -30,6 +39,14 @@ process.on("message", (message) => { const { id, operation: input } = decodeSearchRequest(message); if (input.method === "initialize") directories.set(id, input.cwd); const cwd = directories.get(id) ?? ""; + if ( + input.method === "initialize" && + cwd === "hold-initialize" && + process.env.T3_SEARCH_TEST_HOLD_INITIALIZE === "1" + ) { + hold(null); + return; + } if (input.method === "dispose") { if (cwd === "block-dispose") { block(); @@ -60,21 +77,9 @@ process.on("message", (message) => { return; } if (input.method === "search" && input.query === "hold") { - const socket = NodeNet.connect( - Number(process.env.T3_SEARCH_TEST_RECEIPT_PORT), - "127.0.0.1", - () => socket.write("blocked"), - ); - socket.once("data", () => { - socket.end(); - process.send?.( - encodeSearchResponse( - Exit.succeed({ - entries: [{ path: String(process.pid), kind: "file" }], - truncated: false, - }), - ), - ); + hold({ + entries: [{ path: String(process.pid), kind: "file" }], + truncated: false, }); return; } diff --git a/apps/server/src/workspace/WorkspaceSearchHost.ts b/apps/server/src/workspace/WorkspaceSearchHost.ts index 1539faf3caa83..fd071fd2a91f6 100644 --- a/apps/server/src/workspace/WorkspaceSearchHost.ts +++ b/apps/server/src/workspace/WorkspaceSearchHost.ts @@ -61,47 +61,42 @@ export const make = Effect.gen(function* () { const id = ++nextId; let released = false; const initialize: SearchOperation = { method: "initialize", cwd, variant }; - const run = Effect.fn("WorkspaceSearchHost.request")( - function* (operation: SearchOperation) { - yield* Effect.annotateCurrentSpan({ cwd, variant, operation: operation.method }); - if (closed || released) - return yield* new WorkspaceSearchProcessFailed({ - cause: new Error("Workspace search index is closed."), - }); - activeIndex = id; - return yield* Effect.gen(function* () { - const active = yield* Effect.gen(function* () { - if (current) return current; - const process = yield* startSearchProcess(); - current = { process, indexes: new Set() }; - return current; - }).pipe(Effect.uninterruptible); - if (!active.indexes.has(id)) { - yield* active.process.request({ id, operation: initialize }); - active.indexes.add(id); - } - return operation.method === "initialize" - ? null - : yield* active.process.request({ id, operation }); - }).pipe( - Effect.onError((cause) => - cause.reasons.length > 0 && - cause.reasons.every( - (reason) => Cause.isFailReason(reason) && isRecoverable(reason.error), - ) - ? Effect.void - : stop(), - ), - Effect.ensuring( - Effect.sync(() => { - activeIndex = undefined; - }), - ), - ); - }, - semaphore.withPermits(1), - Effect.timeout("20 seconds"), - ); + const run = Effect.fn("WorkspaceSearchHost.request")(function* (operation: SearchOperation) { + yield* Effect.annotateCurrentSpan({ cwd, variant, operation: operation.method }); + if (closed || released) + return yield* new WorkspaceSearchProcessFailed({ + cause: new Error("Workspace search index is closed."), + }); + activeIndex = id; + return yield* Effect.gen(function* () { + const active = yield* Effect.gen(function* () { + if (current) return current; + const process = yield* startSearchProcess(); + current = { process, indexes: new Set() }; + return current; + }).pipe(Effect.uninterruptible); + if (!active.indexes.has(id)) { + yield* active.process.request({ id, operation: initialize }); + active.indexes.add(id); + } + return operation.method === "initialize" + ? null + : yield* active.process.request({ id, operation }); + }).pipe( + Effect.timeout("20 seconds"), + Effect.onError((cause) => + cause.reasons.length > 0 && + cause.reasons.every((reason) => Cause.isFailReason(reason) && isRecoverable(reason.error)) + ? Effect.void + : stop(), + ), + Effect.ensuring( + Effect.sync(() => { + activeIndex = undefined; + }), + ), + ); + }, semaphore.withPermits(1)); yield* Effect.addFinalizer(() => Effect.gen(function* () { diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 7b66315da8917..fe9e8432dfc1a 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -92,6 +92,39 @@ it.effect( ), ); +it.effect("gives a queued request a fresh deadline for a delayed worker rebuild", () => + Effect.scoped( + Effect.gen(function* () { + const activeControl = yield* receipts(); + const rebuildControl = yield* receipts(); + yield* Effect.gen(function* () { + const index = yield* Index.make("workspace"); + const other = yield* Index.make("hold-initialize"); + const original = yield* index.list(); + const blocked = yield* index.search("block", 1).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(activeControl.blocked); + yield* TestClock.adjust("1 second"); + const queued = yield* other.list().pipe( + Effect.provideService(HostProcessEnvironment, { + ...rebuildControl.environment, + T3_SEARCH_TEST_HOLD_INITIALIZE: "1", + }), + Effect.forkChild, + ); + yield* TestClock.adjust("19 seconds"); + expect(Exit.isFailure(yield* Fiber.join(blocked))).toBe(true); + yield* Deferred.await(activeControl.exited); + yield* Deferred.await(rebuildControl.blocked); + yield* TestClock.adjust("2 seconds"); + yield* rebuildControl.release; + const rebuilt = yield* Fiber.join(queued); + expect(rebuilt.entries).not.toEqual(original.entries); + expect((yield* index.list()).entries).toEqual(rebuilt.entries); + }).pipe(Effect.provideService(HostProcessEnvironment, activeControl.environment)); + }).pipe(withWorker), + ), +); + it.effect("cancels a blocked request before allowing the queued request to rebuild", () => Effect.scoped( Effect.gen(function* () {