From 8065289d69050f5ab758361eabff2820a8c7d7ff Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 07:14:19 +0900 Subject: [PATCH 1/2] perf(knowledge): bound full scans by total file and node budgets (AGT-4871) scanProject only had time, file-size and depth limits, so a large repository kept growing the graph until the timeout. ScanOptions now takes maxFiles/maxNodes (defaults 20,000 / 50,000); hitting either stops the walk and records the budget in incompleteReasons. --- src/knowledge/scanner.ts | 64 ++++++++++++++++++++++--- src/knowledge/scannerIncomplete.test.ts | 41 ++++++++++++++++ 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/src/knowledge/scanner.ts b/src/knowledge/scanner.ts index eb8ef8e2..8f891d8e 100644 --- a/src/knowledge/scanner.ts +++ b/src/knowledge/scanner.ts @@ -60,6 +60,10 @@ const TEST_PATTERNS = [ const MAX_FILE_SIZE = 512 * 1024; // 512KB — skip large generated files const MAX_DEPTH = 15; const SCAN_TIMEOUT_MS = 30_000; +// Total-size budgets: the time limit alone lets a huge repository grow the +// graph without bound until the timeout fires (AGT-4871). +const MAX_SCAN_FILES = 20_000; +const MAX_SCAN_NODES = 50_000; // Import Regex Patterns @@ -77,6 +81,23 @@ const PY_IMPORT = /^import\s+([\w.]+)/gm; export interface ScanOptions { maxDepth?: number; timeoutMs?: number; + /** Maximum number of source files added to the graph (default MAX_SCAN_FILES). */ + maxFiles?: number; + /** Maximum number of graph nodes created by the walk, root included (default MAX_SCAN_NODES). */ + maxNodes?: number; +} + +interface WalkContext { + graph: KnowledgeGraph; + maxDepth: number; + startTime: number; + timeoutMs: number; + maxFiles: number; + maxNodes: number; + fileCount: number; + /** Set once a budget is exhausted; the walk stops descending and adding. */ + budgetExhausted: boolean; + incompleteReasons: string[]; } /** @@ -90,6 +111,8 @@ export async function scanProject( const graph = new KnowledgeGraph(projectSlug, projectPath); const maxDepth = options.maxDepth ?? MAX_DEPTH; const timeoutMs = options.timeoutMs ?? SCAN_TIMEOUT_MS; + const maxFiles = Math.max(0, options.maxFiles ?? MAX_SCAN_FILES); + const maxNodes = Math.max(1, options.maxNodes ?? MAX_SCAN_NODES); const startTime = Date.now(); // Project root node @@ -102,7 +125,18 @@ export async function scanProject( // Phase 1: Directory walking — collect nodes const incompleteReasons: string[] = []; - await walkDirectory(graph, projectPath, '.', 0, maxDepth, startTime, timeoutMs, incompleteReasons); + const ctx: WalkContext = { + graph, + maxDepth, + startTime, + timeoutMs, + maxFiles, + maxNodes, + fileCount: 0, + budgetExhausted: false, + incompleteReasons, + }; + await walkDirectory(ctx, projectPath, '.', 0); // Phase 2: Import parsing — create edges const modules = [...graph.getNodesByType('module'), ...graph.getNodesByType('test_file')]; @@ -218,15 +252,13 @@ export async function incrementalUpdate( // Internal: Directory Walking async function walkDirectory( - graph: KnowledgeGraph, + ctx: WalkContext, currentPath: string, relPath: string, depth: number, - maxDepth: number, - startTime: number, - timeoutMs: number, - incompleteReasons: string[], ): Promise { + const { graph, maxDepth, startTime, timeoutMs, incompleteReasons } = ctx; + if (ctx.budgetExhausted) return; if (depth > maxDepth) { incompleteReasons.push(`${relPath || '.'}: depth limit ${maxDepth} reached`); return; @@ -245,11 +277,13 @@ async function walkDirectory( } for (const entry of entries) { + if (ctx.budgetExhausted) return; const entryPath = join(currentPath, entry.name); const entryRelPath = relPath === '.' ? entry.name : `${relPath}/${entry.name}`; if (entry.isDirectory()) { if (SKIP_DIRS.has(entry.name) || SKIP_DIR_PREFIXES.some(p => entry.name.startsWith(p))) continue; + if (!reserveNode(ctx, entryRelPath)) return; graph.addNode({ id: entryRelPath, @@ -259,11 +293,18 @@ async function walkDirectory( }); graph.addEdge({ source: relPath === '.' ? '.' : relPath, target: entryRelPath, type: 'contains' }); - await walkDirectory(graph, entryPath, entryRelPath, depth + 1, maxDepth, startTime, timeoutMs, incompleteReasons); + await walkDirectory(ctx, entryPath, entryRelPath, depth + 1); } else if (entry.isFile()) { const ext = extname(entry.name); if (!SOURCE_EXTENSIONS.has(ext)) continue; + if (ctx.fileCount >= ctx.maxFiles) { + ctx.budgetExhausted = true; + incompleteReasons.push(`${entryRelPath}: file budget ${ctx.maxFiles} reached`); + return; + } + if (!reserveNode(ctx, entryRelPath)) return; + const language = detectLanguage(ext); const isTest = isTestFile(entry.name); @@ -286,11 +327,20 @@ async function walkDirectory( path: entryRelPath, metrics, }); + ctx.fileCount++; graph.addEdge({ source: relPath === '.' ? '.' : relPath, target: entryRelPath, type: 'contains' }); } } } +/** True when one more node fits the node budget; otherwise marks the walk exhausted. */ +function reserveNode(ctx: WalkContext, entryRelPath: string): boolean { + if (ctx.graph.nodeCount < ctx.maxNodes) return true; + ctx.budgetExhausted = true; + ctx.incompleteReasons.push(`${entryRelPath}: node budget ${ctx.maxNodes} reached`); + return false; +} + // Internal: Import Parsing async function parseImports( diff --git a/src/knowledge/scannerIncomplete.test.ts b/src/knowledge/scannerIncomplete.test.ts index d4759764..1cd536f1 100644 --- a/src/knowledge/scannerIncomplete.test.ts +++ b/src/knowledge/scannerIncomplete.test.ts @@ -111,4 +111,45 @@ describe('knowledge scan incompleteness', () => { expect(snapshot.incomplete).toBe(false); expect(snapshot.incompleteReasons).toEqual([]); }); + + // AGT-4871 — total-size budgets so a huge repository cannot grow the graph + // without bound until the timeout fires. + it('stops at the file budget and records it as an incomplete reason', async () => { + for (let i = 0; i < 12; i++) { + await writeProjectFile(`src/m${i}.ts`, `export const m${i} = ${i};\n`); + } + + const graph = await scanProject(tmp, 'test-project', { maxFiles: 5 }); + const sourceNodes = [...graph.getNodesByType('module'), ...graph.getNodesByType('test_file')]; + + expect(sourceNodes.length).toBe(5); + expect(graph.incomplete).toBe(true); + expect(graph.incompleteReasons.some(reason => reason.includes('file budget 5'))).toBe(true); + }); + + it('keeps the node count at or below the node budget', async () => { + for (let d = 0; d < 6; d++) { + for (let i = 0; i < 4; i++) { + await writeProjectFile(`pkg${d}/m${i}.ts`, `export const m${i} = ${i};\n`); + } + } + + const graph = await scanProject(tmp, 'test-project', { maxNodes: 10 }); + + expect(graph.nodeCount).toBeLessThanOrEqual(10); + expect(graph.incomplete).toBe(true); + expect(graph.incompleteReasons.some(reason => reason.includes('node budget 10'))).toBe(true); + }); + + it('does not report a budget when the project fits inside it', async () => { + for (let i = 0; i < 3; i++) { + await writeProjectFile(`src/m${i}.ts`, `export const m${i} = ${i};\n`); + } + + const graph = await scanProject(tmp, 'test-project', { maxFiles: 3, maxNodes: 5 }); + + // root + src + 3 modules = 5 nodes, exactly at both budgets. + expect(graph.nodeCount).toBe(5); + expect(graph.incomplete).toBe(false); + }); }); From 6b94ea1e803a3fc9c0794b8822ff581e03892b31 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 07:20:42 +0900 Subject: [PATCH 2/2] fix(knowledge): keep budget reasons past the 50-entry cut and walk in name order (AGT-4871) --- src/knowledge/scanner.ts | 8 +++++++- src/knowledge/scannerIncomplete.test.ts | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/knowledge/scanner.ts b/src/knowledge/scanner.ts index 8f891d8e..eb7018ae 100644 --- a/src/knowledge/scanner.ts +++ b/src/knowledge/scanner.ts @@ -159,7 +159,11 @@ export async function scanProject( graph.scannedAt = Date.now(); graph.incomplete = incompleteReasons.length > 0; - graph.incompleteReasons = Array.from(new Set(incompleteReasons)).slice(0, 50); + // Budget reasons go first: a repo with many size/depth reasons would + // otherwise push them past the 50-entry cut. + const unique = Array.from(new Set(incompleteReasons)); + const isBudget = (reason: string) => / budget \d+ reached$/.test(reason); + graph.incompleteReasons = [...unique.filter(isBudget), ...unique.filter(r => !isBudget(r))].slice(0, 50); return graph; } @@ -275,6 +279,8 @@ async function walkDirectory( } catch { return; // Inaccessible directory } + // Name order, so which files a budget cuts off is the same on every scan. + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); for (const entry of entries) { if (ctx.budgetExhausted) return; diff --git a/src/knowledge/scannerIncomplete.test.ts b/src/knowledge/scannerIncomplete.test.ts index 1cd536f1..85e5d8d5 100644 --- a/src/knowledge/scannerIncomplete.test.ts +++ b/src/knowledge/scannerIncomplete.test.ts @@ -123,6 +123,8 @@ describe('knowledge scan incompleteness', () => { const sourceNodes = [...graph.getNodesByType('module'), ...graph.getNodesByType('test_file')]; expect(sourceNodes.length).toBe(5); + // Name-ordered walk: the same five files survive every scan. + expect(sourceNodes.map(n => n.id).sort()).toEqual(['src/m0.ts', 'src/m1.ts', 'src/m10.ts', 'src/m11.ts', 'src/m2.ts']); expect(graph.incomplete).toBe(true); expect(graph.incompleteReasons.some(reason => reason.includes('file budget 5'))).toBe(true); }); @@ -141,6 +143,22 @@ describe('knowledge scan incompleteness', () => { expect(graph.incompleteReasons.some(reason => reason.includes('node budget 10'))).toBe(true); }); + it('keeps the budget reason when more than 50 other reasons were recorded', async () => { + for (let i = 0; i < 60; i++) { + await writeProjectFile(`a/huge${String(i).padStart(2, '0')}.ts`, `// ${'x'.repeat(520 * 1024)}\n`); + } + for (let i = 0; i < 5; i++) { + await writeProjectFile(`z/m${i}.ts`, `export const m${i} = ${i};\n`); + } + + // Size-limited files reserve a node but are not added, so use the file budget. + const graph = await scanProject(tmp, 'test-project', { maxFiles: 2 }); + + // Directories are walked in name order, so all 60 size reasons come first. + expect(graph.incompleteReasons.length).toBe(50); + expect(graph.incompleteReasons.some(reason => reason.includes('file budget 2'))).toBe(true); + }); + it('does not report a budget when the project fits inside it', async () => { for (let i = 0; i < 3; i++) { await writeProjectFile(`src/m${i}.ts`, `export const m${i} = ${i};\n`);