Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 64 additions & 8 deletions src/knowledge/scanner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ const TEST_PATTERNS = [
const MAX_FILE_SIZE = 512 * 1024; // 512KB — skip large generated files
const MAX_DEPTH = 15;
const SCAN_TIMEOUT_MS = 30_000;
// Total-size budgets: the time limit alone lets a huge repository grow the
// graph without bound until the timeout fires (AGT-4871).
const MAX_SCAN_FILES = 20_000;
const MAX_SCAN_NODES = 50_000;

// Import Regex Patterns

Expand All @@ -77,6 +81,23 @@ const PY_IMPORT = /^import\s+([\w.]+)/gm;
export interface ScanOptions {
maxDepth?: number;
timeoutMs?: number;
/** Maximum number of source files added to the graph (default MAX_SCAN_FILES). */
maxFiles?: number;
/** Maximum number of graph nodes created by the walk, root included (default MAX_SCAN_NODES). */
maxNodes?: number;
}

interface WalkContext {
graph: KnowledgeGraph;
maxDepth: number;
startTime: number;
timeoutMs: number;
maxFiles: number;
maxNodes: number;
fileCount: number;
/** Set once a budget is exhausted; the walk stops descending and adding. */
budgetExhausted: boolean;
incompleteReasons: string[];
}

/**
Expand All @@ -90,6 +111,8 @@ export async function scanProject(
const graph = new KnowledgeGraph(projectSlug, projectPath);
const maxDepth = options.maxDepth ?? MAX_DEPTH;
const timeoutMs = options.timeoutMs ?? SCAN_TIMEOUT_MS;
const maxFiles = Math.max(0, options.maxFiles ?? MAX_SCAN_FILES);
const maxNodes = Math.max(1, options.maxNodes ?? MAX_SCAN_NODES);
const startTime = Date.now();

// Project root node
Expand All @@ -102,7 +125,18 @@ export async function scanProject(

// Phase 1: Directory walking — collect nodes
const incompleteReasons: string[] = [];
await walkDirectory(graph, projectPath, '.', 0, maxDepth, startTime, timeoutMs, incompleteReasons);
const ctx: WalkContext = {
graph,
maxDepth,
startTime,
timeoutMs,
maxFiles,
maxNodes,
fileCount: 0,
budgetExhausted: false,
incompleteReasons,
};
await walkDirectory(ctx, projectPath, '.', 0);

// Phase 2: Import parsing — create edges
const modules = [...graph.getNodesByType('module'), ...graph.getNodesByType('test_file')];
Expand All @@ -125,7 +159,11 @@ export async function scanProject(

graph.scannedAt = Date.now();
graph.incomplete = incompleteReasons.length > 0;
graph.incompleteReasons = Array.from(new Set(incompleteReasons)).slice(0, 50);
// Budget reasons go first: a repo with many size/depth reasons would
// otherwise push them past the 50-entry cut.
const unique = Array.from(new Set(incompleteReasons));
const isBudget = (reason: string) => / budget \d+ reached$/.test(reason);
graph.incompleteReasons = [...unique.filter(isBudget), ...unique.filter(r => !isBudget(r))].slice(0, 50);
return graph;
}

Expand Down Expand Up @@ -218,15 +256,13 @@ export async function incrementalUpdate(
// Internal: Directory Walking

async function walkDirectory(
graph: KnowledgeGraph,
ctx: WalkContext,
currentPath: string,
relPath: string,
depth: number,
maxDepth: number,
startTime: number,
timeoutMs: number,
incompleteReasons: string[],
): Promise<void> {
const { graph, maxDepth, startTime, timeoutMs, incompleteReasons } = ctx;
if (ctx.budgetExhausted) return;
if (depth > maxDepth) {
incompleteReasons.push(`${relPath || '.'}: depth limit ${maxDepth} reached`);
return;
Expand All @@ -243,13 +279,17 @@ async function walkDirectory(
} catch {
return; // Inaccessible directory
}
// Name order, so which files a budget cuts off is the same on every scan.
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));

for (const entry of entries) {
if (ctx.budgetExhausted) return;
const entryPath = join(currentPath, entry.name);
const entryRelPath = relPath === '.' ? entry.name : `${relPath}/${entry.name}`;

if (entry.isDirectory()) {
if (SKIP_DIRS.has(entry.name) || SKIP_DIR_PREFIXES.some(p => entry.name.startsWith(p))) continue;
if (!reserveNode(ctx, entryRelPath)) return;

graph.addNode({
id: entryRelPath,
Expand All @@ -259,11 +299,18 @@ async function walkDirectory(
});
graph.addEdge({ source: relPath === '.' ? '.' : relPath, target: entryRelPath, type: 'contains' });

await walkDirectory(graph, entryPath, entryRelPath, depth + 1, maxDepth, startTime, timeoutMs, incompleteReasons);
await walkDirectory(ctx, entryPath, entryRelPath, depth + 1);
} else if (entry.isFile()) {
const ext = extname(entry.name);
if (!SOURCE_EXTENSIONS.has(ext)) continue;

if (ctx.fileCount >= ctx.maxFiles) {
ctx.budgetExhausted = true;
incompleteReasons.push(`${entryRelPath}: file budget ${ctx.maxFiles} reached`);
return;
}
if (!reserveNode(ctx, entryRelPath)) return;

const language = detectLanguage(ext);
const isTest = isTestFile(entry.name);

Expand All @@ -286,11 +333,20 @@ async function walkDirectory(
path: entryRelPath,
metrics,
});
ctx.fileCount++;
graph.addEdge({ source: relPath === '.' ? '.' : relPath, target: entryRelPath, type: 'contains' });
}
}
}

/** True when one more node fits the node budget; otherwise marks the walk exhausted. */
function reserveNode(ctx: WalkContext, entryRelPath: string): boolean {
if (ctx.graph.nodeCount < ctx.maxNodes) return true;
ctx.budgetExhausted = true;
ctx.incompleteReasons.push(`${entryRelPath}: node budget ${ctx.maxNodes} reached`);
return false;
}

// Internal: Import Parsing

async function parseImports(
Expand Down
59 changes: 59 additions & 0 deletions src/knowledge/scannerIncomplete.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,4 +111,63 @@ describe('knowledge scan incompleteness', () => {
expect(snapshot.incomplete).toBe(false);
expect(snapshot.incompleteReasons).toEqual([]);
});

// AGT-4871 — total-size budgets so a huge repository cannot grow the graph
// without bound until the timeout fires.
it('stops at the file budget and records it as an incomplete reason', async () => {
for (let i = 0; i < 12; i++) {
await writeProjectFile(`src/m${i}.ts`, `export const m${i} = ${i};\n`);
}

const graph = await scanProject(tmp, 'test-project', { maxFiles: 5 });
const sourceNodes = [...graph.getNodesByType('module'), ...graph.getNodesByType('test_file')];

expect(sourceNodes.length).toBe(5);
// Name-ordered walk: the same five files survive every scan.
expect(sourceNodes.map(n => n.id).sort()).toEqual(['src/m0.ts', 'src/m1.ts', 'src/m10.ts', 'src/m11.ts', 'src/m2.ts']);
expect(graph.incomplete).toBe(true);
expect(graph.incompleteReasons.some(reason => reason.includes('file budget 5'))).toBe(true);
});

it('keeps the node count at or below the node budget', async () => {
for (let d = 0; d < 6; d++) {
for (let i = 0; i < 4; i++) {
await writeProjectFile(`pkg${d}/m${i}.ts`, `export const m${i} = ${i};\n`);
}
}

const graph = await scanProject(tmp, 'test-project', { maxNodes: 10 });

expect(graph.nodeCount).toBeLessThanOrEqual(10);
expect(graph.incomplete).toBe(true);
expect(graph.incompleteReasons.some(reason => reason.includes('node budget 10'))).toBe(true);
});

it('keeps the budget reason when more than 50 other reasons were recorded', async () => {
for (let i = 0; i < 60; i++) {
await writeProjectFile(`a/huge${String(i).padStart(2, '0')}.ts`, `// ${'x'.repeat(520 * 1024)}\n`);
}
for (let i = 0; i < 5; i++) {
await writeProjectFile(`z/m${i}.ts`, `export const m${i} = ${i};\n`);
}

// Size-limited files reserve a node but are not added, so use the file budget.
const graph = await scanProject(tmp, 'test-project', { maxFiles: 2 });

// Directories are walked in name order, so all 60 size reasons come first.
expect(graph.incompleteReasons.length).toBe(50);
expect(graph.incompleteReasons.some(reason => reason.includes('file budget 2'))).toBe(true);
});

it('does not report a budget when the project fits inside it', async () => {
for (let i = 0; i < 3; i++) {
await writeProjectFile(`src/m${i}.ts`, `export const m${i} = ${i};\n`);
}

const graph = await scanProject(tmp, 'test-project', { maxFiles: 3, maxNodes: 5 });

// root + src + 3 modules = 5 nodes, exactly at both budgets.
expect(graph.nodeCount).toBe(5);
expect(graph.incomplete).toBe(false);
});
});
Loading