Skip to content

fix(indexer): keep session indexing incremental on append #176

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HengYangDS wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(indexer): keep session indexing incremental on append #176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions dist/indexer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { summarizeConversation } from './summarizer.js';
import { getArchiveDir, getExcludedProjects, getConversationSourceDirs, findJsonlFiles, statIfExists } from './paths.js';
import { formatErrorSentinel, shouldQueueForSummary } from './summary-sentinel.js';
import { getMaxMessageBytes, isOversizeExchange } from './message-size.js';
import { copyIfNewer } from './sync.js';
// Set max output tokens for Claude SDK (used by summarizer)
process.env.CLAUDE_CODE_MAX_OUTPUT_TOKENS = '20000';
// Increase max listeners for concurrent API calls
Expand Down Expand Up @@ -184,13 +185,11 @@ export async function indexSession(sessionId, concurrency = 1, noSummaries = fal
const projectArchive = path.join(ARCHIVE_DIR, project);
fs.mkdirSync(projectArchive, { recursive: true });
const archivePath = path.join(projectArchive, file);
const maxIndexedLine = db.prepare('SELECT COALESCE(MAX(line_end), 0) AS maxLine FROM exchanges WHERE archive_path = ?').get(archivePath).maxLine;
// Archive + parse — source may vanish mid-run (Claude Code cleanup).
let exchanges;
try {
if (!fs.existsSync(archivePath)) {
fs.mkdirSync(path.dirname(archivePath), { recursive: true });
fs.copyFileSync(sourcePath, archivePath);
}
copyIfNewer(sourcePath, archivePath);
exchanges = await parseConversation(sourcePath, project, archivePath);
}
catch (error) {
Expand All @@ -199,6 +198,7 @@ export async function indexSession(sessionId, concurrency = 1, noSummaries = fal
break;
}
if (exchanges.length > 0) {
const newExchanges = exchanges.filter(exchange => exchange.lineStart > maxIndexedLine);
// Generate summary (unless --no-summaries)
const summaryPath = archivePath.replace('.jsonl', '-summary.txt');
if (!noSummaries && shouldQueueForSummary(summaryPath)) {
Expand All @@ -220,7 +220,7 @@ export async function indexSession(sessionId, concurrency = 1, noSummaries = fal
// Index
const maxMessageBytes = getMaxMessageBytes();
let oversizeSkipped = 0;
for (const exchange of exchanges) {
for (const exchange of newExchanges) {
// Skip oversize single messages BEFORE embedding — a foreign
// summarizer's pasted transcript is noise, and embedding it is the
// expensive waste (#139).
Expand All @@ -235,7 +235,7 @@ export async function indexSession(sessionId, concurrency = 1, noSummaries = fal
if (oversizeSkipped > 0) {
console.log(` Skipped ${oversizeSkipped} oversize exchange(s) (> ${maxMessageBytes} bytes; set EPISODIC_MEMORY_MAX_MESSAGE_BYTES to change) — likely embedded-transcript payloads (#139)`);
}
console.log(`✅ Indexed session ${sessionId}: ${exchanges.length} exchanges`);
console.log(`✅ Indexed session ${sessionId}: ${newExchanges.length} new exchanges`);
}
db.close();
break;
Expand Down
Loading