diff --git a/src/memory/reembed.store.test.ts b/src/memory/reembed.store.test.ts new file mode 100644 index 00000000..a0b7c2ab --- /dev/null +++ b/src/memory/reembed.store.test.ts @@ -0,0 +1,82 @@ +import { mkdtempSync } from 'node:fs'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, describe, expect, it, vi } from 'vitest'; + +// Same module-load boundary as compaction.store.test.ts: memoryCore resolves +// MEMORY_DIR from homedir() at import time, so HOME is redirected first and +// everything is imported dynamically. Only the encoder is replaced — loading +// the ONNX model is not what this test is about — the table is a real LanceDB +// store, so the batch reads, appends and the swap run against real Arrow rows. +const home = mkdtempSync(join(tmpdir(), 'openswarm-reembed-store-')); +process.env.HOME = home; + +vi.mock('./memoryCore.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + embedPassage: async (text: string) => + Array.from({ length: actual.EMBEDDING_DIM }, (_, d) => ((text.length + d) % 7) / 7 + 0.01), + }; +}); + +const { connect } = await import('@lancedb/lancedb'); +const { EMBEDDING_DIM, PERMANENT_EXPIRY } = await import('./memoryCore.js'); +const { reembedMemoryTable } = await import('./reembed.js'); + +const memoryDir = join(home, '.openswarm', 'memory'); + +afterAll(async () => { + await rm(home, { recursive: true, force: true }); +}); + +function storedRecord(i: number) { + const now = 1_700_000_000_000; + return { + id: `row-${i}`, + type: 'belief', + repo: 'alpha', + derivedFrom: 'src', + metadata: '{}', + title: `title ${i}`, + content: `content ${i}`, + vector: Array.from({ length: EMBEDDING_DIM }, () => 0), + importance: 0.5, + confidence: 0.7, + createdAt: now, + lastUpdated: now, + lastAccessed: now, + trust: 1, + expiresAt: PERMANENT_EXPIRY, + }; +} + +describe('reembedMemoryTable over a stored table', () => { + it('rewrites a store larger than the batch size with every row and a real vector', async () => { + await mkdir(memoryDir, { recursive: true }); + const db = await connect(memoryDir); + const total = 7; + await db.createTable('cognitive_memory', Array.from({ length: total }, (_, i) => storedRecord(i)), { mode: 'overwrite' }); + const before = await db.openTable('cognitive_memory'); + const vectorTypeBefore = String((await before.schema()).fields.find((f) => f.name === 'vector')?.type); + + const result = await reembedMemoryTable({ memoryDir, batchSize: 3 }); + + expect(result.total).toBe(total); + expect(result.reembedded).toBe(total); + const after = await db.openTable('cognitive_memory'); + expect(await after.countRows()).toBe(total); + expect(String((await after.schema()).fields.find((f) => f.name === 'vector')?.type)).toBe(vectorTypeBefore); + const rows = await after.query().limit(total + 1).toArray(); + expect(rows.map((r) => String(r.id)).sort()).toEqual(Array.from({ length: total }, (_, i) => `row-${i}`).sort()); + for (const row of rows) { + const v = Array.from(row.vector as Iterable, Number); + expect(v).toHaveLength(EMBEDDING_DIM); + // The fixture stored zero vectors; every row must now carry the new one. + expect(v.some((x) => x !== 0)).toBe(true); + } + // The temp table is gone once the swap succeeded. + expect((await db.tableNames()).filter((n) => n.includes('_reembed_'))).toEqual([]); + }, 120_000); +}); diff --git a/src/memory/reembed.test.ts b/src/memory/reembed.test.ts index 191a6823..bb7e73a4 100644 --- a/src/memory/reembed.test.ts +++ b/src/memory/reembed.test.ts @@ -4,17 +4,64 @@ import { resolve } from 'node:path'; import { describe, expect, it, vi, beforeEach } from 'vitest'; // memoryCore owns the LanceDB handles and the encoder; reembed is pure orchestration -// over them, so the seam is mocked rather than standing up a real table. +// over them, so the seam is replaced by a small in-memory table store that honours +// offset/limit paging, add, countRows and overwrite. +const LIVE = 'cognitive_memory'; const state = { - rows: [] as any[], - createTable: vi.fn(async (_name: string, _rows: any[], _opts?: any) => ({})), - createEmptyTable: vi.fn(async () => ({})), - dropTable: vi.fn(async () => {}), - openTable: vi.fn(async () => ({ name: 'cognitive_memory' })), + tables: new Map(), + /** Largest page any query asked for or returned, per table. */ + maxPageRead: new Map(), + createTable: vi.fn(), + createEmptyTable: vi.fn(), + dropTable: vi.fn(), + openTable: vi.fn(), setTable: vi.fn(), embedCalls: [] as string[], }; +function fakeTable(name: string) { + return { + name, + schema: async () => ({ fields: [] }), + countRows: async () => state.tables.get(name)!.length, + add: async (rows: any[]) => { + state.tables.get(name)!.push(...rows.map((r) => ({ ...r }))); + return {}; + }, + query: () => { + let offset = 0; + let limit = Infinity; + const q = { + offset: (n: number) => { offset = n; return q; }, + limit: (n: number) => { limit = n; return q; }, + toArray: async () => { + const page = state.tables.get(name)!.slice(offset, offset + limit); + state.maxPageRead.set(name, Math.max(state.maxPageRead.get(name) ?? 0, page.length, limit)); + return page.map((r) => ({ ...r })); + }, + }; + return q; + }, + }; +} + +function resetDb() { + state.tables = new Map([[LIVE, []]]); + state.maxPageRead = new Map(); + state.createTable.mockReset().mockImplementation(async (name: string, rows: any[], opts?: { mode?: string }) => { + if (state.tables.has(name) && opts?.mode !== 'overwrite') throw new Error(`table ${name} exists`); + state.tables.set(name, rows.map((r) => ({ ...r }))); + return fakeTable(name); + }); + state.createEmptyTable.mockReset().mockImplementation(async (name: string) => { + state.tables.set(name, []); + return fakeTable(name); + }); + state.dropTable.mockReset().mockImplementation(async (name: string) => { state.tables.delete(name); }); + state.openTable.mockReset().mockImplementation(async (name: string) => fakeTable(name)); + state.setTable.mockReset(); +} + vi.mock('./memoryCore.js', () => ({ EMBEDDING_DIM: 4, MEMORY_DIR: '/tmp/does-not-matter', @@ -25,18 +72,14 @@ vi.mock('./memoryCore.js', () => ({ dropTable: state.dropTable, openTable: state.openTable, }), - getTable: () => ({ - name: 'cognitive_memory', - schema: async () => ({ fields: [] }), - query: () => ({ limit: () => ({ toArray: async () => state.rows }) }), - }), + getTable: () => fakeTable(LIVE), setTable: state.setTable, embedPassage: vi.fn(async (text: string) => { state.embedCalls.push(text); return [1, 2, 3, 4]; }), normalizeRecords: (rows: any[]) => - rows.map((r) => ({ ...r, title: String(r.title ?? ''), content: String(r.content ?? ''), vector: [] })), + rows.map((r) => ({ ...r, title: String(r.title ?? ''), content: String(r.content ?? ''), vector: r.vector ?? [] })), })); const { reembedMemoryTable } = await import('./reembed.js'); @@ -45,31 +88,37 @@ function row(over: Record = {}) { return { id: 'x', type: 'belief', title: 'T', content: 'C', ...over }; } +const tmp = () => mkdtempSync(resolve(tmpdir(), 'osw-re-')); + +const { embedPassage } = await import('./memoryCore.js'); +const embedMock = () => embedPassage as unknown as ReturnType; + beforeEach(() => { - state.rows = []; + resetDb(); state.embedCalls = []; - state.createTable.mockClear().mockImplementation(async () => ({})); - state.createEmptyTable.mockClear(); - state.dropTable.mockClear(); - state.setTable.mockClear(); + embedMock().mockImplementation(async (text: string) => { + state.embedCalls.push(text); + return [1, 2, 3, 4]; + }); }); describe('reembedMemoryTable', () => { it('re-encodes every record and reports the counts', async () => { - state.rows = [row({ id: 'a' }), row({ id: 'b', title: 'T2', content: 'C2' })]; + state.tables.set(LIVE, [row({ id: 'a' }), row({ id: 'b', title: 'T2', content: 'C2' })]); - const result = await reembedMemoryTable({ memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')) }); + const result = await reembedMemoryTable({ memoryDir: tmp() }); expect(result.total).toBe(2); expect(result.reembedded).toBe(2); expect(result.empty).toBe(0); expect(state.embedCalls).toEqual(['T\nC', 'T2\nC2']); + expect(state.tables.get(LIVE)!.map((r) => r.vector)).toEqual([[1, 2, 3, 4], [1, 2, 3, 4]]); }); it('writes a zero vector for a record with no text instead of calling the encoder', async () => { - state.rows = [row({ id: 'blank', title: '', content: '' })]; + state.tables.set(LIVE, [row({ id: 'blank', title: '', content: '' })]); - const result = await reembedMemoryTable({ memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')) }); + const result = await reembedMemoryTable({ memoryDir: tmp() }); expect(result.empty).toBe(1); expect(result.reembedded).toBe(0); @@ -79,25 +128,28 @@ describe('reembedMemoryTable', () => { }); it('builds a temp table before overwriting the live one, then drops it', async () => { - state.rows = [row()]; + state.tables.set(LIVE, [row()]); - await reembedMemoryTable({ memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')) }); + await reembedMemoryTable({ memoryDir: tmp() }); const names = state.createTable.mock.calls.map((c) => c[0]); expect(names[0]).toMatch(/^cognitive_memory_reembed_/); // validated replacement first - expect(names[1]).toBe('cognitive_memory'); + expect(names[1]).toBe(LIVE); expect(state.createTable.mock.calls[1][2]).toEqual({ mode: 'overwrite' }); expect(state.dropTable).toHaveBeenCalledWith(names[0]); + expect(state.tables.has(names[0])).toBe(false); expect(state.setTable).toHaveBeenCalled(); }); it('keeps the recoverable temp table and does not claim the signature when the swap fails', async () => { - state.rows = [row()]; - const dir = mkdtempSync(resolve(tmpdir(), 'osw-re-')); - // First call builds the temp table; the overwrite of the live table fails. - state.createTable - .mockImplementationOnce(async () => ({})) - .mockImplementationOnce(async () => { throw new Error('disk full'); }); + state.tables.set(LIVE, [row()]); + const dir = tmp(); + const create = state.createTable.getMockImplementation()!; + // Building the temp table works; the overwrite of the live table fails. + state.createTable.mockImplementation(async (name: string, rows: any[], opts?: { mode?: string }) => { + if (opts?.mode === 'overwrite') throw new Error('disk full'); + return create(name, rows, opts); + }); await expect(reembedMemoryTable({ memoryDir: dir })).rejects.toThrow('disk full'); @@ -107,8 +159,8 @@ describe('reembedMemoryTable', () => { }); it('records the embedding signature once the swap succeeded', async () => { - state.rows = [row()]; - const dir = mkdtempSync(resolve(tmpdir(), 'osw-re-')); + state.tables.set(LIVE, [row()]); + const dir = tmp(); const result = await reembedMemoryTable({ memoryDir: dir }); @@ -119,26 +171,58 @@ describe('reembedMemoryTable', () => { }); it('reports progress at the configured interval', async () => { - state.rows = [row(), row(), row(), row()]; + state.tables.set(LIVE, [row(), row(), row(), row()]); const onProgress = vi.fn(); - await reembedMemoryTable({ - memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')), - onProgress, - progressEvery: 2, - }); + await reembedMemoryTable({ memoryDir: tmp(), onProgress, progressEvery: 2 }); expect(onProgress).toHaveBeenCalledWith(2, 4); expect(onProgress).toHaveBeenLastCalledWith(4, 4); }); it('handles an empty store without creating a data table', async () => { - state.rows = []; - - const result = await reembedMemoryTable({ memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')) }); + const result = await reembedMemoryTable({ memoryDir: tmp() }); expect(result.total).toBe(0); expect(state.createEmptyTable).toHaveBeenCalled(); expect(state.createTable).not.toHaveBeenCalled(); }); + + it('re-embeds a store larger than the batch size without reading more than a batch at once', async () => { + const source = Array.from({ length: 23 }, (_, i) => row({ id: `r${i}`, title: `T${i}` })); + state.tables.set(LIVE, source); + + const result = await reembedMemoryTable({ memoryDir: tmp(), batchSize: 5 }); + + expect(result.total).toBe(23); + expect(result.reembedded).toBe(23); + const live = state.tables.get(LIVE)!; + expect(live).toHaveLength(23); + expect(live.map((r) => r.id)).toEqual(source.map((r) => r.id)); + expect(live.every((r) => r.vector.length === 4 && r.vector[0] === 1)).toBe(true); + for (const [, max] of state.maxPageRead) expect(max).toBeLessThanOrEqual(5); + // Every write is a batch, never the whole table. + for (const call of state.createTable.mock.calls) expect(call[1].length).toBeLessThanOrEqual(5); + }); + + it('refuses to swap and keeps the live table when the source row count changes mid-run', async () => { + const source = Array.from({ length: 12 }, (_, i) => row({ id: `r${i}` })); + state.tables.set(LIVE, source); + const dir = tmp(); + // A row lands in the live table while the batches are being read: the + // count taken up front no longer matches what was read. + let reads = 0; + embedMock().mockImplementation(async (text: string) => { + state.embedCalls.push(text); + if (++reads === 1) state.tables.get(LIVE)!.push(row({ id: 'late' })); + return [1, 2, 3, 4]; + }); + + await expect(reembedMemoryTable({ memoryDir: dir, batchSize: 5 })).rejects.toThrow(/Re-embed refused/); + + expect(state.createTable.mock.calls.some((c) => c[0] === LIVE)).toBe(false); + expect(state.tables.get(LIVE)!.every((r) => r.vector === undefined)).toBe(true); + expect([...state.tables.keys()]).toEqual([LIVE]); // incomplete temp table dropped + expect(existsSync(resolve(dir, 'embedding.json'))).toBe(false); + }); }); diff --git a/src/memory/reembed.ts b/src/memory/reembed.ts index c37ddf49..1f031b6f 100644 --- a/src/memory/reembed.ts +++ b/src/memory/reembed.ts @@ -6,8 +6,11 @@ // dimension, passage/query prefixes, or the runtime that executes the ONNX // graph). Vectors from two different encoders coexist silently: writes succeed // and searches return rows, only the ranking is wrong. This rewrites the whole -// table in one pass, following compaction's build-then-swap shape so a failure -// leaves the original table intact. +// table batch by batch, following compaction's build-then-swap shape: a failure +// while building leaves the live table untouched. The swap itself is an +// overwrite followed by appends, so a failure during it can leave the live +// table partly written — the complete replacement is then kept as +// `_reembed_` for recovery. import { join } from 'node:path'; import { withFileLock } from '../support/fileLock.js'; @@ -44,8 +47,13 @@ export interface ReembedOptions { /** Progress callback, invoked every `progressEvery` records. */ onProgress?: (done: number, total: number) => void; progressEvery?: number; + /** Rows read, encoded and written per batch. Defaults to REEMBED_BATCH_SIZE. */ + batchSize?: number; } +/** Rows held in memory at once while re-embedding. */ +export const REEMBED_BATCH_SIZE = 1_000; + export async function reembedMemoryTable(options: ReembedOptions = {}): Promise { const memoryDir = options.memoryDir ?? MEMORY_DIR; const lockPath = join(memoryDir, '.reembed.lock'); @@ -60,52 +68,106 @@ export async function reembedMemoryTable(options: ReembedOptions = {}): Promise< const signature = embeddingSignature(spec); const progressEvery = options.progressEvery ?? 50; - const rows = (await table.query().limit(1_000_000).toArray()) as unknown as CognitiveMemoryRecord[]; - rows.sort((a, b) => String(a.id).localeCompare(String(b.id))); - const total = rows.length; + const batchSize = Math.max(1, Math.floor(options.batchSize ?? REEMBED_BATCH_SIZE)); + const total = await table.countRows(); console.log(`${status.info('[Reembed]')} ${c.dim('rebuilding')} ${c.cyan(String(total))} ${c.dim('vectors with')} ${c.yellow(spec.id)}`); - // normalizeRecords first so the rewritten table lands on the lean v3 schema, - // exactly like compaction does; vectors are replaced immediately after. - const normalized = normalizeRecords(rows); + const targetTableName = table.name; + const tempTableName = `${targetTableName}_reembed_${Date.now()}`; + + // Build the replacement batch by batch: read `batchSize` rows, encode them, + // append them to the temp table, and let them go. Holding the whole table in + // memory exhausted large stores, and a fixed `.limit(1_000_000)` silently + // truncated anything bigger. + let tempTable: Awaited> | null = null; + let read = 0; let reembedded = 0; let empty = 0; + let built = false; + try { + for (;;) { + const page = (await table.query().offset(read).limit(batchSize).toArray()) as unknown as CognitiveMemoryRecord[]; + if (page.length === 0) break; + // normalizeRecords first so the rewritten table lands on the lean v3 schema, + // exactly like compaction does; vectors are replaced immediately after. + const normalized = normalizeRecords(page); + for (const record of normalized) { + const text = embeddingTextFor(String(record.title ?? ''), String(record.content ?? '')); + if (!text) { + record.vector = Array.from({ length: EMBEDDING_DIM }, () => 0); + empty++; + } else { + record.vector = await embedPassage(text); + reembedded++; + } + const done = reembedded + empty; + if (done % progressEvery === 0) { + options.onProgress?.(done, total); + console.log(`${c.dim(`[Reembed] ${done}/${total}`)}`); + } + } + if (tempTable) { + await tempTable.add(normalized); + } else { + tempTable = await db.createTable(tempTableName, normalized); + } + read += page.length; + if (page.length < batchSize) break; + } + if (!tempTable) { + tempTable = await db.createEmptyTable(tempTableName, await table.schema()); + } - for (let i = 0; i < normalized.length; i++) { - const record = normalized[i]; - const text = embeddingTextFor(String(record.title ?? ''), String(record.content ?? '')); - if (!text) { - record.vector = Array.from({ length: EMBEDDING_DIM }, () => 0); - empty++; - } else { - record.vector = await embedPassage(text); - reembedded++; + // The store changed under us, or a page went missing: the replacement is + // not the table we meant to rewrite, so the live table stays as it is. + const builtRows = await tempTable.countRows(); + if (read !== total || builtRows !== total) { + throw new Error( + `Re-embed refused: source has ${total} rows but ${read} were read and ${builtRows} built; live table left unchanged`, + ); } - if ((i + 1) % progressEvery === 0) { - options.onProgress?.(i + 1, total); - console.log(`${c.dim(`[Reembed] ${i + 1}/${total}`)}`); + built = true; + } finally { + if (!built && tempTable) { + try { + await db.dropTable(tempTableName); + } catch (cleanupError) { + console.warn(`[Reembed] Failed to drop incomplete temporary table ${tempTableName}:`, cleanupError); + } } } options.onProgress?.(total, total); + if (!tempTable) throw new Error('Re-embed did not build a replacement table'); + const replacement = tempTable; - const targetTableName = table.name; - const tempTableName = `${targetTableName}_reembed_${Date.now()}`; - - // Build a validated replacement before touching the live table. - if (normalized.length > 0) { - await db.createTable(tempTableName, normalized); - } else { - await db.createEmptyTable(tempTableName, await table.schema()); - } - + // Copy the validated replacement over the live table, again batch by batch. + // If this fails partway the temp table is kept so the vectors are recoverable. let replaced = false; try { - if (normalized.length > 0) { - await db.createTable(targetTableName, normalized, { mode: 'overwrite' }); - } else { + if (total === 0) { await db.createEmptyTable(targetTableName, await table.schema(), { mode: 'overwrite' }); + } else { + let target: Awaited> | null = null; + for (let copied = 0; copied < total; ) { + // Rows read back from Lance carry Arrow vectors; writing them as-is + // zeroed embeddings once already (see compaction.store.test.ts), so + // the copy goes through the same normalisation as the build. + const page = normalizeRecords(await replacement.query().offset(copied).limit(batchSize).toArray()); + if (page.length === 0) break; + if (target) { + await target.add(page); + } else { + target = await db.createTable(targetTableName, page, { mode: 'overwrite' }); + } + copied += page.length; + } + } + const swapped = await db.openTable(targetTableName); + const swappedRows = await swapped.countRows(); + if (swappedRows !== total) { + throw new Error(`Re-embed swap wrote ${swappedRows} rows, expected ${total}`); } - setTable(await db.openTable(targetTableName)); + setTable(swapped); replaced = true; } finally { if (replaced) {