Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions src/memory/reembed.store.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import { mkdtempSync } from 'node:fs';
import { mkdir, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterAll, describe, expect, it, vi } from 'vitest';

// Same module-load boundary as compaction.store.test.ts: memoryCore resolves
// MEMORY_DIR from homedir() at import time, so HOME is redirected first and
// everything is imported dynamically. Only the encoder is replaced — loading
// the ONNX model is not what this test is about — the table is a real LanceDB
// store, so the batch reads, appends and the swap run against real Arrow rows.
const home = mkdtempSync(join(tmpdir(), 'openswarm-reembed-store-'));
process.env.HOME = home;

vi.mock('./memoryCore.js', async (importOriginal) => {
const actual = await importOriginal<typeof import('./memoryCore.js')>();
return {
...actual,
embedPassage: async (text: string) =>
Array.from({ length: actual.EMBEDDING_DIM }, (_, d) => ((text.length + d) % 7) / 7 + 0.01),
};
});

const { connect } = await import('@lancedb/lancedb');
const { EMBEDDING_DIM, PERMANENT_EXPIRY } = await import('./memoryCore.js');
const { reembedMemoryTable } = await import('./reembed.js');

const memoryDir = join(home, '.openswarm', 'memory');

afterAll(async () => {
await rm(home, { recursive: true, force: true });
});

function storedRecord(i: number) {
const now = 1_700_000_000_000;
return {
id: `row-${i}`,
type: 'belief',
repo: 'alpha',
derivedFrom: 'src',
metadata: '{}',
title: `title ${i}`,
content: `content ${i}`,
vector: Array.from({ length: EMBEDDING_DIM }, () => 0),
importance: 0.5,
confidence: 0.7,
createdAt: now,
lastUpdated: now,
lastAccessed: now,
trust: 1,
expiresAt: PERMANENT_EXPIRY,
};
}

describe('reembedMemoryTable over a stored table', () => {
it('rewrites a store larger than the batch size with every row and a real vector', async () => {
await mkdir(memoryDir, { recursive: true });
const db = await connect(memoryDir);
const total = 7;
await db.createTable('cognitive_memory', Array.from({ length: total }, (_, i) => storedRecord(i)), { mode: 'overwrite' });
const before = await db.openTable('cognitive_memory');
const vectorTypeBefore = String((await before.schema()).fields.find((f) => f.name === 'vector')?.type);

const result = await reembedMemoryTable({ memoryDir, batchSize: 3 });

expect(result.total).toBe(total);
expect(result.reembedded).toBe(total);
const after = await db.openTable('cognitive_memory');
expect(await after.countRows()).toBe(total);
expect(String((await after.schema()).fields.find((f) => f.name === 'vector')?.type)).toBe(vectorTypeBefore);
const rows = await after.query().limit(total + 1).toArray();
expect(rows.map((r) => String(r.id)).sort()).toEqual(Array.from({ length: total }, (_, i) => `row-${i}`).sort());
for (const row of rows) {
const v = Array.from(row.vector as Iterable<unknown>, Number);
expect(v).toHaveLength(EMBEDDING_DIM);
// The fixture stored zero vectors; every row must now carry the new one.
expect(v.some((x) => x !== 0)).toBe(true);
}
// The temp table is gone once the swap succeeded.
expect((await db.tableNames()).filter((n) => n.includes('_reembed_'))).toEqual([]);
}, 120_000);
});
166 changes: 125 additions & 41 deletions src/memory/reembed.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,64 @@ import { resolve } from 'node:path';
import { describe, expect, it, vi, beforeEach } from 'vitest';

// memoryCore owns the LanceDB handles and the encoder; reembed is pure orchestration
// over them, so the seam is mocked rather than standing up a real table.
// over them, so the seam is replaced by a small in-memory table store that honours
// offset/limit paging, add, countRows and overwrite.
const LIVE = 'cognitive_memory';
const state = {
rows: [] as any[],
createTable: vi.fn(async (_name: string, _rows: any[], _opts?: any) => ({})),
createEmptyTable: vi.fn(async () => ({})),
dropTable: vi.fn(async () => {}),
openTable: vi.fn(async () => ({ name: 'cognitive_memory' })),
tables: new Map<string, any[]>(),
/** Largest page any query asked for or returned, per table. */
maxPageRead: new Map<string, number>(),
createTable: vi.fn(),
createEmptyTable: vi.fn(),
dropTable: vi.fn(),
openTable: vi.fn(),
setTable: vi.fn(),
embedCalls: [] as string[],
};

function fakeTable(name: string) {
return {
name,
schema: async () => ({ fields: [] }),
countRows: async () => state.tables.get(name)!.length,
add: async (rows: any[]) => {
state.tables.get(name)!.push(...rows.map((r) => ({ ...r })));
return {};
},
query: () => {
let offset = 0;
let limit = Infinity;
const q = {
offset: (n: number) => { offset = n; return q; },
limit: (n: number) => { limit = n; return q; },
toArray: async () => {
const page = state.tables.get(name)!.slice(offset, offset + limit);
state.maxPageRead.set(name, Math.max(state.maxPageRead.get(name) ?? 0, page.length, limit));
return page.map((r) => ({ ...r }));
},
};
return q;
},
};
}

function resetDb() {
state.tables = new Map([[LIVE, []]]);
state.maxPageRead = new Map();
state.createTable.mockReset().mockImplementation(async (name: string, rows: any[], opts?: { mode?: string }) => {
if (state.tables.has(name) && opts?.mode !== 'overwrite') throw new Error(`table ${name} exists`);
state.tables.set(name, rows.map((r) => ({ ...r })));
return fakeTable(name);
});
state.createEmptyTable.mockReset().mockImplementation(async (name: string) => {
state.tables.set(name, []);
return fakeTable(name);
});
state.dropTable.mockReset().mockImplementation(async (name: string) => { state.tables.delete(name); });
state.openTable.mockReset().mockImplementation(async (name: string) => fakeTable(name));
state.setTable.mockReset();
}

vi.mock('./memoryCore.js', () => ({
EMBEDDING_DIM: 4,
MEMORY_DIR: '/tmp/does-not-matter',
Expand All @@ -25,18 +72,14 @@ vi.mock('./memoryCore.js', () => ({
dropTable: state.dropTable,
openTable: state.openTable,
}),
getTable: () => ({
name: 'cognitive_memory',
schema: async () => ({ fields: [] }),
query: () => ({ limit: () => ({ toArray: async () => state.rows }) }),
}),
getTable: () => fakeTable(LIVE),
setTable: state.setTable,
embedPassage: vi.fn(async (text: string) => {
state.embedCalls.push(text);
return [1, 2, 3, 4];
}),
normalizeRecords: (rows: any[]) =>
rows.map((r) => ({ ...r, title: String(r.title ?? ''), content: String(r.content ?? ''), vector: [] })),
rows.map((r) => ({ ...r, title: String(r.title ?? ''), content: String(r.content ?? ''), vector: r.vector ?? [] })),
}));

const { reembedMemoryTable } = await import('./reembed.js');
Expand All @@ -45,31 +88,37 @@ function row(over: Record<string, unknown> = {}) {
return { id: 'x', type: 'belief', title: 'T', content: 'C', ...over };
}

const tmp = () => mkdtempSync(resolve(tmpdir(), 'osw-re-'));

const { embedPassage } = await import('./memoryCore.js');
const embedMock = () => embedPassage as unknown as ReturnType<typeof vi.fn>;

beforeEach(() => {
state.rows = [];
resetDb();
state.embedCalls = [];
state.createTable.mockClear().mockImplementation(async () => ({}));
state.createEmptyTable.mockClear();
state.dropTable.mockClear();
state.setTable.mockClear();
embedMock().mockImplementation(async (text: string) => {
state.embedCalls.push(text);
return [1, 2, 3, 4];
});
});

describe('reembedMemoryTable', () => {
it('re-encodes every record and reports the counts', async () => {
state.rows = [row({ id: 'a' }), row({ id: 'b', title: 'T2', content: 'C2' })];
state.tables.set(LIVE, [row({ id: 'a' }), row({ id: 'b', title: 'T2', content: 'C2' })]);

const result = await reembedMemoryTable({ memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')) });
const result = await reembedMemoryTable({ memoryDir: tmp() });

expect(result.total).toBe(2);
expect(result.reembedded).toBe(2);
expect(result.empty).toBe(0);
expect(state.embedCalls).toEqual(['T\nC', 'T2\nC2']);
expect(state.tables.get(LIVE)!.map((r) => r.vector)).toEqual([[1, 2, 3, 4], [1, 2, 3, 4]]);
});

it('writes a zero vector for a record with no text instead of calling the encoder', async () => {
state.rows = [row({ id: 'blank', title: '', content: '' })];
state.tables.set(LIVE, [row({ id: 'blank', title: '', content: '' })]);

const result = await reembedMemoryTable({ memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')) });
const result = await reembedMemoryTable({ memoryDir: tmp() });

expect(result.empty).toBe(1);
expect(result.reembedded).toBe(0);
Expand All @@ -79,25 +128,28 @@ describe('reembedMemoryTable', () => {
});

it('builds a temp table before overwriting the live one, then drops it', async () => {
state.rows = [row()];
state.tables.set(LIVE, [row()]);

await reembedMemoryTable({ memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')) });
await reembedMemoryTable({ memoryDir: tmp() });

const names = state.createTable.mock.calls.map((c) => c[0]);
expect(names[0]).toMatch(/^cognitive_memory_reembed_/); // validated replacement first
expect(names[1]).toBe('cognitive_memory');
expect(names[1]).toBe(LIVE);
expect(state.createTable.mock.calls[1][2]).toEqual({ mode: 'overwrite' });
expect(state.dropTable).toHaveBeenCalledWith(names[0]);
expect(state.tables.has(names[0])).toBe(false);
expect(state.setTable).toHaveBeenCalled();
});

it('keeps the recoverable temp table and does not claim the signature when the swap fails', async () => {
state.rows = [row()];
const dir = mkdtempSync(resolve(tmpdir(), 'osw-re-'));
// First call builds the temp table; the overwrite of the live table fails.
state.createTable
.mockImplementationOnce(async () => ({}))
.mockImplementationOnce(async () => { throw new Error('disk full'); });
state.tables.set(LIVE, [row()]);
const dir = tmp();
const create = state.createTable.getMockImplementation()!;
// Building the temp table works; the overwrite of the live table fails.
state.createTable.mockImplementation(async (name: string, rows: any[], opts?: { mode?: string }) => {
if (opts?.mode === 'overwrite') throw new Error('disk full');
return create(name, rows, opts);
});

await expect(reembedMemoryTable({ memoryDir: dir })).rejects.toThrow('disk full');

Expand All @@ -107,8 +159,8 @@ describe('reembedMemoryTable', () => {
});

it('records the embedding signature once the swap succeeded', async () => {
state.rows = [row()];
const dir = mkdtempSync(resolve(tmpdir(), 'osw-re-'));
state.tables.set(LIVE, [row()]);
const dir = tmp();

const result = await reembedMemoryTable({ memoryDir: dir });

Expand All @@ -119,26 +171,58 @@ describe('reembedMemoryTable', () => {
});

it('reports progress at the configured interval', async () => {
state.rows = [row(), row(), row(), row()];
state.tables.set(LIVE, [row(), row(), row(), row()]);
const onProgress = vi.fn();

await reembedMemoryTable({
memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')),
onProgress,
progressEvery: 2,
});
await reembedMemoryTable({ memoryDir: tmp(), onProgress, progressEvery: 2 });

expect(onProgress).toHaveBeenCalledWith(2, 4);
expect(onProgress).toHaveBeenLastCalledWith(4, 4);
});

it('handles an empty store without creating a data table', async () => {
state.rows = [];

const result = await reembedMemoryTable({ memoryDir: mkdtempSync(resolve(tmpdir(), 'osw-re-')) });
const result = await reembedMemoryTable({ memoryDir: tmp() });

expect(result.total).toBe(0);
expect(state.createEmptyTable).toHaveBeenCalled();
expect(state.createTable).not.toHaveBeenCalled();
});

it('re-embeds a store larger than the batch size without reading more than a batch at once', async () => {
const source = Array.from({ length: 23 }, (_, i) => row({ id: `r${i}`, title: `T${i}` }));
state.tables.set(LIVE, source);

const result = await reembedMemoryTable({ memoryDir: tmp(), batchSize: 5 });

expect(result.total).toBe(23);
expect(result.reembedded).toBe(23);
const live = state.tables.get(LIVE)!;
expect(live).toHaveLength(23);
expect(live.map((r) => r.id)).toEqual(source.map((r) => r.id));
expect(live.every((r) => r.vector.length === 4 && r.vector[0] === 1)).toBe(true);
for (const [, max] of state.maxPageRead) expect(max).toBeLessThanOrEqual(5);
// Every write is a batch, never the whole table.
for (const call of state.createTable.mock.calls) expect(call[1].length).toBeLessThanOrEqual(5);
});

it('refuses to swap and keeps the live table when the source row count changes mid-run', async () => {
const source = Array.from({ length: 12 }, (_, i) => row({ id: `r${i}` }));
state.tables.set(LIVE, source);
const dir = tmp();
// A row lands in the live table while the batches are being read: the
// count taken up front no longer matches what was read.
let reads = 0;
embedMock().mockImplementation(async (text: string) => {
state.embedCalls.push(text);
if (++reads === 1) state.tables.get(LIVE)!.push(row({ id: 'late' }));
return [1, 2, 3, 4];
});

await expect(reembedMemoryTable({ memoryDir: dir, batchSize: 5 })).rejects.toThrow(/Re-embed refused/);

expect(state.createTable.mock.calls.some((c) => c[0] === LIVE)).toBe(false);
expect(state.tables.get(LIVE)!.every((r) => r.vector === undefined)).toBe(true);
expect([...state.tables.keys()]).toEqual([LIVE]); // incomplete temp table dropped
expect(existsSync(resolve(dir, 'embedding.json'))).toBe(false);
});
});
Loading
Loading