From c14f42ec377d207389dc7e843cf270235117bba6 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Mon, 13 Jul 2026 22:40:10 +0530 Subject: [PATCH 1/3] ANG-008:Similarity computation, edge creation & SQLite cache --- src/data/Database/VectorDatabase.ts | 78 +++ src/data/Database/VectorRepository.test.ts | 125 +++++ src/data/Database/VectorRepository.ts | 121 ++++ src/services/embeddings/Orchestrator.test.ts | 157 +++++- src/services/embeddings/Orchestrator.ts | 86 ++- .../embeddings/ProviderResolver.test.ts | 27 +- src/services/embeddings/ProviderResolver.ts | 13 +- .../Providers/JoplinNativeProvider.test.ts | 42 +- .../Providers/JoplinNativeProvider.ts | 113 ++-- src/services/graph/GraphBuilder.test.ts | 44 ++ src/services/graph/GraphBuilder.ts | 65 ++- src/services/similarity/EdgeFactory.test.ts | 21 + src/services/similarity/EdgeFactory.ts | 58 +- .../similarity/SimilarityEngine.test.ts | 515 ++++++++++++++++++ src/services/similarity/SimilarityEngine.ts | 316 +++++++++++ src/services/similarity/ThresholdPresets.ts | 26 + 16 files changed, 1738 insertions(+), 69 deletions(-) create mode 100644 src/data/Database/VectorDatabase.ts create mode 100644 src/data/Database/VectorRepository.test.ts create mode 100644 src/data/Database/VectorRepository.ts create mode 100644 src/services/similarity/SimilarityEngine.test.ts create mode 100644 src/services/similarity/SimilarityEngine.ts create mode 100644 src/services/similarity/ThresholdPresets.ts diff --git a/src/data/Database/VectorDatabase.ts b/src/data/Database/VectorDatabase.ts new file mode 100644 index 0000000..c6780f4 --- /dev/null +++ b/src/data/Database/VectorDatabase.ts @@ -0,0 +1,78 @@ +import joplin from 'api'; + +export interface IVectorDatabase { + open(): Promise; + run(sql: string, params: unknown[]): Promise; + all(sql: string, params: unknown[]): Promise; +} + +interface Sqlite3Database { + run(sql: string, params: unknown[], callback: (err: Error | null) => void): void; + all(sql: string, params: unknown[], callback: (err: Error | null, rows: unknown[]) => void): void; +} + +/** + * Thin promisified wrapper around Joplin's bundled sqlite3 module (accessed via + * `joplin.require('sqlite3')`, since native packages can't be bundled with a + * plugin). Owns only the connection and schema; query logic lives in + * VectorRepository. + */ +export class VectorDatabase implements IVectorDatabase { + private static readonly DB_FILE_NAME = 'note-graph-vectors.sqlite'; + private static readonly SCHEMA = ` + CREATE TABLE IF NOT EXISTS note_vectors ( + note_id TEXT PRIMARY KEY, + model_id TEXT NOT NULL, + updated_time INTEGER NOT NULL, + vector BLOB NOT NULL + ) + `; + + private db: Sqlite3Database | null = null; + private opening: Promise | null = null; + + /** Opens (creating if needed) the vector cache database. Safe to call repeatedly. */ + public async open(): Promise { + if (this.db) return; + if (!this.opening) { + this.opening = this.openInternal(); + } + await this.opening; + } + + public async run(sql: string, params: unknown[]): Promise { + const db = this.requireDb(); + await new Promise((resolve, reject) => { + db.run(sql, params, (err) => (err ? reject(err) : resolve())); + }); + } + + public async all(sql: string, params: unknown[]): Promise { + const db = this.requireDb(); + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => (err ? reject(err) : resolve(rows as T[]))); + }); + } + + private async openInternal(): Promise { + const sqlite3 = joplin.require('sqlite3'); + const dataDir = await joplin.plugins.dataDir(); + const dbPath = `${dataDir}/${VectorDatabase.DB_FILE_NAME}`; + + this.db = await new Promise((resolve, reject) => { + const db = new sqlite3.Database(dbPath, (err: Error | null) => { + if (err) reject(err); + else resolve(db); + }); + }); + + await this.run(VectorDatabase.SCHEMA, []); + } + + private requireDb(): Sqlite3Database { + if (!this.db) { + throw new Error('VectorDatabase used before open() completed.'); + } + return this.db; + } +} diff --git a/src/data/Database/VectorRepository.test.ts b/src/data/Database/VectorRepository.test.ts new file mode 100644 index 0000000..4b50115 --- /dev/null +++ b/src/data/Database/VectorRepository.test.ts @@ -0,0 +1,125 @@ +import { VectorRepository } from './VectorRepository'; +import { IVectorDatabase } from './VectorDatabase'; + +/** + * In-memory stand-in for VectorDatabase. sqlite3 is only reachable at runtime + * via joplin.require(), so VectorRepository is tested against this fake + * rather than a real database; it emulates the single upsert statement and + * the `note_id IN (...)` select that VectorRepository issues. + */ +class FakeVectorDatabase implements IVectorDatabase { + public opened = false; + public allCallBatchSizes: number[] = []; + private rows = new Map(); + + public async open(): Promise { + this.opened = true; + } + + public async run(_sql: string, params: unknown[]): Promise { + if (params.length === 0) { + return; // BEGIN TRANSACTION / COMMIT / ROLLBACK + } + const [noteId, modelId, updatedTime, vector] = params as [string, string, number, Buffer]; + this.rows.set(noteId, { note_id: noteId, model_id: modelId, updated_time: updatedTime, vector }); + } + + public async all(_sql: string, params: unknown[]): Promise { + const ids = params as string[]; + this.allCallBatchSizes.push(ids.length); + const found = ids.map(id => this.rows.get(id)).filter((r): r is NonNullable => !!r); + return found as unknown as T[]; + } +} + +describe('VectorRepository', () => { + let db: FakeVectorDatabase; + let repo: VectorRepository; + + beforeEach(() => { + db = new FakeVectorDatabase(); + repo = new VectorRepository(db); + }); + + describe('getMany', () => { + it('returns an empty map without opening the database for no IDs', async () => { + const result = await repo.getMany([]); + expect(result.size).toBe(0); + expect(db.opened).toBe(false); + }); + + it('returns nothing for IDs that were never saved', async () => { + const result = await repo.getMany(['missing']); + expect(result.size).toBe(0); + expect(db.opened).toBe(true); + }); + }); + + describe('saveMany + getMany round trip', () => { + it('round-trips vector values through the Float32 BLOB encoding', async () => { + const vector = [0.1, -0.25, 0.987654, 1, -1, 0]; + await repo.saveMany([{ noteId: 'n1', vector, modelId: 'm1', updatedTime: 100 }]); + + const result = await repo.getMany(['n1']); + const entry = result.get('n1'); + + expect(entry).toBeDefined(); + expect(entry!.modelId).toBe('m1'); + expect(entry!.updatedTime).toBe(100); + expect(entry!.vector).toHaveLength(vector.length); + for (let i = 0; i < vector.length; i++) { + // Float32 storage loses some precision relative to the JS float64 input. + expect(entry!.vector[i]).toBeCloseTo(vector[i], 5); + } + }); + + it('overwrites the previous entry for the same note ID', async () => { + await repo.saveMany([{ noteId: 'n1', vector: [1, 0], modelId: 'm1', updatedTime: 100 }]); + await repo.saveMany([{ noteId: 'n1', vector: [0, 1], modelId: 'm2', updatedTime: 200 }]); + + const result = await repo.getMany(['n1']); + const entry = result.get('n1'); + + expect(entry!.modelId).toBe('m2'); + expect(entry!.updatedTime).toBe(200); + expect(entry!.vector[0]).toBeCloseTo(0, 5); + expect(entry!.vector[1]).toBeCloseTo(1, 5); + }); + + it('only returns entries for the requested IDs that exist', async () => { + await repo.saveMany([ + { noteId: 'n1', vector: [1, 0], modelId: 'm1', updatedTime: 100 }, + { noteId: 'n2', vector: [0, 1], modelId: 'm1', updatedTime: 100 }, + ]); + + const result = await repo.getMany(['n1', 'n3']); + + expect(result.has('n1')).toBe(true); + expect(result.has('n2')).toBe(false); + expect(result.has('n3')).toBe(false); + }); + + it('does nothing for an empty entries array', async () => { + await repo.saveMany([]); + expect(db.opened).toBe(false); + }); + }); + + describe('large vaults', () => { + it('chunks getMany so a single query never exceeds SQLite\'s bound-parameter limit', async () => { + const noteIds = Array.from({ length: 1200 }, (_, i) => `n${i}`); + await repo.saveMany( + noteIds.map(id => ({ noteId: id, vector: [1, 0], modelId: 'm1', updatedTime: 1 })), + ); + + const result = await repo.getMany(noteIds); + + expect(result.size).toBe(1200); + expect(db.allCallBatchSizes.length).toBeGreaterThan(1); + for (const size of db.allCallBatchSizes) { + expect(size).toBeLessThanOrEqual(500); + } + expect(db.allCallBatchSizes.reduce((a, b) => a + b, 0)).toBe(1200); + }); + }); +}); diff --git a/src/data/Database/VectorRepository.ts b/src/data/Database/VectorRepository.ts new file mode 100644 index 0000000..4757521 --- /dev/null +++ b/src/data/Database/VectorRepository.ts @@ -0,0 +1,121 @@ +import { IVectorDatabase, VectorDatabase } from './VectorDatabase'; + +export interface CachedVector { + vector: number[]; + modelId: string; + updatedTime: number; +} + +export interface VectorCacheEntry { + noteId: string; + vector: number[]; + modelId: string; + updatedTime: number; +} + +export interface VectorCache { + getMany(noteIds: string[]): Promise>; + saveMany(entries: VectorCacheEntry[]): Promise; +} + +interface VectorRow { + note_id: string; + model_id: string; + updated_time: number; + vector: Buffer; +} + +/** + * Persists note embedding vectors in SQLite so unchanged notes aren't + * re-fetched from joplin.ai.getEmbeddings(). A cached vector is only reused + * when both its note_id and model_id match, since staleness is decided by + * the caller comparing `updatedTime` against the note's current updated_time. + */ +export class VectorRepository implements VectorCache { + /** SQLite caps bound parameters per statement (as low as 999 on some builds); stay well under it. */ + private static readonly QUERY_BATCH_SIZE = 500; + + public constructor(private readonly db: IVectorDatabase = new VectorDatabase()) {} + + /** Returns cached vectors for the given note IDs, keyed by note ID. Missing notes are omitted. */ + public async getMany(noteIds: string[]): Promise> { + if (noteIds.length === 0) { + return new Map(); + } + + await this.db.open(); + + const result = new Map(); + for (const batch of this.chunk(noteIds, VectorRepository.QUERY_BATCH_SIZE)) { + const rows = await this.queryBatch(batch); + for (const row of rows) { + result.set(row.note_id, { + vector: this.decodeVector(row.vector), + modelId: row.model_id, + updatedTime: row.updated_time, + }); + } + } + return result; + } + + /** Inserts or updates vectors for the given notes, in a single transaction. */ + public async saveMany(entries: VectorCacheEntry[]): Promise { + if (entries.length === 0) { + return; + } + + await this.db.open(); + + await this.db.run('BEGIN TRANSACTION', []); + try { + for (const entry of entries) { + await this.db.run( + `INSERT INTO note_vectors (note_id, model_id, updated_time, vector) + VALUES (?, ?, ?, ?) + ON CONFLICT(note_id) DO UPDATE SET + model_id = excluded.model_id, + updated_time = excluded.updated_time, + vector = excluded.vector`, + [entry.noteId, entry.modelId, entry.updatedTime, this.encodeVector(entry.vector)], + ); + } + await this.db.run('COMMIT', []); + } catch (e) { + await this.db.run('ROLLBACK', []); + throw e; + } + } + + private async queryBatch(noteIds: string[]): Promise { + const placeholders = noteIds.map(() => '?').join(','); + return this.db.all( + `SELECT note_id, model_id, updated_time, vector FROM note_vectors WHERE note_id IN (${placeholders})`, + noteIds, + ); + } + + private chunk(items: T[], size: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += size) { + batches.push(items.slice(i, i + size)); + } + return batches; + } + + /** Encodes a vector as a Float32 BLOB for compact SQLite storage. */ + private encodeVector(vector: number[]): Buffer { + const floats = Float32Array.from(vector); + return Buffer.from(floats.buffer, floats.byteOffset, floats.byteLength); + } + + /** Decodes a Float32 BLOB back into a plain number array. */ + private decodeVector(blob: Buffer): number[] { + const floats = new Float32Array( + blob.buffer, + blob.byteOffset, + blob.byteLength / Float32Array.BYTES_PER_ELEMENT, + ); + return Array.from(floats); + } +} diff --git a/src/services/embeddings/Orchestrator.test.ts b/src/services/embeddings/Orchestrator.test.ts index d89e820..366d4c6 100644 --- a/src/services/embeddings/Orchestrator.test.ts +++ b/src/services/embeddings/Orchestrator.test.ts @@ -2,14 +2,14 @@ import { EmbeddingOrchestrator } from './Orchestrator'; import { BatchProgress } from './Types'; import { Note } from '../../data/Types'; -function makeNote(id: string, title: string, body: string): Note { +function makeNote(id: string, title: string, body: string, updatedTime = 0): Note { return { id, parent_id: 'p1', title, body, created_time: 0, - updated_time: 0, + updated_time: updatedTime, }; } @@ -128,4 +128,157 @@ describe('EmbeddingOrchestrator', () => { expect(result.errors[0].error).toBe('API failure'); }); }); + + describe('vector caching', () => { + it('serves an unchanged, same-model note from the cache without calling the provider', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 50)]); + + expect(fetchVectorsByNoteIds).not.toHaveBeenCalled(); + expect(result.embeddedNotes).toHaveLength(1); + expect(result.embeddedNotes[0].embedding).toEqual([0.1, 0.2]); + }); + + it('re-fetches a note whose updated_time no longer matches the cached entry', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 999)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes[0].embedding).toEqual([0.9, 0.9]); + }); + + it('does not fall back to a stale cached vector when the re-fetch omits the note', async () => { + // The note changed (updated_time no longer matches), so it's correctly + // queued for re-fetch — but Joplin's AI index hasn't caught up yet and + // returns nothing for it. The stale pre-edit vector must not be used. + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 999)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes).toHaveLength(0); + expect(result.errors).toEqual([{ noteId: 'n1', error: 'Note not yet indexed by Joplin AI.' }]); + }); + + it('re-fetches a note whose cached entry belongs to a different embedding model', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm2', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 50)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes[0].embedding).toEqual([0.9, 0.9]); + }); + + it('only asks the provider for stale/missing notes, mixing in cache hits', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n2', [0.4, 0.5]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 10 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([ + makeNote('n1', 'T1', 'B1', 10), + makeNote('n2', 'T2', 'B2', 20), + ]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n2']); + expect(result.embeddedNotes).toHaveLength(2); + expect(result.errors).toHaveLength(0); + }); + + it('saves freshly fetched vectors back to the cache with the note updated_time and model', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + const saveMany = jest.fn().mockResolvedValue(undefined); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany }); + + await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(saveMany).toHaveBeenCalledWith([ + { noteId: 'n1', vector: [0.4, 0.5], modelId: 'm1', updatedTime: 42 }, + ]); + }); + + it('does not save notes the provider failed to return', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); + const saveMany = jest.fn().mockResolvedValue(undefined); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany }); + + await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(saveMany).not.toHaveBeenCalled(); + }); + + it('falls back to a full fetch when the cache read throws', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockRejectedValue(new Error('disk error')), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes[0].embedding).toEqual([0.4, 0.5]); + }); + + it('still returns results when the cache write throws', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue(new Map()), + saveMany: jest.fn().mockRejectedValue(new Error('disk full')), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(result.embeddedNotes[0].embedding).toEqual([0.4, 0.5]); + expect(result.errors).toHaveLength(0); + }); + + it('behaves exactly as before when no cache is set', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes[0].embedding).toEqual([0.4, 0.5]); + }); + }); }); diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts index 896f736..130ff9a 100644 --- a/src/services/embeddings/Orchestrator.ts +++ b/src/services/embeddings/Orchestrator.ts @@ -1,4 +1,5 @@ import { Note } from '../../data/Types'; +import { CachedVector, VectorCache, VectorCacheEntry } from '../../data/Database/VectorRepository'; import { EmbeddingProvider, EmbeddedNote, @@ -8,6 +9,7 @@ import { export class EmbeddingOrchestrator { private provider: EmbeddingProvider | null = null; + private cache: VectorCache | null = null; private cancelled: boolean = false; private onProgress: ((progress: BatchProgress) => void) | null = null; @@ -16,6 +18,11 @@ export class EmbeddingOrchestrator { this.cancelled = false; } + /** Injects a persistent vector cache so unchanged notes skip re-fetching. Optional. */ + public setCache(cache: VectorCache): void { + this.cache = cache; + } + public setOnProgress(callback: (progress: BatchProgress) => void): void { this.onProgress = callback; } @@ -37,8 +44,7 @@ export class EmbeddingOrchestrator { try { this.reportProgress(0, notes.length); - const noteIds = notes.map(n => n.id); - const vectorsByNoteId = await this.provider.fetchVectorsByNoteIds(noteIds); + const vectorsByNoteId = await this.resolveVectors(notes, this.provider); const embeddedNotes: EmbeddedNote[] = []; const errors: Array<{ noteId: string; error: string }> = []; @@ -63,6 +69,82 @@ export class EmbeddingOrchestrator { } } + /** + * Resolves a vector per note, reusing cached vectors for notes whose + * `updated_time` and model haven't changed and only asking the provider + * to (re-)fetch the rest. + */ + private async resolveVectors(notes: Note[], provider: EmbeddingProvider): Promise> { + const modelId = provider.modelName; + const cached = await this.getCachedVectors(notes); + + const notesToFetch = notes.filter(n => !this.isFreshCacheHit(cached.get(n.id), n, modelId)); + + const fresh = notesToFetch.length > 0 + ? await provider.fetchVectorsByNoteIds(notesToFetch.map(n => n.id)) + : new Map(); + + await this.saveFreshVectors(notesToFetch, fresh, modelId); + + return this.mergeVectors(notes, cached, fresh, modelId); + } + + /** Combines still-fresh cached vectors with newly fetched ones, keyed by note ID. */ + private mergeVectors( + notes: Note[], + cached: Map, + fresh: Map, + modelId: string, + ): Map { + const merged = new Map(); + for (const note of notes) { + const entry = cached.get(note.id); + if (entry && this.isFreshCacheHit(entry, note, modelId)) { + merged.set(note.id, entry.vector); + } + } + for (const [noteId, vector] of fresh) merged.set(noteId, vector); + return merged; + } + + /** A cache entry is only reusable if the note is unchanged and the embedding model hasn't changed. */ + private isFreshCacheHit(entry: CachedVector | undefined, note: Note, modelId: string): boolean { + return !!entry && entry.updatedTime === note.updated_time && entry.modelId === modelId; + } + + private async getCachedVectors(notes: Note[]): Promise> { + if (!this.cache) return new Map(); + try { + return await this.cache.getMany(notes.map(n => n.id)); + } catch (e) { + console.error('Vector cache read failed, falling back to a full fetch:', e); + return new Map(); + } + } + + private async saveFreshVectors( + notes: Note[], + vectors: Map, + modelId: string, + ): Promise { + if (!this.cache || vectors.size === 0) return; + + const entries: VectorCacheEntry[] = notes + .filter(n => vectors.has(n.id)) + .map(n => ({ + noteId: n.id, + vector: vectors.get(n.id)!, + modelId, + updatedTime: n.updated_time, + })); + + try { + await this.cache.saveMany(entries); + } catch (e) { + console.error('Vector cache write failed:', e); + } + } + private reportProgress(current: number, total: number): void { if (this.onProgress) { this.onProgress({ current, total }); diff --git a/src/services/embeddings/ProviderResolver.test.ts b/src/services/embeddings/ProviderResolver.test.ts index e8b4072..9d9ad58 100644 --- a/src/services/embeddings/ProviderResolver.test.ts +++ b/src/services/embeddings/ProviderResolver.test.ts @@ -17,19 +17,29 @@ describe('ProviderResolver', () => { ); }); - it('throws when index is not ready', async () => { + it('throws when index is disabled', async () => { (joplin as any).ai = { getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'disabled' }), getEmbeddings: jest.fn(), }; await expect(ProviderResolver.resolveWithValidation()).rejects.toThrow( - 'Joplin AI index is not ready' + 'Joplin AI index is not usable yet (state: disabled)' + ); + }); + + it('throws while the embedding model is still preparing', async () => { + (joplin as any).ai = { + getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'preparing' }), + getEmbeddings: jest.fn(), + }; + await expect(ProviderResolver.resolveWithValidation()).rejects.toThrow( + 'Joplin AI index is not usable yet (state: preparing)' ); }); it('returns provider when index is ready', async () => { (joplin as any).ai = { - getIndexStatus: jest.fn().mockResolvedValue({ ready: true, modelId: 'test-model' }), + getIndexStatus: jest.fn().mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }), getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); @@ -37,9 +47,18 @@ describe('ProviderResolver', () => { expect(provider.modelName).toBe('test-model'); }); + it('returns provider while the index is still indexing, since search still works with partial data', async () => { + (joplin as any).ai = { + getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }), + getEmbeddings: jest.fn(), + }; + const provider = await ProviderResolver.resolveWithValidation(); + expect(provider.modelName).toBe('test-model'); + }); + it('uses the default native model when index status omits modelId', async () => { (joplin as any).ai = { - getIndexStatus: jest.fn().mockResolvedValue({ ready: true }), + getIndexStatus: jest.fn().mockResolvedValue({ ready: true, state: 'ready' }), getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 842b450..446d52b 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -1,21 +1,24 @@ import joplin from 'api'; import { EmbeddingProvider, ProviderConfig } from './Types'; -import { JoplinNativeProvider, JoplinAiApi } from './providers/JoplinNativeProvider'; +import { JoplinNativeProvider, JoplinAiApi, isIndexUsable } from './providers/JoplinNativeProvider'; export class ProviderResolver { /** * Resolves the native embedding provider after verifying that Joplin AI is - * available and its embedding index is ready. + * available and its embedding index is usable. */ public static async resolveWithValidation(): Promise { const joplinAi = joplin.ai as unknown as JoplinAiApi | undefined; - if (!joplinAi || typeof joplinAi.getIndexStatus !== 'function' || typeof joplinAi.getEmbeddings !== 'function') { + if (!joplinAi) { throw new Error('joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.'); } const status = await joplinAi.getIndexStatus(); - if (!status || !status.ready) { - throw new Error('Joplin AI index is not ready. Enable AI and the embedding index in Settings → AI.'); + if (!status || !isIndexUsable(status.state)) { + throw new Error( + `Joplin AI index is not usable yet (state: ${status?.state ?? 'unknown'}). ` + + 'Enable AI and the embedding index in Settings → AI.' + ); } return new JoplinNativeProvider(status.modelId ?? JoplinNativeProvider.DEFAULT_MODEL_ID); } diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts index 63130a2..99a773b 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts @@ -9,7 +9,7 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'test-model' }); + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }); ai.getEmbeddings.mockResolvedValue({ modelId: 'test-model', dimension: 3, @@ -30,7 +30,7 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'fresh-model' }); + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'fresh-model' }); ai.getEmbeddings.mockResolvedValue({ modelId: 'fresh-model', dimension: 2, @@ -56,7 +56,7 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'test-model' }); + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }); ai.getEmbeddings .mockResolvedValueOnce({ modelId: 'test-model', @@ -88,7 +88,7 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'model-a' }); + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'model-a' }); ai.getEmbeddings .mockResolvedValueOnce({ modelId: 'model-a', @@ -117,4 +117,38 @@ describe('JoplinNativeProvider', () => { expect(provider.getFetchedModelId()).toBe('model-b'); expect(provider.modelName).toBe('model-b'); }); + + it('fetches vectors while the index is still indexing, since results are just partial', async () => { + const provider = new JoplinNativeProvider(); + const ai = joplin.ai as unknown as { + getIndexStatus: jest.Mock; + getEmbeddings: jest.Mock; + }; + + ai.getIndexStatus.mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }); + ai.getEmbeddings.mockResolvedValue({ + modelId: 'test-model', + dimension: 2, + chunks: [{ noteId: 'n1', vector: [1, 0] }], + nextCursor: undefined, + }); + + const vectors = await provider.fetchVectorsByNoteIds(['n1']); + + expect(vectors.get('n1')).toEqual([1, 0]); + }); + + it('throws when the index is disabled', async () => { + const provider = new JoplinNativeProvider(); + const ai = joplin.ai as unknown as { + getIndexStatus: jest.Mock; + getEmbeddings: jest.Mock; + }; + + ai.getIndexStatus.mockResolvedValue({ ready: false, state: 'disabled', modelId: null }); + + await expect(provider.fetchVectorsByNoteIds(['n1'])).rejects.toThrow( + 'Joplin AI index is not usable yet (state: disabled)' + ); + }); }); diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index ed6a47a..9a13758 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -1,18 +1,52 @@ import joplin from 'api'; import { EmbeddingProvider, ProviderId } from '../Types'; +/** + * Mirrors Joplin's official AiIndexState type (joplinapp.org/api/references/plugin_api). + * 'unavailable' | 'disabled' | 'preparing' block any fetch (no data yet). + * 'indexing' still allows fetching — results are partial, handled downstream + * as per-note "not yet indexed" errors. 'ready' is the fully-indexed state. + */ +export type AiIndexState = 'unavailable' | 'disabled' | 'preparing' | 'indexing' | 'ready'; + +export interface AiIndexStatus { + modelId: string | null; + notesIndexed: number; + ready: boolean; + state: AiIndexState; + totalNotes: number; +} + +export interface EmbeddingChunk { + chunkIndex: number; + chunkText: string; + noteId: string; + vector: number[]; +} + +export interface EmbeddingsPage { + chunks: EmbeddingChunk[]; + dimension: number; + modelId: string; + nextCursor?: string; +} + +export interface GetEmbeddingsOptions { + cursor?: string; + limit?: number; + noteIds?: string[]; +} + export interface JoplinAiApi { - getIndexStatus: () => Promise<{ ready: boolean; modelId?: string | null }>; - getEmbeddings: (params: { - noteIds?: string[]; - cursor?: string; - limit: number; - }) => Promise<{ - modelId?: string | null; - dimension: number; - chunks: Array<{ noteId: string; vector: number[] }>; - nextCursor?: string; - }>; + getIndexStatus: () => Promise; + getEmbeddings: (options: GetEmbeddingsOptions) => Promise; +} + +const BLOCKING_STATES: ReadonlySet = new Set(['unavailable', 'disabled', 'preparing']); + +/** True once the index has enough data to fetch from, even if still indexing. */ +export function isIndexUsable(state: AiIndexState | undefined): boolean { + return !!state && !BLOCKING_STATES.has(state); } export class JoplinNativeProvider implements EmbeddingProvider { @@ -60,15 +94,20 @@ export class JoplinNativeProvider implements EmbeddingProvider { return this.fetchedModelId; } - /** Checks that joplin.ai exists and has the required methods. */ + /** + * Checks that joplin.ai exists. Deliberately does not probe individual + * method properties (e.g. `typeof api.getEmbeddings`) — Joplin's plugin + * RPC bridge exposes joplin.ai as a proxy that accumulates property-path + * state across accesses, so a property read that's never invoked can + * corrupt the path used by a later real call. Always access a method and + * invoke it in the same expression; let a genuinely missing method throw + * on invocation instead of pre-checking with typeof. + */ private validateAiApi(): JoplinAiApi { const api = joplin.ai as unknown as JoplinAiApi | undefined; - if (!api || typeof api.getEmbeddings !== 'function') { - throw new Error('joplin.ai.getEmbeddings is not available. Enable AI in Settings → AI.'); - } - if (typeof api.getIndexStatus !== 'function') { - throw new Error('joplin.ai.getIndexStatus is not available. Enable AI in Settings → AI.'); + if (!api) { + throw new Error('joplin.ai is not available. Enable AI in Settings → AI.'); } return api; @@ -82,12 +121,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { api: JoplinAiApi, noteIds: string[], ): Promise> { - const status = await api.getIndexStatus(); - if (!status || !status.ready) { - throw new Error('Joplin AI index is not ready. Wait for indexing to complete or enable AI in Settings → AI.'); - } - - let trackedModelId: string | null = status.modelId ?? null; + let trackedModelId = await this.requireUsableIndex(api); const grouped = new Map(); let cursor: string | undefined; @@ -123,14 +157,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { } } - for (const chunk of page.chunks) { - const list = grouped.get(chunk.noteId); - if (list) { - list.push(chunk.vector); - } else { - grouped.set(chunk.noteId, [chunk.vector]); - } - } + this.addChunksToGroup(grouped, page.chunks); cursor = page.nextCursor; if (!cursor) { @@ -142,6 +169,30 @@ export class JoplinNativeProvider implements EmbeddingProvider { return grouped; } + /** Throws if the index isn't usable yet; otherwise returns the model ID it's currently indexed with. */ + private async requireUsableIndex(api: JoplinAiApi): Promise { + const status = await api.getIndexStatus(); + if (!status || !isIndexUsable(status.state)) { + throw new Error( + `Joplin AI index is not usable yet (state: ${status?.state ?? 'unknown'}). ` + + 'Enable AI and wait for the embedding model to finish loading in Settings → AI.' + ); + } + return status.modelId ?? null; + } + + /** Appends each chunk's vector onto its note's running vector list. */ + private addChunksToGroup(grouped: Map, chunks: EmbeddingChunk[]): void { + for (const chunk of chunks) { + const list = grouped.get(chunk.noteId); + if (list) { + list.push(chunk.vector); + } else { + grouped.set(chunk.noteId, [chunk.vector]); + } + } + } + /** Averages multiple chunk vectors per note into one vector and L2-normalizes. */ private poolAndNormalize(grouped: Map): Map { const result = new Map(); diff --git a/src/services/graph/GraphBuilder.test.ts b/src/services/graph/GraphBuilder.test.ts index 5c31c71..0864f56 100644 --- a/src/services/graph/GraphBuilder.test.ts +++ b/src/services/graph/GraphBuilder.test.ts @@ -1,10 +1,13 @@ import { GraphBuilder } from './GraphBuilder'; import { EdgeFactory } from '../similarity/EdgeFactory'; +import { SimilarityEngine } from '../similarity/SimilarityEngine'; import { Note } from '../../data/Types'; jest.mock('../similarity/EdgeFactory'); +jest.mock('../similarity/SimilarityEngine'); const MockEdgeFactory = EdgeFactory as jest.MockedClass; +const MockSimilarityEngine = SimilarityEngine as jest.MockedClass; function note( id: string, @@ -77,4 +80,45 @@ describe('GraphBuilder', () => { expect(result.edges).toHaveLength(1); expect(result.edges[0].data).toEqual({ source: 'a', target: 'b', type: 'link' }); }); + + describe('buildWithSimilarity', () => { + it('adds semantic edges computed from embeddings alongside structural edges', async () => { + mockEdgeFactory.createEdges.mockReturnValue([{ source: 'a', target: 'c', type: 'link' }]); + mockEdgeFactory.createSemanticEdges.mockReturnValue([ + { source: 'a', target: 'b', type: 'semantic' }, + ]); + MockSimilarityEngine.mockImplementation(() => ({ + compute: jest.fn().mockResolvedValue([{ source: 'a', target: 'b', score: 0.8 }]), + }) as unknown as SimilarityEngine); + + const notes = [note('a', 'A'), note('b', 'B'), note('c', 'C')]; + const embeddedNotes = [ + { note: notes[0], embedding: [1, 0] }, + { note: notes[1], embedding: [0.9, 0.1] }, + ]; + + const result = await builder.buildWithSimilarity(notes, embeddedNotes); + + expect(mockEdgeFactory.createSemanticEdges).toHaveBeenCalledWith([ + { source: 'a', target: 'b', score: 0.8 }, + ]); + expect(result.edges).toContainEqual({ data: { source: 'a', target: 'b', type: 'semantic' } }); + expect(result.edges).toContainEqual({ data: { source: 'a', target: 'c', type: 'link' } }); + expect(result.edges).toHaveLength(2); + }); + + it('still returns a graph when there are no semantic matches', async () => { + mockEdgeFactory.createEdges.mockReturnValue([]); + mockEdgeFactory.createSemanticEdges.mockReturnValue([]); + MockSimilarityEngine.mockImplementation(() => ({ + compute: jest.fn().mockResolvedValue([]), + }) as unknown as SimilarityEngine); + + const notes = [note('a', 'A'), note('b', 'B')]; + const result = await builder.buildWithSimilarity(notes, []); + + expect(result.nodes).toHaveLength(2); + expect(result.edges).toEqual([]); + }); + }); }); diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 1921cfa..88f1526 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -1,6 +1,8 @@ import { Note } from '../../data/Types'; import { EdgeFactory } from '../similarity/EdgeFactory'; -import { GraphData, GraphNode } from './types'; +import { SimilarityEngine } from '../similarity/SimilarityEngine'; +import { EmbeddedNote } from '../embeddings/Types'; +import { GraphData, GraphEdge, GraphNode } from './types'; export class GraphBuilder { private readonly edgeFactory: EdgeFactory; @@ -15,20 +17,57 @@ export class GraphBuilder { * @returns graph data ready for rendering (nodes and edges). */ public build(notes: Note[]): GraphData { + const edges = this.edgeFactory.createEdges(notes); + return this.buildData(notes, edges); + } + + /** + * Builds a graph with semantic edges computed from embedding vectors, + * in addition to link and tag edges. + */ + public async buildWithSimilarity( + notes: Note[], + embeddedNotes: EmbeddedNote[], + ): Promise { + const structuralEdges = this.edgeFactory.createEdges(notes); + + const engine = new SimilarityEngine(notes, embeddedNotes); + const pairs = await engine.compute(); + const semanticEdges = this.edgeFactory.createSemanticEdges(pairs); + + const allEdges = [...structuralEdges, ...semanticEdges]; + return this.buildData(notes, allEdges); + } + + private buildData(notes: Note[], edges: GraphEdge[]): GraphData { + const degreeMap = this.computeDegreeMap(notes, edges); + const nodes = this.buildNodes(notes, degreeMap); + + const nodeIdSet = new Set(nodes.map((n) => n.data.id)); + const visibleEdges = this.filterVisibleEdges(edges, nodeIdSet); + + this.logGraphStats(nodes, visibleEdges, degreeMap); + + return { nodes, edges: visibleEdges.map((e) => ({ data: e })) }; + } + + /** Counts each note's connections, including notes an edge references that aren't in `notes`. */ + private computeDegreeMap(notes: Note[], edges: GraphEdge[]): Map { const degreeMap = new Map(); for (const note of notes) { degreeMap.set(note.id, 0); } - const edges = this.edgeFactory.createEdges(notes); - for (const edge of edges) { degreeMap.set(edge.source, (degreeMap.get(edge.source) ?? 0) + 1); degreeMap.set(edge.target, (degreeMap.get(edge.target) ?? 0) + 1); } - const maxDegree = Math.max(1, ...degreeMap.values()); + return degreeMap; + } + /** Builds one node per note, truncating long titles to keep labels readable in the graph. */ + private buildNodes(notes: Note[], degreeMap: Map): Array<{ data: GraphNode }> { const nodes: Array<{ data: GraphNode }> = []; for (const note of notes) { const degree = degreeMap.get(note.id) ?? 0; @@ -42,24 +81,30 @@ export class GraphBuilder { }, }); } + return nodes; + } - const nodeIdSet = new Set(nodes.map((n) => n.data.id)); - const visibleEdges = edges.filter( - (e) => nodeIdSet.has(e.source) && nodeIdSet.has(e.target) - ); + /** Drops edges referencing a note outside the current node set. */ + private filterVisibleEdges(edges: GraphEdge[], nodeIdSet: Set): GraphEdge[] { + return edges.filter((e) => nodeIdSet.has(e.source) && nodeIdSet.has(e.target)); + } + private logGraphStats( + nodes: Array<{ data: GraphNode }>, + visibleEdges: GraphEdge[], + degreeMap: Map, + ): void { const connectedIds = new Set(); for (const edge of visibleEdges) { connectedIds.add(edge.source); connectedIds.add(edge.target); } const isolatedCount = nodes.length - connectedIds.size; + const maxDegree = Math.max(1, ...degreeMap.values()); console.info( `Graph built: ${nodes.length} nodes, ${visibleEdges.length} edges ` + `(${isolatedCount} isolated, max degree ${maxDegree})` ); - - return { nodes, edges: visibleEdges.map((e) => ({ data: e })) }; } } diff --git a/src/services/similarity/EdgeFactory.test.ts b/src/services/similarity/EdgeFactory.test.ts index 90c1924..c753663 100644 --- a/src/services/similarity/EdgeFactory.test.ts +++ b/src/services/similarity/EdgeFactory.test.ts @@ -101,4 +101,25 @@ describe('EdgeFactory', () => { it('ignores self-referencing links', () => { expect(factory.createEdges([note('a', 'A', ['a'])])).toEqual([]); }); + + describe('createSemanticEdges', () => { + it('returns empty for no pairs', () => { + expect(factory.createSemanticEdges([])).toEqual([]); + }); + + it('creates a semantic edge for each positive-score pair', () => { + const edges = factory.createSemanticEdges([ + { source: 'a', target: 'b', score: 0.8 }, + ]); + expect(edges).toEqual([{ source: 'a', target: 'b', type: 'semantic' }]); + }); + + it('excludes pairs with a non-positive score', () => { + const edges = factory.createSemanticEdges([ + { source: 'a', target: 'b', score: 0 }, + { source: 'c', target: 'd', score: -0.1 }, + ]); + expect(edges).toEqual([]); + }); + }); }); diff --git a/src/services/similarity/EdgeFactory.ts b/src/services/similarity/EdgeFactory.ts index 585c344..0197272 100644 --- a/src/services/similarity/EdgeFactory.ts +++ b/src/services/similarity/EdgeFactory.ts @@ -1,5 +1,6 @@ import { Note } from '../../data/Types'; import { GraphEdge } from '../graph/types'; +import { SimilarityPair } from './SimilarityEngine'; export class EdgeFactory { /** @@ -8,6 +9,11 @@ export class EdgeFactory { * @returns deduplicated edges of type `link` and `tag`. */ public createEdges(notes: Note[]): GraphEdge[] { + return [...this.createLinkEdges(notes), ...this.createTagEdges(notes)]; + } + + /** Builds one deduplicated edge per explicit `:/noteId` link between two notes in scope. */ + private createLinkEdges(notes: Note[]): GraphEdge[] { const noteIdSet = new Set(notes.map((n) => n.id)); const edges: GraphEdge[] = []; const linkKeySet = new Set(); @@ -24,17 +30,18 @@ export class EdgeFactory { } } - const tagToNotes = new Map(); - for (const note of notes) { - for (const tag of note.tags ?? []) { - if (!tagToNotes.has(tag)) { - tagToNotes.set(tag, []); - } - tagToNotes.get(tag)!.push(note.id); - } - } + return edges; + } + /** + * Builds one edge per pair of notes sharing a tag, merging multiple shared + * tag names onto the same edge. Tags shared by more than 20 notes are + * skipped to avoid a combinatorial blowup of pairs. + */ + private createTagEdges(notes: Note[]): GraphEdge[] { + const tagToNotes = this.groupNoteIdsByTag(notes); const tagEdgeMap = new Map(); + for (const [tagName, noteIds] of tagToNotes) { if (noteIds.length > 20) continue; @@ -59,8 +66,37 @@ export class EdgeFactory { } } - for (const edge of tagEdgeMap.values()) { - edges.push(edge); + return Array.from(tagEdgeMap.values()); + } + + private groupNoteIdsByTag(notes: Note[]): Map { + const tagToNotes = new Map(); + for (const note of notes) { + for (const tag of note.tags ?? []) { + if (!tagToNotes.has(tag)) { + tagToNotes.set(tag, []); + } + tagToNotes.get(tag)!.push(note.id); + } + } + return tagToNotes; + } + + /** + * Creates semantic edges from similarity pairs. + * Each pair represents a strong semantic connection between two notes. + */ + public createSemanticEdges(pairs: SimilarityPair[]): GraphEdge[] { + const edges: GraphEdge[] = []; + + for (const pair of pairs) { + if (pair.score <= 0) continue; + + edges.push({ + source: pair.source, + target: pair.target, + type: 'semantic', + }); } return edges; diff --git a/src/services/similarity/SimilarityEngine.test.ts b/src/services/similarity/SimilarityEngine.test.ts new file mode 100644 index 0000000..f9c8f1c --- /dev/null +++ b/src/services/similarity/SimilarityEngine.test.ts @@ -0,0 +1,515 @@ +import { SimilarityEngine } from './SimilarityEngine'; +import { Note } from '../../data/Types'; +import { EmbeddedNote } from '../embeddings/Types'; + +function makeNote( + id: string, + title: string, + links: string[] = [], + tags: string[] = [], + createdTime = 0, +): Note { + return { + id, + parent_id: 'p1', + title, + body: '', + created_time: createdTime, + updated_time: 1, + links, + tags, + }; +} + +const DAY_MS = 1000 * 60 * 60 * 24; + +function embed(id: string, vector: number[]): EmbeddedNote { + return { note: makeNote(id, 'Note ' + id), embedding: vector }; +} + +describe('SimilarityEngine', () => { + describe('compute', () => { + it('returns empty for no notes', async () => { + const engine = new SimilarityEngine([], []); + const pairs = await engine.compute(); + expect(pairs).toEqual([]); + }); + + it('returns empty for a single note', async () => { + const engine = new SimilarityEngine( + [makeNote('a', 'A')], + [embed('a', [1, 0, 0])], + ); + const pairs = await engine.compute(); + expect(pairs).toEqual([]); + }); + + it('computes cosine similarity for two similar notes', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B')]; + const embedded = [embed('a', [1, 0]), embed('b', [0.95, 0.3])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toHaveLength(1); + expect(pairs[0].source).toBe('a'); + expect(pairs[0].target).toBe('b'); + expect(pairs[0].score).toBeGreaterThan(0.5); + }); + + it('orders source before target deterministically', async () => { + const notes = [makeNote('b', 'B'), makeNote('a', 'A')]; + const embedded = [embed('b', [1, 0]), embed('a', [0.95, 0.3])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toHaveLength(1); + expect(pairs[0].source).toBe('a'); + expect(pairs[0].target).toBe('b'); + }); + + it('returns empty when all notes lack vectors', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B')]; + const engine = new SimilarityEngine(notes, []); + const pairs = await engine.compute(); + expect(pairs).toEqual([]); + }); + + it('skips notes without vectors in the mapping', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B'), makeNote('c', 'C')]; + const embedded = [embed('a', [0.95, 0.3]), embed('c', [0.3, 0.95])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const involvesB = pairs.some( + p => p.source === 'b' || p.target === 'b', + ); + expect(involvesB).toBe(false); + }); + }); + + describe('normalization', () => { + it('produces scores in [0, 1] range', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B'), makeNote('c', 'C')]; + const embedded = [ + embed('a', [1, 0]), + embed('b', [0.95, 0.3]), + embed('c', [0.3, 0.95]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + for (const p of pairs) { + expect(p.score).toBeGreaterThanOrEqual(0); + expect(p.score).toBeLessThanOrEqual(1.5); + } + }); + + it('gives higher scores to more similar notes', async () => { + const notes = [ + makeNote('a', 'A'), makeNote('b', 'B'), + makeNote('c', 'C'), makeNote('d', 'D'), + ]; + const embedded = [ + embed('a', [1, 0]), + embed('b', [0.95, 0.31]), + embed('c', [0.7, 0.71]), + embed('d', [-1, 0]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const abScore = pairs.find( + p => + (p.source === 'a' && p.target === 'b') || + (p.source === 'b' && p.target === 'a'), + ); + + const acScore = pairs.find( + p => + (p.source === 'a' && p.target === 'c') || + (p.source === 'c' && p.target === 'a'), + ); + + expect(abScore).toBeDefined(); + expect(acScore).toBeDefined(); + expect(abScore!.score).toBeGreaterThan(acScore!.score); + }); + }); + + describe('tag bonuses', () => { + it('boosts score when notes share tags', async () => { + // Padded to 7 notes so the 'shared' tag (on 2 of them) stays under the + // 30% organizational-tag threshold and isn't excluded from the signal. + const notes = [ + makeNote('a', 'A', [], ['shared']), + makeNote('b', 'B', [], ['shared']), + makeNote('c', 'C', [], []), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [0.95, 0.3]), + embed('b', [0.98, 0.2]), + embed('c', [0.96, 0.28]), + embed('pad0', [-1, 0]), + embed('pad1', [0, -1]), + embed('pad2', [-0.7, 0.7]), + embed('pad3', [0.7, -0.7]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const abWithTag = pairs.find( + p => + (p.source === 'a' && p.target === 'b') || + (p.source === 'b' && p.target === 'a'), + ); + const acNoTag = pairs.find( + p => + (p.source === 'a' && p.target === 'c') || + (p.source === 'c' && p.target === 'a'), + ); + + expect(abWithTag).toBeDefined(); + expect(acNoTag).toBeDefined(); + expect(abWithTag!.score).toBeGreaterThan(acNoTag!.score); + }); + + it('scales the tag bonus by Jaccard overlap, not raw shared-tag count', async () => { + // a-b and c-d each have the highest raw dot product (0.9) in the whole + // fixture, so both normalize to exactly 1.0 regardless of the padding + // notes' spread — isolating the tag bonus as the only source of + // difference between their final scores. a/b fully overlap in tags + // (jaccard=1.0 -> bonus=0.1); c/d partially overlap (jaccard=0.5 -> + // bonus=0.05). A raw-count bonus would instead give both pairs the + // same 2 * TAG_BONUS = 0.2, making them equal. + const notes = [ + makeNote('a', 'A', [], ['p', 'q']), + makeNote('b', 'B', [], ['p', 'q']), + makeNote('c', 'C', [], ['r', 's']), + makeNote('d', 'D', [], ['r', 's', 't', 'u']), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [1, 0, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0, 0]), + embed('c', [0, 1, 0, 0]), + embed('d', [0, 0.9, Math.sqrt(1 - 0.81), 0]), + embed('pad0', [-1, 0, 0, 0]), + embed('pad1', [0, -1, 0, 0]), + embed('pad2', [0, 0, -1, 0]), + embed('pad3', [0, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const ab = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const cd = pairs.find( + p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + ); + + expect(ab).toBeDefined(); + expect(cd).toBeDefined(); + expect(ab!.score - cd!.score).toBeCloseTo(0.05, 5); + }); + + it('excludes organizational tags (present on more than 30% of notes) from the bonus', async () => { + // 'inbox' appears on a, c, d, e (4 of 10 notes = 40%) — organizational, excluded. + // 'project' appears only on a and b (2 of 10 = 20%) — meaningful, included. + const notes = [ + makeNote('a', 'A', [], ['inbox', 'project']), + makeNote('b', 'B', [], ['project']), + makeNote('c', 'C', [], ['inbox']), + makeNote('d', 'D', [], ['inbox']), + makeNote('e', 'E', [], ['inbox']), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + makeNote('pad4', 'Pad 4'), + ]; + const embedded = [ + embed('a', [0.95, 0.3]), + embed('b', [0.98, 0.2]), + embed('c', [0.96, 0.28]), + embed('d', [-1, 0]), + embed('e', [0, -1]), + embed('pad0', [-0.7, 0.7]), + embed('pad1', [0.7, -0.7]), + embed('pad2', [-0.9, 0.1]), + embed('pad3', [0.1, -0.9]), + embed('pad4', [-0.5, -0.5]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const abSharesProject = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const acSharesOnlyInbox = pairs.find( + p => (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a'), + ); + + expect(abSharesProject).toBeDefined(); + expect(acSharesOnlyInbox).toBeDefined(); + expect(abSharesProject!.score).toBeGreaterThan(acSharesOnlyInbox!.score); + }); + }); + + describe('link bonuses', () => { + it('boosts score when notes link to each other', async () => { + const notes = [ + makeNote('a', 'A', ['b'], []), + makeNote('b', 'B', [], []), + makeNote('c', 'C', [], []), + ]; + const embedded = [ + embed('a', [0.95, 0.3]), + embed('b', [0.98, 0.2]), + embed('c', [0.96, 0.28]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const abWithLink = pairs.find( + p => + (p.source === 'a' && p.target === 'b') || + (p.source === 'b' && p.target === 'a'), + ); + const acNoLink = pairs.find( + p => + (p.source === 'a' && p.target === 'c') || + (p.source === 'c' && p.target === 'a'), + ); + + expect(abWithLink).toBeDefined(); + expect(acNoLink).toBeDefined(); + expect(abWithLink!.score).toBeGreaterThan(acNoLink!.score); + }); + }); + + describe('temporal proximity bonus', () => { + it('gives a stronger boost to notes created within a day than notes created within a week', async () => { + // a-b and c-d each have the same raw dot product (0.9) in the whole + // fixture, so both normalize to exactly 1.0 regardless of the padding + // notes' spread — isolating the temporal bonus as the only source of + // difference between their final scores. a/b were created 12 hours + // apart (same-day bonus 0.1); c/d were created 3 days apart (same-week + // bonus 0.05). + const notes = [ + makeNote('a', 'A', [], [], 0), + makeNote('b', 'B', [], [], 12 * 60 * 60 * 1000), + makeNote('c', 'C', [], [], 0), + makeNote('d', 'D', [], [], 3 * DAY_MS), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [1, 0, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0, 0]), + embed('c', [0, 1, 0, 0]), + embed('d', [0, 0.9, Math.sqrt(1 - 0.81), 0]), + embed('pad0', [-1, 0, 0, 0]), + embed('pad1', [0, -1, 0, 0]), + embed('pad2', [0, 0, -1, 0]), + embed('pad3', [0, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const ab = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const cd = pairs.find( + p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + ); + + expect(ab).toBeDefined(); + expect(cd).toBeDefined(); + expect(ab!.score - cd!.score).toBeCloseTo(0.05, 5); + }); + + it('gives no temporal bonus to notes created more than a week apart', async () => { + const notes = [ + makeNote('a', 'A', [], [], 0), + makeNote('b', 'B', [], [], DAY_MS), + makeNote('c', 'C', [], [], 0), + makeNote('d', 'D', [], [], 30 * DAY_MS), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [1, 0, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0, 0]), + embed('c', [0, 1, 0, 0]), + embed('d', [0, 0.9, Math.sqrt(1 - 0.81), 0]), + embed('pad0', [-1, 0, 0, 0]), + embed('pad1', [0, -1, 0, 0]), + embed('pad2', [0, 0, -1, 0]), + embed('pad3', [0, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const ab = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const cd = pairs.find( + p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + ); + + expect(ab).toBeDefined(); + expect(cd).toBeDefined(); + expect(ab!.score - cd!.score).toBeCloseTo(0.1, 5); + }); + + it('applies the bonus inclusively at exactly 1 day and exactly 7 days', async () => { + const notes = [ + makeNote('a', 'A', [], [], 0), + makeNote('b', 'B', [], [], DAY_MS), + makeNote('c', 'C', [], [], 0), + makeNote('d', 'D', [], [], 7 * DAY_MS), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [1, 0, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0, 0]), + embed('c', [0, 1, 0, 0]), + embed('d', [0, 0.9, Math.sqrt(1 - 0.81), 0]), + embed('pad0', [-1, 0, 0, 0]), + embed('pad1', [0, -1, 0, 0]), + embed('pad2', [0, 0, -1, 0]), + embed('pad3', [0, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const ab = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const cd = pairs.find( + p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + ); + + expect(ab).toBeDefined(); + expect(cd).toBeDefined(); + expect(ab!.score - cd!.score).toBeCloseTo(0.05, 5); + }); + }); + + describe('top-K filtering', () => { + it('limits edges per note', async () => { + const notes = []; + const embedded = []; + for (let i = 0; i < 6; i++) { + notes.push(makeNote(`n${i}`, `Note ${i}`)); + embedded.push( + embed(`n${i}`, [ + Math.cos((i * Math.PI) / 3), + Math.sin((i * Math.PI) / 3), + ]), + ); + } + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + for (const note of notes) { + const edges = pairs.filter( + p => p.source === note.id || p.target === note.id, + ); + expect(edges.length).toBeLessThanOrEqual(5); + } + }); + + it('lets a hub note exceed K edges when more than K notes independently pick it', async () => { + // A shares a "topic" dimension with 8 satellites, and each satellite + // also has its own unique dimension. That makes every satellite closer + // to A (dot = 0.9) than to any other satellite (dot = 0.81), so A is + // always each satellite's #1 pick. Top-K is per-note (union), not a + // hard cap on incoming edges, so A ends up with more than 5 edges here. + const dims = 9; + const aVector = new Array(dims).fill(0); + aVector[0] = 1; + + const notes = [makeNote('a', 'A')]; + const embedded = [embed('a', aVector)]; + + for (let i = 0; i < 8; i++) { + const satelliteVector = new Array(dims).fill(0); + satelliteVector[0] = 0.9; + satelliteVector[i + 1] = 0.3; + notes.push(makeNote(`n${i}`, `Note ${i}`)); + embedded.push(embed(`n${i}`, satelliteVector)); + } + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const aEdges = pairs.filter(p => p.source === 'a' || p.target === 'a'); + expect(aEdges.length).toBeGreaterThan(5); + }); + }); + + describe('SEMANTIC_FLOOR and threshold ordering', () => { + it('rejects a below-floor pair even with a shared tag', async () => { + // a and b are nearly orthogonal (cosine ~0), share a tag but are not + // linked. SEMANTIC_FLOOR must reject them before bonuses or threshold + // ever apply — tags alone can never manufacture an edge out of a weak + // semantic score. + const notes = [ + makeNote('a', 'A', [], ['shared']), + makeNote('b', 'B', [], ['shared']), + ]; + const embedded = [embed('a', [1, 0]), embed('b', [0, 1])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toEqual([]); + }); + + it('lets a direct link bypass the floor, but the boosted score must still clear the threshold', async () => { + // a and b are nearly orthogonal (cosine ~0) but directly link to each other + // and share a tag. The link bypasses SEMANTIC_FLOOR (a user-created edge + // isn't a false positive), but the resulting boosted score (~0.25) still + // isn't enough to clear DEFAULT_THRESHOLD (0.5). + const notes = [ + makeNote('a', 'A', ['b'], ['shared']), + makeNote('b', 'B', [], ['shared']), + ]; + const embedded = [embed('a', [1, 0]), embed('b', [0, 1])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toEqual([]); + }); + }); +}); diff --git a/src/services/similarity/SimilarityEngine.ts b/src/services/similarity/SimilarityEngine.ts new file mode 100644 index 0000000..dd8a839 --- /dev/null +++ b/src/services/similarity/SimilarityEngine.ts @@ -0,0 +1,316 @@ +import joplin from 'api'; +import { SearchOptions, SearchResult } from 'api/types'; +import { Note } from '../../data/Types'; +import { EmbeddedNote } from '../embeddings/Types'; +import { + SEMANTIC_FLOOR, + DEFAULT_THRESHOLD, + TOP_K, + LARGE_VAULT_THRESHOLD, + TAG_BONUS, + LINK_BONUS, + ORGANIZATIONAL_TAG_RATIO, + TEMPORAL_BONUS_1_DAY, + TEMPORAL_BONUS_7_DAYS, + MS_PER_DAY, +} from './ThresholdPresets'; + +export interface SimilarityPair { + source: string; + target: string; + score: number; +} + +export class SimilarityEngine { + private readonly noteIds: string[]; + private readonly vectors: Map; + private readonly tagMap: Map>; + private readonly linkSet: Set; + private readonly createdTimeMap: Map; + + public constructor(notes: Note[], embeddedNotes: EmbeddedNote[]) { + this.noteIds = notes.map(n => n.id); + this.vectors = new Map(); + for (const en of embeddedNotes) { + this.vectors.set(en.note.id, en.embedding); + } + this.tagMap = this.buildTagMap(notes); + this.linkSet = this.buildLinkSet(notes); + this.createdTimeMap = new Map(notes.map(n => [n.id, n.created_time])); + } + + /** Orchestrates the full similarity pipeline: compute → normalize → floor → enrich → threshold → top-K. */ + public async compute(): Promise { + if (this.noteIds.length <= 1) { + return []; + } + + const rawPairs = await this.computeRawPairs(); + + if (rawPairs.length === 0) { + return []; + } + + const normalized = this.normalize(rawPairs); + const aboveFloor = this.filterBelowFloor(normalized, SEMANTIC_FLOOR); + const enriched = this.addBonusPoints(aboveFloor); + const aboveThreshold = this.filterBelowThreshold(enriched, DEFAULT_THRESHOLD); + const topPairs = this.selectTopK(aboveThreshold, TOP_K); + + return topPairs; + } + + /** Picks the appropriate similarity strategy based on vault size. */ + private computeRawPairs(): Promise { + if (this.noteIds.length <= LARGE_VAULT_THRESHOLD) { + return Promise.resolve(this.computeCosinePairs()); + } + return this.computeSearchPairs(); + } + + /** O(n²) pairwise cosine similarity via dot product on unit-norm vectors. */ + private computeCosinePairs(): SimilarityPair[] { + const pairs: SimilarityPair[] = []; + const n = this.noteIds.length; + + for (let i = 0; i < n; i++) { + const a = this.noteIds[i]; + const vecA = this.vectors.get(a); + if (!vecA) continue; + + for (let j = i + 1; j < n; j++) { + const b = this.noteIds[j]; + const vecB = this.vectors.get(b); + if (!vecB) continue; + + const score = this.dotProduct(vecA, vecB); + const [source, target] = a < b ? [a, b] : [b, a]; + pairs.push({ source, target, score }); + } + } + + return pairs; + } + + /** + * Uses joplin.ai.search({ noteId }) to find candidate pairs via vector index. + * Only checks that joplin.ai itself exists — never probes a specific method + * property without invoking it (see JoplinNativeProvider.validateAiApi for why). + */ + private async computeSearchPairs(): Promise { + const joplinAi = joplin.ai as unknown as + | { search: (options: SearchOptions) => Promise } + | undefined; + if (!joplinAi) { + return this.computeCosinePairs(); + } + + const seen = new Set(); + const pairs: SimilarityPair[] = []; + + for (const noteId of this.noteIds) { + try { + const results = await joplinAi.search({ + query: { noteId }, + relevance: 'normal', + }); + + for (const r of results) { + if (!this.vectors.has(r.noteId) || r.noteId === noteId) { + continue; + } + + const key = this.makePairKey(noteId, r.noteId); + if (seen.has(key)) continue; + seen.add(key); + + const [source, target] = noteId < r.noteId + ? [noteId, r.noteId] + : [r.noteId, noteId]; + + pairs.push({ source, target, score: r.score }); + } + } catch { + continue; + } + } + + return pairs; + } + + /** Dot product of two same-length vectors. */ + private dotProduct(a: number[], b: number[]): number { + let sum = 0; + for (let i = 0; i < a.length; i++) { + sum += a[i] * b[i]; + } + return sum; + } + + /** Min-max normalizes scores to [0, 1]. Skips if spread is too narrow. */ + private normalize(pairs: SimilarityPair[]): SimilarityPair[] { + let min = Infinity; + let max = -Infinity; + + for (const p of pairs) { + if (p.score < min) min = p.score; + if (p.score > max) max = p.score; + } + + const spread = max - min; + if (spread < 0.1) { + return pairs; + } + + for (const p of pairs) { + p.score = (p.score - min) / spread; + } + + return pairs; + } + + /** Adds shared-tag, direct-link, and temporal-proximity bonuses to each pair's score. */ + private addBonusPoints(pairs: SimilarityPair[]): SimilarityPair[] { + for (const p of pairs) { + p.score += this.sharedTagBonus(p) + this.directLinkBonus(p) + this.temporalProximityBonus(p); + } + return pairs; + } + + /** Jaccard overlap (intersection / union) between two notes' tag sets, scaled by TAG_BONUS. */ + private sharedTagBonus(pair: SimilarityPair): number { + const tagsA = this.tagMap.get(pair.source) ?? new Set(); + const tagsB = this.tagMap.get(pair.target) ?? new Set(); + if (tagsA.size === 0 && tagsB.size === 0) return 0; + + let intersectionSize = 0; + for (const t of tagsA) { + if (tagsB.has(t)) intersectionSize++; + } + const unionSize = new Set([...tagsA, ...tagsB]).size; + + return (intersectionSize / unionSize) * TAG_BONUS; + } + + private directLinkBonus(pair: SimilarityPair): number { + return this.isDirectlyLinked(pair) ? LINK_BONUS : 0; + } + + private isDirectlyLinked(pair: SimilarityPair): boolean { + return this.linkSet.has(this.makePairKey(pair.source, pair.target)); + } + + /** Boosts notes created close together in time: same day scores higher than same week. */ + private temporalProximityBonus(pair: SimilarityPair): number { + const createdA = this.createdTimeMap.get(pair.source); + const createdB = this.createdTimeMap.get(pair.target); + if (createdA === undefined || createdB === undefined) return 0; + + const daysApart = Math.abs(createdA - createdB) / MS_PER_DAY; + if (daysApart <= 1) return TEMPORAL_BONUS_1_DAY; + if (daysApart <= 7) return TEMPORAL_BONUS_7_DAYS; + return 0; + } + + /** + * Removes pairs below the safety floor — unless the notes are directly + * linked, in which case they're kept and left for the threshold check + * below. SEMANTIC_FLOOR guards against spurious tag-only edges, not + * against edges the user already created explicitly. + */ + private filterBelowFloor(pairs: SimilarityPair[], floor: number): SimilarityPair[] { + return pairs.filter(p => p.score >= floor || this.isDirectlyLinked(p)); + } + + /** Keeps only pairs whose bonus-boosted score clears the threshold. */ + private filterBelowThreshold(pairs: SimilarityPair[], threshold: number): SimilarityPair[] { + return pairs.filter(p => p.score >= threshold); + } + + /** + * Keeps each note's own K strongest connections; the returned edge set is + * their union, so a note that many others pick as one of their top-K can + * end up with more than K edges. This is the standard k-nearest-neighbor + * graph definition and preserves degree as a centrality signal. + */ + private selectTopK(pairs: SimilarityPair[], k: number): SimilarityPair[] { + const bySource = new Map(); + + for (const p of pairs) { + this.appendPair(bySource, p.source, p); + this.appendPair(bySource, p.target, { source: p.target, target: p.source, score: p.score }); + } + + const deduped = new Map(); + + for (const [, candidates] of bySource) { + candidates.sort((a, b) => b.score - a.score); + const kept = candidates.slice(0, k); + + for (const p of kept) { + const key = this.makePairKey(p.source, p.target); + if (!deduped.has(key)) { + deduped.set(key, p); + } + } + } + + return Array.from(deduped.values()); + } + + private appendPair( + map: Map, + noteId: string, + pair: SimilarityPair, + ): void { + let list = map.get(noteId); + if (!list) { + list = []; + map.set(noteId, list); + } + list.push(pair); + } + + /** Deterministic ordered key for an undirected note pair. */ + private makePairKey(a: string, b: string): string { + return a < b ? `${a}::${b}` : `${b}::${a}`; + } + + /** Maps each note to its tags, excluding organizational tags shared by too much of the vault to be a meaningful signal. */ + private buildTagMap(notes: Note[]): Map> { + const organizationalTags = this.findOrganizationalTags(notes); + + const map = new Map>(); + for (const n of notes) { + const meaningfulTags = (n.tags ?? []).filter(t => !organizationalTags.has(t)); + map.set(n.id, new Set(meaningfulTags)); + } + return map; + } + + private findOrganizationalTags(notes: Note[]): Set { + const tagCounts = new Map(); + for (const n of notes) { + for (const t of n.tags ?? []) { + tagCounts.set(t, (tagCounts.get(t) ?? 0) + 1); + } + } + + const threshold = notes.length * ORGANIZATIONAL_TAG_RATIO; + const organizational = new Set(); + for (const [tag, count] of tagCounts) { + if (count > threshold) organizational.add(tag); + } + return organizational; + } + + private buildLinkSet(notes: Note[]): Set { + const set = new Set(); + for (const n of notes) { + for (const link of n.links ?? []) { + set.add(this.makePairKey(n.id, link)); + } + } + return set; + } +} diff --git a/src/services/similarity/ThresholdPresets.ts b/src/services/similarity/ThresholdPresets.ts new file mode 100644 index 0000000..c7e2ed7 --- /dev/null +++ b/src/services/similarity/ThresholdPresets.ts @@ -0,0 +1,26 @@ +/** Only a direct link bypasses this floor — tags alone can never create an edge below it. */ +export const SEMANTIC_FLOOR = 0.3; + +export const DEFAULT_THRESHOLD = 0.5; + +export const TOP_K = 5; + +/** Vault size above which joplin.ai.search() is used instead of O(n²) cosine. */ +export const LARGE_VAULT_THRESHOLD = 300; + +/** Scaled by Jaccard tag overlap between two notes. */ +export const TAG_BONUS = 0.1; + +/** Smaller than TAG_BONUS on purpose — a link already renders its own edge via EdgeFactory, so this only affects redundant semantic edges. */ +export const LINK_BONUS = 0.05; + +/** Tags on more than this fraction of notes (e.g. "inbox") are excluded as organizational noise. */ +export const ORGANIZATIONAL_TAG_RATIO = 0.3; + +/** Score boost when two notes were created within a day of each other. */ +export const TEMPORAL_BONUS_1_DAY = 0.1; + +/** Score boost when two notes were created within a week of each other. */ +export const TEMPORAL_BONUS_7_DAYS = 0.05; + +export const MS_PER_DAY = 1000 * 60 * 60 * 24; From 7695e2a71a37445d6c246516b5a84193fabb7b8b Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sat, 18 Jul 2026 23:25:40 +0530 Subject: [PATCH 2/3] ANG-008:bug fixes and format --- src/data/Database/VectorDatabase.ts | 18 ++- src/data/Database/VectorRepository.test.ts | 28 +++- src/data/Database/VectorRepository.ts | 54 +++++-- src/services/embeddings/Orchestrator.test.ts | 142 +++++++++++++----- src/services/embeddings/Orchestrator.ts | 37 ++--- .../embeddings/ProviderResolver.test.ts | 8 +- src/services/embeddings/ProviderResolver.ts | 5 +- .../Providers/JoplinNativeProvider.test.ts | 12 +- .../Providers/JoplinNativeProvider.ts | 12 +- src/services/graph/GraphBuilder.test.ts | 42 +++--- src/services/graph/GraphBuilder.ts | 4 +- src/services/similarity/EdgeFactory.test.ts | 30 +--- .../similarity/SimilarityEngine.test.ts | 137 +++++++++-------- src/services/similarity/SimilarityEngine.ts | 93 +++++++++--- 14 files changed, 409 insertions(+), 213 deletions(-) diff --git a/src/data/Database/VectorDatabase.ts b/src/data/Database/VectorDatabase.ts index c6780f4..598bc31 100644 --- a/src/data/Database/VectorDatabase.ts +++ b/src/data/Database/VectorDatabase.ts @@ -8,7 +8,11 @@ export interface IVectorDatabase { interface Sqlite3Database { run(sql: string, params: unknown[], callback: (err: Error | null) => void): void; - all(sql: string, params: unknown[], callback: (err: Error | null, rows: unknown[]) => void): void; + all( + sql: string, + params: unknown[], + callback: (err: Error | null, rows: unknown[]) => void + ): void; } /** @@ -31,11 +35,19 @@ export class VectorDatabase implements IVectorDatabase { private db: Sqlite3Database | null = null; private opening: Promise | null = null; - /** Opens (creating if needed) the vector cache database. Safe to call repeatedly. */ + /** + * Opens (creating if needed) the vector cache database. Safe to call + * repeatedly. A failed open is not cached: `opening` is reset on + * rejection so a later call can retry (e.g. after a transient lock), + * instead of every future open() re-awaiting the same stale rejection. + */ public async open(): Promise { if (this.db) return; if (!this.opening) { - this.opening = this.openInternal(); + this.opening = this.openInternal().catch((e) => { + this.opening = null; + throw e; + }); } await this.opening; } diff --git a/src/data/Database/VectorRepository.test.ts b/src/data/Database/VectorRepository.test.ts index 4b50115..177154b 100644 --- a/src/data/Database/VectorRepository.test.ts +++ b/src/data/Database/VectorRepository.test.ts @@ -10,7 +10,10 @@ import { IVectorDatabase } from './VectorDatabase'; class FakeVectorDatabase implements IVectorDatabase { public opened = false; public allCallBatchSizes: number[] = []; - private rows = new Map(); + private rows = new Map< + string, + { note_id: string; model_id: string; updated_time: number; vector: Buffer } + >(); public async open(): Promise { this.opened = true; @@ -21,13 +24,20 @@ class FakeVectorDatabase implements IVectorDatabase { return; // BEGIN TRANSACTION / COMMIT / ROLLBACK } const [noteId, modelId, updatedTime, vector] = params as [string, string, number, Buffer]; - this.rows.set(noteId, { note_id: noteId, model_id: modelId, updated_time: updatedTime, vector }); + this.rows.set(noteId, { + note_id: noteId, + model_id: modelId, + updated_time: updatedTime, + vector, + }); } public async all(_sql: string, params: unknown[]): Promise { const ids = params as string[]; this.allCallBatchSizes.push(ids.length); - const found = ids.map(id => this.rows.get(id)).filter((r): r is NonNullable => !!r); + const found = ids + .map((id) => this.rows.get(id)) + .filter((r): r is NonNullable => !!r); return found as unknown as T[]; } } @@ -74,8 +84,12 @@ describe('VectorRepository', () => { }); it('overwrites the previous entry for the same note ID', async () => { - await repo.saveMany([{ noteId: 'n1', vector: [1, 0], modelId: 'm1', updatedTime: 100 }]); - await repo.saveMany([{ noteId: 'n1', vector: [0, 1], modelId: 'm2', updatedTime: 200 }]); + await repo.saveMany([ + { noteId: 'n1', vector: [1, 0], modelId: 'm1', updatedTime: 100 }, + ]); + await repo.saveMany([ + { noteId: 'n1', vector: [0, 1], modelId: 'm2', updatedTime: 200 }, + ]); const result = await repo.getMany(['n1']); const entry = result.get('n1'); @@ -106,10 +120,10 @@ describe('VectorRepository', () => { }); describe('large vaults', () => { - it('chunks getMany so a single query never exceeds SQLite\'s bound-parameter limit', async () => { + it("chunks getMany so a single query never exceeds SQLite's bound-parameter limit", async () => { const noteIds = Array.from({ length: 1200 }, (_, i) => `n${i}`); await repo.saveMany( - noteIds.map(id => ({ noteId: id, vector: [1, 0], modelId: 'm1', updatedTime: 1 })), + noteIds.map((id) => ({ noteId: id, vector: [1, 0], modelId: 'm1', updatedTime: 1 })) ); const result = await repo.getMany(noteIds); diff --git a/src/data/Database/VectorRepository.ts b/src/data/Database/VectorRepository.ts index 4757521..55c2127 100644 --- a/src/data/Database/VectorRepository.ts +++ b/src/data/Database/VectorRepository.ts @@ -35,6 +35,12 @@ export class VectorRepository implements VectorCache { /** SQLite caps bound parameters per statement (as low as 999 on some builds); stay well under it. */ private static readonly QUERY_BATCH_SIZE = 500; + /** + * Serializes writes: sqlite transactions live on the shared connection, so + * two interleaved saveMany calls would nest BEGIN TRANSACTION and error. + */ + private writeLock: Promise = Promise.resolve(); + public constructor(private readonly db: IVectorDatabase = new VectorDatabase()) {} /** Returns cached vectors for the given note IDs, keyed by note ID. Missing notes are omitted. */ @@ -59,12 +65,22 @@ export class VectorRepository implements VectorCache { return result; } - /** Inserts or updates vectors for the given notes, in a single transaction. */ - public async saveMany(entries: VectorCacheEntry[]): Promise { + /** Inserts or updates vectors for the given notes, in a single transaction. Calls are serialized. */ + public saveMany(entries: VectorCacheEntry[]): Promise { if (entries.length === 0) { - return; + return Promise.resolve(); } + const task = this.writeLock.then(() => this.saveManyInternal(entries)); + // Keep the lock chain alive whether this write succeeds or fails. + this.writeLock = task.then( + () => undefined, + () => undefined + ); + return task; + } + + private async saveManyInternal(entries: VectorCacheEntry[]): Promise { await this.db.open(); await this.db.run('BEGIN TRANSACTION', []); @@ -77,12 +93,23 @@ export class VectorRepository implements VectorCache { model_id = excluded.model_id, updated_time = excluded.updated_time, vector = excluded.vector`, - [entry.noteId, entry.modelId, entry.updatedTime, this.encodeVector(entry.vector)], + [ + entry.noteId, + entry.modelId, + entry.updatedTime, + this.encodeVector(entry.vector), + ] ); } await this.db.run('COMMIT', []); } catch (e) { - await this.db.run('ROLLBACK', []); + // A failed ROLLBACK (e.g. "database is locked") must not mask the + // original write error. + try { + await this.db.run('ROLLBACK', []); + } catch (rollbackError) { + console.error('Vector cache rollback failed after a write error:', rollbackError); + } throw e; } } @@ -91,7 +118,7 @@ export class VectorRepository implements VectorCache { const placeholders = noteIds.map(() => '?').join(','); return this.db.all( `SELECT note_id, model_id, updated_time, vector FROM note_vectors WHERE note_id IN (${placeholders})`, - noteIds, + noteIds ); } @@ -109,13 +136,14 @@ export class VectorRepository implements VectorCache { return Buffer.from(floats.buffer, floats.byteOffset, floats.byteLength); } - /** Decodes a Float32 BLOB back into a plain number array. */ + /** + * Decodes a Float32 BLOB back into a plain number array. Copies the bytes + * first: Node pools small Buffers at arbitrary byte offsets, and viewing + * an unaligned offset with `new Float32Array(buffer, byteOffset, …)` + * throws a RangeError. + */ private decodeVector(blob: Buffer): number[] { - const floats = new Float32Array( - blob.buffer, - blob.byteOffset, - blob.byteLength / Float32Array.BYTES_PER_ELEMENT, - ); - return Array.from(floats); + const copy = blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength); + return Array.from(new Float32Array(copy)); } } diff --git a/src/services/embeddings/Orchestrator.test.ts b/src/services/embeddings/Orchestrator.test.ts index 366d4c6..85cb386 100644 --- a/src/services/embeddings/Orchestrator.test.ts +++ b/src/services/embeddings/Orchestrator.test.ts @@ -44,7 +44,10 @@ describe('EmbeddingOrchestrator', () => { fetchVectorsByNoteIds: jest.fn().mockResolvedValue(mockVectors), }); - const notes = [makeNote('n1', 'Title 1', 'Body 1'), makeNote('n2', 'Title 2', 'Body 2')]; + const notes = [ + makeNote('n1', 'Title 1', 'Body 1'), + makeNote('n2', 'Title 2', 'Body 2'), + ]; const result = await orchestrator.embedNotes(notes); expect(result.embeddedNotes).toHaveLength(2); @@ -89,10 +92,7 @@ describe('EmbeddingOrchestrator', () => { fetchVectorsByNoteIds: jest.fn().mockResolvedValue(mockVectors), }); - await orchestrator.embedNotes([ - makeNote('n1', 'T1', 'B1'), - makeNote('n2', 'T2', 'B2'), - ]); + await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1'), makeNote('n2', 'T2', 'B2')]); expect(progressUpdates).toEqual([ { current: 0, total: 2 }, @@ -132,11 +132,17 @@ describe('EmbeddingOrchestrator', () => { describe('vector caching', () => { it('serves an unchanged, same-model note from the cache without calling the provider', async () => { const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -148,12 +154,20 @@ describe('EmbeddingOrchestrator', () => { }); it('re-fetches a note whose updated_time no longer matches the cached entry', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -168,11 +182,17 @@ describe('EmbeddingOrchestrator', () => { // queued for re-fetch — but Joplin's AI index hasn't caught up yet and // returns nothing for it. The stale pre-edit vector must not be used. const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -180,16 +200,26 @@ describe('EmbeddingOrchestrator', () => { expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); expect(result.embeddedNotes).toHaveLength(0); - expect(result.errors).toEqual([{ noteId: 'n1', error: 'Note not yet indexed by Joplin AI.' }]); + expect(result.errors).toEqual([ + { noteId: 'n1', error: 'Note not yet indexed by Joplin AI.' }, + ]); }); it('re-fetches a note whose cached entry belongs to a different embedding model', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm2', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm2', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -200,12 +230,20 @@ describe('EmbeddingOrchestrator', () => { }); it('only asks the provider for stale/missing notes, mixing in cache hits', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n2', [0.4, 0.5]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n2', [0.4, 0.5]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 10 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 10 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -220,9 +258,15 @@ describe('EmbeddingOrchestrator', () => { }); it('saves freshly fetched vectors back to the cache with the note updated_time and model', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); const saveMany = jest.fn().mockResolvedValue(undefined); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany }); await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); @@ -235,7 +279,11 @@ describe('EmbeddingOrchestrator', () => { it('does not save notes the provider failed to return', async () => { const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); const saveMany = jest.fn().mockResolvedValue(undefined); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany }); await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); @@ -244,8 +292,14 @@ describe('EmbeddingOrchestrator', () => { }); it('falls back to a full fetch when the cache read throws', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ getMany: jest.fn().mockRejectedValue(new Error('disk error')), saveMany: jest.fn().mockResolvedValue(undefined), @@ -258,8 +312,14 @@ describe('EmbeddingOrchestrator', () => { }); it('still returns results when the cache write throws', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany: jest.fn().mockRejectedValue(new Error('disk full')), @@ -272,8 +332,14 @@ describe('EmbeddingOrchestrator', () => { }); it('behaves exactly as before when no cache is set', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts index 130ff9a..f1fc4c8 100644 --- a/src/services/embeddings/Orchestrator.ts +++ b/src/services/embeddings/Orchestrator.ts @@ -1,11 +1,6 @@ import { Note } from '../../data/Types'; import { CachedVector, VectorCache, VectorCacheEntry } from '../../data/Database/VectorRepository'; -import { - EmbeddingProvider, - EmbeddedNote, - EmbeddingResult, - BatchProgress, -} from './Types'; +import { EmbeddingProvider, EmbeddedNote, EmbeddingResult, BatchProgress } from './Types'; export class EmbeddingOrchestrator { private provider: EmbeddingProvider | null = null; @@ -37,7 +32,7 @@ export class EmbeddingOrchestrator { } if (!this.provider) { - const errors = notes.map(n => ({ noteId: n.id, error: 'No provider configured' })); + const errors = notes.map((n) => ({ noteId: n.id, error: 'No provider configured' })); return { embeddedNotes: [], errors }; } @@ -64,7 +59,7 @@ export class EmbeddingOrchestrator { return { embeddedNotes, errors }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); - const errors = notes.map(n => ({ noteId: n.id, error: msg })); + const errors = notes.map((n) => ({ noteId: n.id, error: msg })); return { embeddedNotes: [], errors }; } } @@ -74,15 +69,21 @@ export class EmbeddingOrchestrator { * `updated_time` and model haven't changed and only asking the provider * to (re-)fetch the rest. */ - private async resolveVectors(notes: Note[], provider: EmbeddingProvider): Promise> { + private async resolveVectors( + notes: Note[], + provider: EmbeddingProvider + ): Promise> { const modelId = provider.modelName; const cached = await this.getCachedVectors(notes); - const notesToFetch = notes.filter(n => !this.isFreshCacheHit(cached.get(n.id), n, modelId)); + const notesToFetch = notes.filter( + (n) => !this.isFreshCacheHit(cached.get(n.id), n, modelId) + ); - const fresh = notesToFetch.length > 0 - ? await provider.fetchVectorsByNoteIds(notesToFetch.map(n => n.id)) - : new Map(); + const fresh = + notesToFetch.length > 0 + ? await provider.fetchVectorsByNoteIds(notesToFetch.map((n) => n.id)) + : new Map(); await this.saveFreshVectors(notesToFetch, fresh, modelId); @@ -94,7 +95,7 @@ export class EmbeddingOrchestrator { notes: Note[], cached: Map, fresh: Map, - modelId: string, + modelId: string ): Map { const merged = new Map(); for (const note of notes) { @@ -115,7 +116,7 @@ export class EmbeddingOrchestrator { private async getCachedVectors(notes: Note[]): Promise> { if (!this.cache) return new Map(); try { - return await this.cache.getMany(notes.map(n => n.id)); + return await this.cache.getMany(notes.map((n) => n.id)); } catch (e) { console.error('Vector cache read failed, falling back to a full fetch:', e); return new Map(); @@ -125,13 +126,13 @@ export class EmbeddingOrchestrator { private async saveFreshVectors( notes: Note[], vectors: Map, - modelId: string, + modelId: string ): Promise { if (!this.cache || vectors.size === 0) return; const entries: VectorCacheEntry[] = notes - .filter(n => vectors.has(n.id)) - .map(n => ({ + .filter((n) => vectors.has(n.id)) + .map((n) => ({ noteId: n.id, vector: vectors.get(n.id)!, modelId, diff --git a/src/services/embeddings/ProviderResolver.test.ts b/src/services/embeddings/ProviderResolver.test.ts index 9d9ad58..26c7fa6 100644 --- a/src/services/embeddings/ProviderResolver.test.ts +++ b/src/services/embeddings/ProviderResolver.test.ts @@ -39,7 +39,9 @@ describe('ProviderResolver', () => { it('returns provider when index is ready', async () => { (joplin as any).ai = { - getIndexStatus: jest.fn().mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }), + getIndexStatus: jest + .fn() + .mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }), getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); @@ -49,7 +51,9 @@ describe('ProviderResolver', () => { it('returns provider while the index is still indexing, since search still works with partial data', async () => { (joplin as any).ai = { - getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }), + getIndexStatus: jest + .fn() + .mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }), getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 446d52b..17371ea 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -3,7 +3,6 @@ import { EmbeddingProvider, ProviderConfig } from './Types'; import { JoplinNativeProvider, JoplinAiApi, isIndexUsable } from './providers/JoplinNativeProvider'; export class ProviderResolver { - /** * Resolves the native embedding provider after verifying that Joplin AI is * available and its embedding index is usable. @@ -11,7 +10,9 @@ export class ProviderResolver { public static async resolveWithValidation(): Promise { const joplinAi = joplin.ai as unknown as JoplinAiApi | undefined; if (!joplinAi) { - throw new Error('joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.'); + throw new Error( + 'joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.' + ); } const status = await joplinAi.getIndexStatus(); if (!status || !isIndexUsable(status.state)) { diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts index 99a773b..8e8d67f 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts @@ -30,7 +30,11 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'fresh-model' }); + ai.getIndexStatus.mockResolvedValue({ + ready: true, + state: 'ready', + modelId: 'fresh-model', + }); ai.getEmbeddings.mockResolvedValue({ modelId: 'fresh-model', dimension: 2, @@ -125,7 +129,11 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }); + ai.getIndexStatus.mockResolvedValue({ + ready: false, + state: 'indexing', + modelId: 'test-model', + }); ai.getEmbeddings.mockResolvedValue({ modelId: 'test-model', dimension: 2, diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index 9a13758..8ee2aae 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -42,7 +42,11 @@ export interface JoplinAiApi { getEmbeddings: (options: GetEmbeddingsOptions) => Promise; } -const BLOCKING_STATES: ReadonlySet = new Set(['unavailable', 'disabled', 'preparing']); +const BLOCKING_STATES: ReadonlySet = new Set([ + 'unavailable', + 'disabled', + 'preparing', +]); /** True once the index has enough data to fetch from, even if still indexing. */ export function isIndexUsable(state: AiIndexState | undefined): boolean { @@ -119,7 +123,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { */ private async fetchAllPages( api: JoplinAiApi, - noteIds: string[], + noteIds: string[] ): Promise> { let trackedModelId = await this.requireUsableIndex(api); @@ -130,7 +134,9 @@ export class JoplinNativeProvider implements EmbeddingProvider { while (true) { if (pageCount >= JoplinNativeProvider.MAX_PAGES) { - throw new Error('Too many pages. The embedding index may be in an unexpected state.'); + throw new Error( + 'Too many pages. The embedding index may be in an unexpected state.' + ); } pageCount++; diff --git a/src/services/graph/GraphBuilder.test.ts b/src/services/graph/GraphBuilder.test.ts index 0864f56..1d4c721 100644 --- a/src/services/graph/GraphBuilder.test.ts +++ b/src/services/graph/GraphBuilder.test.ts @@ -9,11 +9,7 @@ jest.mock('../similarity/SimilarityEngine'); const MockEdgeFactory = EdgeFactory as jest.MockedClass; const MockSimilarityEngine = SimilarityEngine as jest.MockedClass; -function note( - id: string, - title: string, - links: string[] = [] -): Note { +function note(id: string, title: string, links: string[] = []): Note { return { id, parent_id: 'p1', @@ -45,9 +41,7 @@ describe('GraphBuilder', () => { }); it('computes degree from edges', () => { - mockEdgeFactory.createEdges.mockReturnValue([ - { source: 'a', target: 'b', type: 'link' }, - ]); + mockEdgeFactory.createEdges.mockReturnValue([{ source: 'a', target: 'b', type: 'link' }]); const notes = [note('a', 'A'), note('b', 'B')]; const result = builder.build(notes); expect(result.nodes[0].data.degree).toBe(1); @@ -83,13 +77,20 @@ describe('GraphBuilder', () => { describe('buildWithSimilarity', () => { it('adds semantic edges computed from embeddings alongside structural edges', async () => { - mockEdgeFactory.createEdges.mockReturnValue([{ source: 'a', target: 'c', type: 'link' }]); + mockEdgeFactory.createEdges.mockReturnValue([ + { source: 'a', target: 'c', type: 'link' }, + ]); mockEdgeFactory.createSemanticEdges.mockReturnValue([ { source: 'a', target: 'b', type: 'semantic' }, ]); - MockSimilarityEngine.mockImplementation(() => ({ - compute: jest.fn().mockResolvedValue([{ source: 'a', target: 'b', score: 0.8 }]), - }) as unknown as SimilarityEngine); + MockSimilarityEngine.mockImplementation( + () => + ({ + compute: jest + .fn() + .mockResolvedValue([{ source: 'a', target: 'b', score: 0.8 }]), + } as unknown as SimilarityEngine) + ); const notes = [note('a', 'A'), note('b', 'B'), note('c', 'C')]; const embeddedNotes = [ @@ -102,17 +103,24 @@ describe('GraphBuilder', () => { expect(mockEdgeFactory.createSemanticEdges).toHaveBeenCalledWith([ { source: 'a', target: 'b', score: 0.8 }, ]); - expect(result.edges).toContainEqual({ data: { source: 'a', target: 'b', type: 'semantic' } }); - expect(result.edges).toContainEqual({ data: { source: 'a', target: 'c', type: 'link' } }); + expect(result.edges).toContainEqual({ + data: { source: 'a', target: 'b', type: 'semantic' }, + }); + expect(result.edges).toContainEqual({ + data: { source: 'a', target: 'c', type: 'link' }, + }); expect(result.edges).toHaveLength(2); }); it('still returns a graph when there are no semantic matches', async () => { mockEdgeFactory.createEdges.mockReturnValue([]); mockEdgeFactory.createSemanticEdges.mockReturnValue([]); - MockSimilarityEngine.mockImplementation(() => ({ - compute: jest.fn().mockResolvedValue([]), - }) as unknown as SimilarityEngine); + MockSimilarityEngine.mockImplementation( + () => + ({ + compute: jest.fn().mockResolvedValue([]), + } as unknown as SimilarityEngine) + ); const notes = [note('a', 'A'), note('b', 'B')]; const result = await builder.buildWithSimilarity(notes, []); diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 88f1526..85aa096 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -27,7 +27,7 @@ export class GraphBuilder { */ public async buildWithSimilarity( notes: Note[], - embeddedNotes: EmbeddedNote[], + embeddedNotes: EmbeddedNote[] ): Promise { const structuralEdges = this.edgeFactory.createEdges(notes); @@ -92,7 +92,7 @@ export class GraphBuilder { private logGraphStats( nodes: Array<{ data: GraphNode }>, visibleEdges: GraphEdge[], - degreeMap: Map, + degreeMap: Map ): void { const connectedIds = new Set(); for (const edge of visibleEdges) { diff --git a/src/services/similarity/EdgeFactory.test.ts b/src/services/similarity/EdgeFactory.test.ts index c753663..61ef6ee 100644 --- a/src/services/similarity/EdgeFactory.test.ts +++ b/src/services/similarity/EdgeFactory.test.ts @@ -1,12 +1,7 @@ import { EdgeFactory } from './EdgeFactory'; import { Note } from '../../data/Types'; -function note( - id: string, - title: string, - links: string[] = [], - tags: string[] = [] -): Note { +function note(id: string, title: string, links: string[] = [], tags: string[] = []): Note { return { id, parent_id: 'p1', @@ -35,26 +30,17 @@ describe('EdgeFactory', () => { }); it('ignores resource links that are not note IDs', () => { - const notes = [ - note('a', 'A', ['resource123']), - note('b', 'B', ['resource123']), - ]; + const notes = [note('a', 'A', ['resource123']), note('b', 'B', ['resource123'])]; expect(factory.createEdges(notes)).toEqual([]); }); it('creates link edge when note body references another note ID', () => { - const edges = factory.createEdges([ - note('a', 'A', ['b']), - note('b', 'B', []), - ]); + const edges = factory.createEdges([note('a', 'A', ['b']), note('b', 'B', [])]); expect(edges).toEqual([{ source: 'a', target: 'b', type: 'link' }]); }); it('creates bidirectional links when notes reference each other', () => { - const edges = factory.createEdges([ - note('a', 'A', ['b']), - note('b', 'B', ['a']), - ]); + const edges = factory.createEdges([note('a', 'A', ['b']), note('b', 'B', ['a'])]); expect(edges).toHaveLength(2); expect(edges).toContainEqual({ source: 'a', target: 'b', type: 'link' }); expect(edges).toContainEqual({ source: 'b', target: 'a', type: 'link' }); @@ -65,9 +51,7 @@ describe('EdgeFactory', () => { note('a', 'A', [], ['shared']), note('b', 'B', [], ['shared']), ]); - expect(edges).toEqual([ - { source: 'a', target: 'b', type: 'tag', tagName: 'shared' }, - ]); + expect(edges).toEqual([{ source: 'a', target: 'b', type: 'tag', tagName: 'shared' }]); }); it('merges multiple shared tag names into one edge', () => { @@ -108,9 +92,7 @@ describe('EdgeFactory', () => { }); it('creates a semantic edge for each positive-score pair', () => { - const edges = factory.createSemanticEdges([ - { source: 'a', target: 'b', score: 0.8 }, - ]); + const edges = factory.createSemanticEdges([{ source: 'a', target: 'b', score: 0.8 }]); expect(edges).toEqual([{ source: 'a', target: 'b', type: 'semantic' }]); }); diff --git a/src/services/similarity/SimilarityEngine.test.ts b/src/services/similarity/SimilarityEngine.test.ts index f9c8f1c..a94d374 100644 --- a/src/services/similarity/SimilarityEngine.test.ts +++ b/src/services/similarity/SimilarityEngine.test.ts @@ -7,7 +7,7 @@ function makeNote( title: string, links: string[] = [], tags: string[] = [], - createdTime = 0, + createdTime = 0 ): Note { return { id, @@ -36,10 +36,7 @@ describe('SimilarityEngine', () => { }); it('returns empty for a single note', async () => { - const engine = new SimilarityEngine( - [makeNote('a', 'A')], - [embed('a', [1, 0, 0])], - ); + const engine = new SimilarityEngine([makeNote('a', 'A')], [embed('a', [1, 0, 0])]); const pairs = await engine.compute(); expect(pairs).toEqual([]); }); @@ -83,9 +80,7 @@ describe('SimilarityEngine', () => { const engine = new SimilarityEngine(notes, embedded); const pairs = await engine.compute(); - const involvesB = pairs.some( - p => p.source === 'b' || p.target === 'b', - ); + const involvesB = pairs.some((p) => p.source === 'b' || p.target === 'b'); expect(involvesB).toBe(false); }); }); @@ -93,11 +88,7 @@ describe('SimilarityEngine', () => { describe('normalization', () => { it('produces scores in [0, 1] range', async () => { const notes = [makeNote('a', 'A'), makeNote('b', 'B'), makeNote('c', 'C')]; - const embedded = [ - embed('a', [1, 0]), - embed('b', [0.95, 0.3]), - embed('c', [0.3, 0.95]), - ]; + const embedded = [embed('a', [1, 0]), embed('b', [0.95, 0.3]), embed('c', [0.3, 0.95])]; const engine = new SimilarityEngine(notes, embedded); const pairs = await engine.compute(); @@ -109,30 +100,29 @@ describe('SimilarityEngine', () => { }); it('gives higher scores to more similar notes', async () => { - const notes = [ - makeNote('a', 'A'), makeNote('b', 'B'), - makeNote('c', 'C'), makeNote('d', 'D'), - ]; + // With the floor applied to the raw score before normalize, whichever + // pair is weakest among the floor survivors normalizes to exactly 0 — + // b-c (raw ~0.589) plays that role here so it doesn't drag a-c down + // with it, letting both a-b and a-c clear the threshold with a-b + // still scoring higher. + const notes = [makeNote('a', 'A'), makeNote('b', 'B'), makeNote('c', 'C')]; const embedded = [ - embed('a', [1, 0]), - embed('b', [0.95, 0.31]), - embed('c', [0.7, 0.71]), - embed('d', [-1, 0]), + embed('a', [1, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0]), + embed('c', [0.8, -0.3, Math.sqrt(0.27)]), ]; const engine = new SimilarityEngine(notes, embedded); const pairs = await engine.compute(); const abScore = pairs.find( - p => - (p.source === 'a' && p.target === 'b') || - (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const acScore = pairs.find( - p => - (p.source === 'a' && p.target === 'c') || - (p.source === 'c' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a') ); expect(abScore).toBeDefined(); @@ -168,14 +158,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const abWithTag = pairs.find( - p => - (p.source === 'a' && p.target === 'b') || - (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const acNoTag = pairs.find( - p => - (p.source === 'a' && p.target === 'c') || - (p.source === 'c' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a') ); expect(abWithTag).toBeDefined(); @@ -216,10 +204,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const ab = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const cd = pairs.find( - p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + (p) => + (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c') ); expect(ab).toBeDefined(); @@ -259,10 +249,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const abSharesProject = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const acSharesOnlyInbox = pairs.find( - p => (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a') ); expect(abSharesProject).toBeDefined(); @@ -288,14 +280,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const abWithLink = pairs.find( - p => - (p.source === 'a' && p.target === 'b') || - (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const acNoLink = pairs.find( - p => - (p.source === 'a' && p.target === 'c') || - (p.source === 'c' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a') ); expect(abWithLink).toBeDefined(); @@ -337,10 +327,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const ab = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const cd = pairs.find( - p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + (p) => + (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c') ); expect(ab).toBeDefined(); @@ -374,10 +366,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const ab = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const cd = pairs.find( - p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + (p) => + (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c') ); expect(ab).toBeDefined(); @@ -411,10 +405,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const ab = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const cd = pairs.find( - p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + (p) => + (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c') ); expect(ab).toBeDefined(); @@ -430,10 +426,7 @@ describe('SimilarityEngine', () => { for (let i = 0; i < 6; i++) { notes.push(makeNote(`n${i}`, `Note ${i}`)); embedded.push( - embed(`n${i}`, [ - Math.cos((i * Math.PI) / 3), - Math.sin((i * Math.PI) / 3), - ]), + embed(`n${i}`, [Math.cos((i * Math.PI) / 3), Math.sin((i * Math.PI) / 3)]) ); } @@ -441,9 +434,7 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); for (const note of notes) { - const edges = pairs.filter( - p => p.source === note.id || p.target === note.id, - ); + const edges = pairs.filter((p) => p.source === note.id || p.target === note.id); expect(edges.length).toBeLessThanOrEqual(5); } }); @@ -472,7 +463,7 @@ describe('SimilarityEngine', () => { const engine = new SimilarityEngine(notes, embedded); const pairs = await engine.compute(); - const aEdges = pairs.filter(p => p.source === 'a' || p.target === 'a'); + const aEdges = pairs.filter((p) => p.source === 'a' || p.target === 'a'); expect(aEdges.length).toBeGreaterThan(5); }); }); @@ -483,10 +474,7 @@ describe('SimilarityEngine', () => { // linked. SEMANTIC_FLOOR must reject them before bonuses or threshold // ever apply — tags alone can never manufacture an edge out of a weak // semantic score. - const notes = [ - makeNote('a', 'A', [], ['shared']), - makeNote('b', 'B', [], ['shared']), - ]; + const notes = [makeNote('a', 'A', [], ['shared']), makeNote('b', 'B', [], ['shared'])]; const embedded = [embed('a', [1, 0]), embed('b', [0, 1])]; const engine = new SimilarityEngine(notes, embedded); @@ -512,4 +500,31 @@ describe('SimilarityEngine', () => { expect(pairs).toEqual([]); }); }); + + describe('floor is applied to raw scores, before normalization', () => { + it('returns zero pairs for a vault of unrelated notes even when normalization runs', async () => { + // Raw dots span 0 to 0.12 (spread >= 0.1, so normalization would run + // and map the best pair to 1.0). Vectors are near-orthogonal with + // only a small "leakage" component on a shared axis, so every + // pairwise raw score stays below SEMANTIC_FLOOR (0.3) — with the + // floor applied on the raw scale, nothing survives. + const notes = [ + makeNote('a', 'A'), + makeNote('b', 'B'), + makeNote('c', 'C'), + makeNote('d', 'D'), + ]; + const embedded = [ + embed('a', [1, 0.05, 0, 0]), + embed('b', [0, 1, 0.08, 0]), + embed('c', [0, 0, 1, 0.12]), + embed('d', [0.03, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toEqual([]); + }); + }); }); diff --git a/src/services/similarity/SimilarityEngine.ts b/src/services/similarity/SimilarityEngine.ts index dd8a839..f9ff86b 100644 --- a/src/services/similarity/SimilarityEngine.ts +++ b/src/services/similarity/SimilarityEngine.ts @@ -29,17 +29,27 @@ export class SimilarityEngine { private readonly createdTimeMap: Map; public constructor(notes: Note[], embeddedNotes: EmbeddedNote[]) { - this.noteIds = notes.map(n => n.id); + this.noteIds = notes.map((n) => n.id); this.vectors = new Map(); for (const en of embeddedNotes) { this.vectors.set(en.note.id, en.embedding); } this.tagMap = this.buildTagMap(notes); this.linkSet = this.buildLinkSet(notes); - this.createdTimeMap = new Map(notes.map(n => [n.id, n.created_time])); + this.createdTimeMap = new Map(notes.map((n) => [n.id, n.created_time])); } - /** Orchestrates the full similarity pipeline: compute → normalize → floor → enrich → threshold → top-K. */ + /** + * Orchestrates the full similarity pipeline: + * compute → floor (raw scores) → normalize → enrich → threshold → top-K. + * + * SEMANTIC_FLOOR is applied to *raw* scores, before normalization. Min-max + * normalization always maps the batch's most-similar pair to exactly 1.0, + * so a post-normalization floor can never reject it — even in a vault of + * completely unrelated notes. Flooring on the raw scale (where 0.3 has an + * absolute meaning) is what actually guarantees that tags alone can never + * manufacture an edge out of a weak semantic score. + */ public async compute(): Promise { if (this.noteIds.length <= 1) { return []; @@ -51,9 +61,14 @@ export class SimilarityEngine { return []; } - const normalized = this.normalize(rawPairs); - const aboveFloor = this.filterBelowFloor(normalized, SEMANTIC_FLOOR); - const enriched = this.addBonusPoints(aboveFloor); + const aboveFloor = this.filterBelowFloor(rawPairs, SEMANTIC_FLOOR); + + if (aboveFloor.length === 0) { + return []; + } + + const normalized = this.normalize(aboveFloor); + const enriched = this.addBonusPoints(normalized); const aboveThreshold = this.filterBelowThreshold(enriched, DEFAULT_THRESHOLD); const topPairs = this.selectTopK(aboveThreshold, TOP_K); @@ -96,6 +111,16 @@ export class SimilarityEngine { * Uses joplin.ai.search({ noteId }) to find candidate pairs via vector index. * Only checks that joplin.ai itself exists — never probes a specific method * property without invoking it (see JoplinNativeProvider.validateAiApi for why). + * + * Score-scale assumption: search relevance scores are treated as raw + * similarity scores and flow through the same floor → normalize pipeline + * as cosine scores. + * + * Failure handling: individual per-note search failures are skipped (a + * partial candidate set is still useful), but if *every* call fails — + * e.g. joplin.ai exists but search doesn't on this Joplin version — we + * fall back to O(n²) cosine instead of silently returning zero pairs. + * Retry/backoff and progress/cancel for this path are ANG-012. */ private async computeSearchPairs(): Promise { const joplinAi = joplin.ai as unknown as @@ -107,6 +132,8 @@ export class SimilarityEngine { const seen = new Set(); const pairs: SimilarityPair[] = []; + let successCount = 0; + let firstError: unknown = null; for (const noteId of this.noteIds) { try { @@ -114,6 +141,7 @@ export class SimilarityEngine { query: { noteId }, relevance: 'normal', }); + successCount++; for (const r of results) { if (!this.vectors.has(r.noteId) || r.noteId === noteId) { @@ -124,17 +152,31 @@ export class SimilarityEngine { if (seen.has(key)) continue; seen.add(key); - const [source, target] = noteId < r.noteId - ? [noteId, r.noteId] - : [r.noteId, noteId]; + const [source, target] = + noteId < r.noteId ? [noteId, r.noteId] : [r.noteId, noteId]; pairs.push({ source, target, score: r.score }); } - } catch { + } catch (e) { + if (firstError === null) { + firstError = e; + console.warn( + 'joplin.ai.search failed for a note; skipping it. First error:', + e + ); + } continue; } } + if (successCount === 0 && this.noteIds.length > 0) { + console.warn( + 'All joplin.ai.search calls failed; falling back to pairwise cosine similarity.', + firstError + ); + return this.computeCosinePairs(); + } + return pairs; } @@ -172,7 +214,8 @@ export class SimilarityEngine { /** Adds shared-tag, direct-link, and temporal-proximity bonuses to each pair's score. */ private addBonusPoints(pairs: SimilarityPair[]): SimilarityPair[] { for (const p of pairs) { - p.score += this.sharedTagBonus(p) + this.directLinkBonus(p) + this.temporalProximityBonus(p); + p.score += + this.sharedTagBonus(p) + this.directLinkBonus(p) + this.temporalProximityBonus(p); } return pairs; } @@ -213,18 +256,19 @@ export class SimilarityEngine { } /** - * Removes pairs below the safety floor — unless the notes are directly - * linked, in which case they're kept and left for the threshold check - * below. SEMANTIC_FLOOR guards against spurious tag-only edges, not - * against edges the user already created explicitly. + * Removes pairs whose *raw* score is below the safety floor — unless the + * notes are directly linked, in which case they're kept and left for the + * threshold check later. Runs before normalization on purpose: the floor + * guards against spurious tag-only edges, which requires an absolute + * scale, not a batch-relative one. */ private filterBelowFloor(pairs: SimilarityPair[], floor: number): SimilarityPair[] { - return pairs.filter(p => p.score >= floor || this.isDirectlyLinked(p)); + return pairs.filter((p) => p.score >= floor || this.isDirectlyLinked(p)); } /** Keeps only pairs whose bonus-boosted score clears the threshold. */ private filterBelowThreshold(pairs: SimilarityPair[], threshold: number): SimilarityPair[] { - return pairs.filter(p => p.score >= threshold); + return pairs.filter((p) => p.score >= threshold); } /** @@ -232,13 +276,18 @@ export class SimilarityEngine { * their union, so a note that many others pick as one of their top-K can * end up with more than K edges. This is the standard k-nearest-neighbor * graph definition and preserves degree as a centrality signal. + * Output pairs are always oriented source < target. */ private selectTopK(pairs: SimilarityPair[], k: number): SimilarityPair[] { const bySource = new Map(); for (const p of pairs) { this.appendPair(bySource, p.source, p); - this.appendPair(bySource, p.target, { source: p.target, target: p.source, score: p.score }); + this.appendPair(bySource, p.target, { + source: p.target, + target: p.source, + score: p.score, + }); } const deduped = new Map(); @@ -250,7 +299,9 @@ export class SimilarityEngine { for (const p of kept) { const key = this.makePairKey(p.source, p.target); if (!deduped.has(key)) { - deduped.set(key, p); + const [source, target] = + p.source < p.target ? [p.source, p.target] : [p.target, p.source]; + deduped.set(key, { source, target, score: p.score }); } } } @@ -261,7 +312,7 @@ export class SimilarityEngine { private appendPair( map: Map, noteId: string, - pair: SimilarityPair, + pair: SimilarityPair ): void { let list = map.get(noteId); if (!list) { @@ -282,7 +333,7 @@ export class SimilarityEngine { const map = new Map>(); for (const n of notes) { - const meaningfulTags = (n.tags ?? []).filter(t => !organizationalTags.has(t)); + const meaningfulTags = (n.tags ?? []).filter((t) => !organizationalTags.has(t)); map.set(n.id, new Set(meaningfulTags)); } return map; From fce2a0fe98d849f154e8ff5a522b4aae6634b3bf Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 19 Jul 2026 00:26:55 +0530 Subject: [PATCH 3/3] ANG-008:few bug fixes --- src/data/Database/VectorDatabase.ts | 13 ++++++++++--- .../JoplinNativeProvider.test.ts | 0 .../JoplinNativeProvider.ts | 0 src/services/similarity/SimilarityEngine.ts | 14 ++++++++------ 4 files changed, 18 insertions(+), 9 deletions(-) rename src/services/embeddings/{Providers => providers}/JoplinNativeProvider.test.ts (100%) rename src/services/embeddings/{Providers => providers}/JoplinNativeProvider.ts (100%) diff --git a/src/data/Database/VectorDatabase.ts b/src/data/Database/VectorDatabase.ts index 598bc31..a3a79b7 100644 --- a/src/data/Database/VectorDatabase.ts +++ b/src/data/Database/VectorDatabase.ts @@ -13,6 +13,7 @@ interface Sqlite3Database { params: unknown[], callback: (err: Error | null, rows: unknown[]) => void ): void; + close(callback?: (err: Error | null) => void): void; } /** @@ -37,15 +38,21 @@ export class VectorDatabase implements IVectorDatabase { /** * Opens (creating if needed) the vector cache database. Safe to call - * repeatedly. A failed open is not cached: `opening` is reset on - * rejection so a later call can retry (e.g. after a transient lock), - * instead of every future open() re-awaiting the same stale rejection. + * repeatedly. A failed open is not cached: both `opening` and `db` are + * reset on rejection so a later call can retry from scratch, instead of + * either re-awaiting the same stale rejection or (if the connection + * itself succeeded but schema creation failed) treating a half-open + * database as ready forever. */ public async open(): Promise { if (this.db) return; if (!this.opening) { this.opening = this.openInternal().catch((e) => { this.opening = null; + if (this.db) { + this.db.close(); + this.db = null; + } throw e; }); } diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts b/src/services/embeddings/providers/JoplinNativeProvider.test.ts similarity index 100% rename from src/services/embeddings/Providers/JoplinNativeProvider.test.ts rename to src/services/embeddings/providers/JoplinNativeProvider.test.ts diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/providers/JoplinNativeProvider.ts similarity index 100% rename from src/services/embeddings/Providers/JoplinNativeProvider.ts rename to src/services/embeddings/providers/JoplinNativeProvider.ts diff --git a/src/services/similarity/SimilarityEngine.ts b/src/services/similarity/SimilarityEngine.ts index f9ff86b..3ec2ada 100644 --- a/src/services/similarity/SimilarityEngine.ts +++ b/src/services/similarity/SimilarityEngine.ts @@ -130,8 +130,7 @@ export class SimilarityEngine { return this.computeCosinePairs(); } - const seen = new Set(); - const pairs: SimilarityPair[] = []; + const pairs = new Map(); let successCount = 0; let firstError: unknown = null; @@ -149,13 +148,16 @@ export class SimilarityEngine { } const key = this.makePairKey(noteId, r.noteId); - if (seen.has(key)) continue; - seen.add(key); + const existing = pairs.get(key); + if (existing) { + existing.score = Math.max(existing.score, r.score); + continue; + } const [source, target] = noteId < r.noteId ? [noteId, r.noteId] : [r.noteId, noteId]; - pairs.push({ source, target, score: r.score }); + pairs.set(key, { source, target, score: r.score }); } } catch (e) { if (firstError === null) { @@ -177,7 +179,7 @@ export class SimilarityEngine { return this.computeCosinePairs(); } - return pairs; + return Array.from(pairs.values()); } /** Dot product of two same-length vectors. */