From 2d311d398df67432c2785cecc91544136a1abcf4 Mon Sep 17 00:00:00 2001 From: only21mil Date: Sat, 31 Oct 2026 23:11:57 +0000 Subject: [PATCH 1/3] fix(server): thread search finds phrases inside quotes and their comments A quote is stored in message text as a serialized t3-citation link, so its text and comment are percent-encoded: a phrase with spaces never matched, and accented, CJK and apostrophe words in a quote never matched through the index. Projected messages now keep a readable search_text, made with assistantCitationsToPlainText, whenever it differs from the stored text. Migration 077 adds the column, recreates the search triggers and backfills messages that hold a citation link; the update trigger re-indexes each one it changes. createMessageSearchTriggers and the doctor rebuild index search_text once the column exists, so migrations 69 and 74, which run before it, keep indexing the stored text, and a later migration that recreates the triggers keeps quote indexing. Thread search matches and builds snippets from the same text. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/ProjectionStore.ts | 12 +- .../src/orchestration-v2/ThreadSearch.test.ts | 138 ++++++++++++++++++ .../src/orchestration-v2/ThreadSearch.ts | 12 +- .../orchestration-v2/messageSearchIndex.ts | 61 ++++++-- apps/server/src/persistence/Migrations.ts | 2 + .../077_ProjectionMessageSearchText.test.ts | 131 +++++++++++++++++ .../077_ProjectionMessageSearchText.ts | 51 +++++++ .../reconcileV2PreviewMigration.test.ts | 2 + 8 files changed, 391 insertions(+), 18 deletions(-) create mode 100644 apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.test.ts create mode 100644 apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.ts diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2e88fdc119081..adebc66223e58 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -90,6 +90,7 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import type * as Statement from "effect/sql/Statement"; +import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp"; import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts"; import { @@ -2416,6 +2417,8 @@ export const layer: Layer.Layer = case "message.updated": { const payloadJson = yield* encodeMessagePayload(event.payload); const payload = parseEncodedPayload(payloadJson); + // Thread search matches this readable text; NULL when it equals the stored text. + const searchText = assistantCitationsToPlainText(event.payload.text); yield* sql` INSERT INTO orchestration_v2_projection_messages ( message_id, @@ -2426,7 +2429,8 @@ export const layer: Layer.Layer = streaming, created_at, updated_at, - payload_json + payload_json, + search_text ) VALUES ( ${event.payload.id}, @@ -2437,7 +2441,8 @@ export const layer: Layer.Layer = ${booleanInt(event.payload.streaming)}, ${stringField(payload, "createdAt")}, ${stringField(payload, "updatedAt")}, - ${payloadJson} + ${payloadJson}, + ${searchText === event.payload.text ? null : searchText} ) ON CONFLICT(message_id) DO UPDATE SET @@ -2448,7 +2453,8 @@ export const layer: Layer.Layer = streaming = excluded.streaming, created_at = excluded.created_at, updated_at = excluded.updated_at, - payload_json = excluded.payload_json + payload_json = excluded.payload_json, + search_text = excluded.search_text `; break; } diff --git a/apps/server/src/orchestration-v2/ThreadSearch.test.ts b/apps/server/src/orchestration-v2/ThreadSearch.test.ts index b8d75340ec6bc..da95e781937a6 100644 --- a/apps/server/src/orchestration-v2/ThreadSearch.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSearch.test.ts @@ -1,5 +1,6 @@ import { assert, it } from "@effect/vitest"; import { + EnvironmentId, EventId, MessageId, ProjectId, @@ -18,9 +19,13 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as SqlClient from "effect/sql/SqlClient"; +import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations"; import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as EventStore from "./EventStore.ts"; +import { rebuildMessageSearch } from "./messageSearchIndex.ts"; +import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as ThreadSearch from "./ThreadSearch.ts"; @@ -150,6 +155,20 @@ const message = ( }, }); +// The prefix and suffix are hidden selection context, not text the reader sees. +const quote = serializeAssistantCitation({ + version: 1, + environmentId: EnvironmentId.make("environment:local"), + threadId: ThreadId.make("thread:quoted"), + messageId: MessageId.make("quoted-answer"), + text: "A lighthouse guides ships safely through the darkness.", + comment: "what are you talking about", + start: 0, + end: 54, + prefix: "beacon01", + suffix: "harbor", +}); + it.effect.each([true, false])( "ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)", (indexed) => @@ -349,6 +368,89 @@ it.layer(layerTest)("ThreadSearch", (it) => { }), ); + it.effect("matches the readable text of quotes and their comments", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const search = yield* ThreadSearch.ThreadSearch; + const project = ProjectId.make("project:search-citation"); + const threadId = ThreadId.make("thread:citation"); + yield* createProject(project); + yield* Effect.forEach( + [ + thread(threadId, project), + message(threadId, "citation", "user", `${quote} quiet silver lantern`), + ], + projections.apply, + { discard: true }, + ); + const snippets = (query: string) => + Effect.map(search.search({ query }), (result) => + result.matches + .filter((match) => match.threadId === threadId) + .map((match) => match.snippet), + ); + + const readable = + "A lighthouse guides ships safely through the darkness. Comment: what are you talking about quiet silver lantern"; + assert.deepEqual(yield* snippets("what are you talking about"), [ + `…${readable.slice(readable.indexOf("the darkness"))}`, + ]); + assert.deepEqual(yield* snippets("ships safely through"), [readable]); + assert.deepEqual(yield* snippets("beacon01"), []); + assert.deepEqual(yield* snippets("harbor"), []); + + yield* projections.apply(message(threadId, "citation", "user", "quiet silver lantern")); + assert.deepEqual(yield* snippets("what are you talking about"), []); + assert.deepEqual(yield* snippets("lantern"), ["quiet silver lantern"]); + }), + ); + + it.effect("finds accented, CJK, and apostrophe words of a quote through the index", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const search = yield* ThreadSearch.ThreadSearch; + const project = ProjectId.make("project:search-citation-index"); + const threadId = ThreadId.make("thread:citation-index"); + // The stored link percent-encodes these words; the index must hold them as they read. + const encodedQuote = serializeAssistantCitation({ + version: 1, + environmentId: EnvironmentId.make("environment:local"), + threadId: ThreadId.make("thread:quoted"), + messageId: MessageId.make("quoted-answer"), + text: "The kittiwake's résumé mentions 灯台の光 twice.", + comment: "naïve question", + start: 0, + end: 44, + prefix: "", + suffix: "", + }); + yield* createProject(project); + yield* Effect.forEach( + [thread(threadId, project), message(threadId, "citation-index", "user", encodedQuote)], + projections.apply, + { discard: true }, + ); + const index = yield* sql` + SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' + `; + assert.lengthOf(index, 1); + const threadIds = (query: string) => + search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); + + for (const query of ["kittiwake's", "kittiwake’s", "résumé", "灯台の光", "naïve"]) { + assert.deepEqual(yield* threadIds(query), [threadId], query); + } + // `t3 doctor` rebuilds the index after VACUUM; quotes must still read. + yield* rebuildMessageSearch(sql); + assert.deepEqual(yield* threadIds("résumé"), [threadId]); + + yield* projections.apply(message(threadId, "citation-index", "user", "plain words")); + assert.deepEqual(yield* threadIds("résumé"), []); + assert.deepEqual(yield* threadIds("plain"), [threadId]); + }), + ); + it.effect("indexes a message when it finishes streaming and follows later edits", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; @@ -1115,6 +1217,42 @@ it.effect("starts snippets a few words before the match so one line shows it", ( }), ); +it.effect("finds quote text after projections replay from events", () => + Effect.gen(function* () { + const eventStore = yield* EventStore.EventStoreV2; + const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + const search = yield* ThreadSearch.ThreadSearch; + const project = ProjectId.make("project:search-replay"); + const threadId = ThreadId.make("thread:replay"); + yield* createProject(project); + yield* eventStore.append({ + events: [thread(threadId, project), message(threadId, "replay", "user", quote)], + }); + + assert.isTrue((yield* maintenance.rebuild).valid); + assert.deepEqual( + (yield* search.search({ query: "what are you talking about" })).matches.map( + (match) => match.threadId, + ), + [threadId], + ); + }).pipe( + Effect.provide( + ProjectionMaintenance.layer.pipe( + Layer.provideMerge( + Layer.mergeAll( + ThreadSearch.layer, + ProjectionStore.layer, + ProjectStore.layer, + EventStore.layer, + ), + ), + Layer.provideMerge(SqlitePersistence.layerMemory), + ), + ), + ), +); + it.effect("searches on the reader worker while the writer is busy", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/orchestration-v2/ThreadSearch.ts b/apps/server/src/orchestration-v2/ThreadSearch.ts index 3c97122c1c63e..2f355398ee6c0 100644 --- a/apps/server/src/orchestration-v2/ThreadSearch.ts +++ b/apps/server/src/orchestration-v2/ThreadSearch.ts @@ -33,7 +33,11 @@ import { type ThreadSearchSubject, } from "@t3tools/shared/paletteSearch"; import { threadSearchPullRequests } from "@t3tools/shared/threadPullRequests"; -import { indexedMessageCondition, nonPlaceholderMessageCondition } from "./messageSearchIndex.ts"; +import { + indexedMessageCondition, + messageSearchText, + nonPlaceholderMessageCondition, +} from "./messageSearchIndex.ts"; import { loadOlderSettledThreadsPage } from "./ShellStream.ts"; // When the user last wrote in the thread aliased `threads`, as the clients @@ -185,7 +189,7 @@ export const messageHitsQuery = ( messages.thread_id AS "threadId", eligible.project_id AS "projectId", messages.role AS source, - json_extract(messages.payload_json, '$.text') AS text, + ${messageSearchText(sql)} AS text, messages.created_at AS "messageCreatedAt" FROM orchestration_v2_projection_messages AS messages INNER JOIN eligible ON eligible.thread_id = messages.thread_id @@ -202,7 +206,7 @@ export const messageHitsQuery = ( messages.thread_id AS "threadId", eligible.project_id AS "projectId", messages.role AS source, - json_extract(messages.payload_json, '$.text') AS text, + ${messageSearchText(sql)} AS text, messages.created_at AS "messageCreatedAt" FROM orchestration_v2_message_search INNER JOIN orchestration_v2_projection_messages AS messages @@ -223,7 +227,7 @@ export const messageHitsQuery = ( AND messages.message_id NOT LIKE 'async-answer:%' AND NOT EXISTS ( SELECT 1 FROM json_each(${substrings}) AS token - WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0 + WHERE instr(lower(${messageSearchText(sql)}), token.value) = 0 ) -- The index's own rowid, so FTS5 returns hits newest first and stops at -- the limit; ordering by the messages table's rowid sorts every hit. diff --git a/apps/server/src/orchestration-v2/messageSearchIndex.ts b/apps/server/src/orchestration-v2/messageSearchIndex.ts index dbd3545e5906b..2f84078f6cf15 100644 --- a/apps/server/src/orchestration-v2/messageSearchIndex.ts +++ b/apps/server/src/orchestration-v2/messageSearchIndex.ts @@ -40,34 +40,71 @@ export const indexedMessageCondition = (sql: SqlClient.SqlClient, row = "message AND ${nonPlaceholderMessageCondition(sql, row)} `; +// Curly apostrophes straight, as search folds a query, since iOS keyboards +// type "’" and others type "'". +const foldApostrophes = (text: string) => `replace(replace(${text}, '’', ''''), '‘', '''')`; + +const readableText = (row: string) => + `COALESCE(${row}.search_text, json_extract(${row}.payload_json, '$.text'))`; + +/** + * A message's text as it reads: migration 77's search_text where a quote's + * percent-encoded link reads differently, else the stored text. `row` names + * the message row. + */ +export const messageSearchText = (sql: SqlClient.SqlClient, row = "messages") => + sql.literal(readableText(row)); + /** - * What the index stores for a message: its text with curly apostrophes - * straight, as search folds a query, since iOS keyboards type "’" and others - * type "'". Migration 69's triggers use it too. `payload` is the message's + * A message's stored text with apostrophes folded, what the index held before + * migration 77. Migration 69's backfill uses it. `payload` is the message's * JSON column. */ export const indexedMessageText = (sql: SqlClient.SqlClient, payload: string) => - sql.literal(`replace(replace(json_extract(${payload}, '$.text'), '’', ''''), '‘', '''')`); + sql.literal(foldApostrophes(`json_extract(${payload}, '$.text')`)); + +// What the index stores: a message's readable text with apostrophes folded. +// Migrations 69 and 74 create the triggers before migration 77 adds +// search_text, and `t3 doctor` can rebuild a database that has not migrated, +// so those fall back to the stored text. +const indexedText = Effect.fn("MessageSearchIndex.indexedText")(function* ( + sql: SqlClient.SqlClient, +) { + const column = yield* sql` + SELECT 1 FROM pragma_table_info('orchestration_v2_projection_messages') + WHERE name = 'search_text' + `; + const readable = column.length > 0; + return { + readable, + of: (row: string) => + readable + ? sql.literal(foldApostrophes(readableText(row))) + : indexedMessageText(sql, `${row}.payload_json`), + }; +}); export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createTriggers")( function* (sql: SqlClient.SqlClient) { + const text = yield* indexedText(sql); yield* sql` CREATE TRIGGER orchestration_v2_message_search_insert AFTER INSERT ON orchestration_v2_projection_messages WHEN ${indexedMessageCondition(sql, "new")} BEGIN INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")} + SELECT new.rowid, ${text.of("new")} WHERE json_extract(new.payload_json, '$.text') IS NOT NULL; END `; // Streaming deltas rewrite the row many times a second; skip them until the - // message settles. Upserts rewrite every column, so a finished message - // re-indexes only when what the index reads changed. Deleting a rowid the - // index never held is a no-op. + // message settles. Upserts rewrite every column, so re-index only when what + // the index reads changed, search_text included for migration 77's backfill. + // Deleting a rowid the index never held is a no-op. yield* sql` CREATE TRIGGER orchestration_v2_message_search_update - AFTER UPDATE OF payload_json, streaming, role ON orchestration_v2_projection_messages + AFTER UPDATE OF payload_json, streaming, role${sql.literal(text.readable ? ", search_text" : "")} + ON orchestration_v2_projection_messages WHEN (old.streaming = 0 OR new.streaming = 0) AND ( old.streaming IS NOT new.streaming @@ -75,11 +112,12 @@ export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createT OR json_extract(old.payload_json, '$.text') IS NOT json_extract(new.payload_json, '$.text') OR json_extract(old.payload_json, '$.createdBy') IS NOT json_extract(new.payload_json, '$.createdBy') + ${sql.literal(text.readable ? "OR old.search_text IS NOT new.search_text" : "")} ) BEGIN DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid AND old.streaming = 0; INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")} + SELECT new.rowid, ${text.of("new")} WHERE ${indexedMessageCondition(sql, "new")} AND json_extract(new.payload_json, '$.text') IS NOT NULL; END @@ -108,6 +146,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' `; if (index.length === 0) return; + const text = yield* indexedText(sql); // One transaction: an interrupt between clear and refill would leave an // empty index that nothing repairs, since search only checks the table exists. yield* sql.withTransaction( @@ -118,7 +157,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func `; yield* sql` INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT messages.rowid, ${indexedMessageText(sql, "messages.payload_json")} + SELECT messages.rowid, ${text.of("messages")} FROM orchestration_v2_projection_messages AS messages WHERE ${indexedMessageCondition(sql)} AND json_extract(messages.payload_json, '$.text') IS NOT NULL diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index e13c2b449d9e7..c2a2db6bce7d4 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -79,6 +79,7 @@ import Migration0073 from "./Migrations/073_ActiveTurnItemIndexes.ts"; import Migration0074 from "./Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts"; import Migration0075 from "./Migrations/075_SubagentDeliveryState.ts"; import Migration0076 from "./Migrations/076_ProjectionProjectsPinArchive.ts"; +import Migration0077 from "./Migrations/077_ProjectionMessageSearchText.ts"; const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -158,6 +159,7 @@ const migrationEntries = [ [74, "ExcludeAttachmentPlaceholdersFromSearch", Migration0074], [75, "SubagentDeliveryState", Migration0075], [76, "ProjectionProjectsPinArchive", Migration0076], + [77, "ProjectionMessageSearchText", Migration0077], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.test.ts b/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.test.ts new file mode 100644 index 0000000000000..2a70d4624fd0d --- /dev/null +++ b/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.test.ts @@ -0,0 +1,131 @@ +import { assert, it } from "@effect/vitest"; +import { EnvironmentId, MessageId, ThreadId } from "@t3tools/contracts"; +import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +import { createMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; +import { runMigrations } from "../Migrations.ts"; + +const createdAt = "2026-10-01T00:00:00.000Z"; +const quote = serializeAssistantCitation({ + version: 1, + environmentId: EnvironmentId.make("environment:local"), + threadId: ThreadId.make("thread:quoted"), + messageId: MessageId.make("quoted-answer"), + text: "A lighthouse guides ships safely through the darkness.", + comment: "what are you talking about", + start: 0, + end: 54, + prefix: "beacon01", + suffix: "harbor", +}); +const readable = + "A lighthouse guides ships safely through the darkness.\nComment: what are you talking about"; + +it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))( + "077_ProjectionMessageSearchText", + (it) => { + it.effect("backfills readable text only for messages whose citations read differently", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations({ toMigrationInclusive: 76 }); + const insertMessage = (messageId: string, text: unknown) => sql` + INSERT INTO orchestration_v2_projection_messages ( + message_id, thread_id, role, streaming, created_at, updated_at, payload_json + ) VALUES ( + ${messageId}, 'thread:quote', 'user', 0, ${createdAt}, ${createdAt}, + ${JSON.stringify({ text })} + ) + `; + yield* insertMessage("quoted", `${quote} quiet silver lantern`); + yield* insertMessage("plain", "quiet silver lantern"); + yield* insertMessage("invalid", "[Assistant quote](t3-citation://v1/not-a-citation)"); + yield* insertMessage("unreadable", 20261001); + yield* insertMessage( + "accented", + serializeAssistantCitation({ + version: 1, + environmentId: EnvironmentId.make("environment:local"), + threadId: ThreadId.make("thread:quoted"), + messageId: MessageId.make("quoted-answer"), + text: "The kittiwake's résumé", + start: 0, + end: 22, + prefix: "", + suffix: "", + }), + ); + yield* Effect.forEach( + Array.from({ length: 501 }, (_, index) => `page-${String(index).padStart(3, "0")}`), + (messageId) => insertMessage(messageId, quote), + { discard: true }, + ); + + assert.deepEqual(yield* runMigrations({ toMigrationInclusive: 77 }), [ + [77, "ProjectionMessageSearchText"], + ]); + const rows = yield* sql<{ + readonly message_id: string; + readonly search_text: string | null; + }>`SELECT message_id, search_text FROM orchestration_v2_projection_messages`; + const searchText = new Map(rows.map((row) => [row.message_id, row.search_text])); + assert.equal(searchText.get("quoted"), `${readable} quiet silver lantern`); + assert.isNull(searchText.get("plain")); + assert.isNull(searchText.get("invalid")); + assert.isNull(searchText.get("unreadable")); + assert.equal(searchText.get("page-000"), readable); + assert.equal(searchText.get("page-500"), readable); + + // Messages indexed as stored are indexed again as they read. + const indexed = (phrase: string) => + sql<{ readonly message_id: string }>` + SELECT messages.message_id + FROM orchestration_v2_message_search AS search + INNER JOIN orchestration_v2_projection_messages AS messages + ON messages.rowid = search.rowid + WHERE orchestration_v2_message_search MATCH ${phrase} + `.pipe(Effect.map((rows) => rows.map((row) => row.message_id))); + assert.deepEqual(yield* indexed('"résumé"'), ["accented"]); + assert.deepEqual(yield* indexed(`"kittiwake's"`), ["accented"]); + assert.deepEqual(yield* indexed('"beacon01"'), []); + assert.lengthOf(yield* indexed('"what are you talking about"'), 502); + }), + ); + + it.effect("keeps indexing readable text when a later migration recreates the triggers", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations(); + // As migration 74 does. + yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; + yield* createMessageSearchTriggers(sql); + + yield* sql` + INSERT INTO orchestration_v2_projection_messages ( + message_id, thread_id, role, streaming, created_at, updated_at, payload_json, + search_text + ) VALUES ( + 'recreated', 'thread:quote', 'user', 0, ${createdAt}, ${createdAt}, + ${JSON.stringify({ text: `${quote} lantern recreated` })}, + ${`${readable} lantern recreated`} + ) + `; + const indexed = yield* sql<{ readonly message_id: string }>` + SELECT messages.message_id + FROM orchestration_v2_message_search AS search + INNER JOIN orchestration_v2_projection_messages AS messages + ON messages.rowid = search.rowid + WHERE orchestration_v2_message_search MATCH '"talking about lantern recreated"' + `; + assert.deepEqual( + indexed.map((row) => row.message_id), + ["recreated"], + ); + }), + ); + }, +); diff --git a/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.ts b/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.ts new file mode 100644 index 0000000000000..0d60f4c57e54f --- /dev/null +++ b/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.ts @@ -0,0 +1,51 @@ +import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +import { createMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + // Readable text thread search matches; NULL when it equals the stored text. + yield* sql`ALTER TABLE orchestration_v2_projection_messages ADD COLUMN search_text TEXT`; + + // The triggers from migration 74 index the stored text, where a quote's + // words are percent-encoded. Recreated now that search_text exists, they + // index it instead, and the backfill below re-indexes, through the update + // trigger, every message that reads differently. + const index = yield* sql` + SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' + `; + if (index.length > 0) { + yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; + yield* createMessageSearchTriggers(sql); + } + + // Only messages holding a citation link read differently from their stored text. + const pageSize = 500; + let afterMessageId = ""; + while (true) { + const rows = yield* sql<{ readonly message_id: string; readonly text: string }>` + SELECT message_id, json_extract(payload_json, '$.text') AS text + FROM orchestration_v2_projection_messages + WHERE message_id > ${afterMessageId} + AND json_type(payload_json, '$.text') = 'text' + AND json_extract(payload_json, '$.text') LIKE '%t3-citation://%' + ORDER BY message_id + LIMIT ${pageSize} + `; + for (const row of rows) { + const searchText = assistantCitationsToPlainText(row.text); + if (searchText === row.text) continue; + yield* sql` + UPDATE orchestration_v2_projection_messages + SET search_text = ${searchText} + WHERE message_id = ${row.message_id} + `; + } + if (rows.length < pageSize) break; + afterMessageId = rows.at(-1)!.message_id; + } +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index 08c5094bf49c8..ddb19d2d797d6 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -56,6 +56,7 @@ describe("V2 preview upgrade", () => { [74, "ExcludeAttachmentPlaceholdersFromSearch"], [75, "SubagentDeliveryState"], [76, "ProjectionProjectsPinArchive"], + [77, "ProjectionMessageSearchText"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -155,6 +156,7 @@ describe("V2 preview upgrade", () => { [74, "ExcludeAttachmentPlaceholdersFromSearch"], [75, "SubagentDeliveryState"], [76, "ProjectionProjectsPinArchive"], + [77, "ProjectionMessageSearchText"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); From 386e0ff217ac04a5bda5c1b5f1b472bb83916a20 Mon Sep 17 00:00:00 2001 From: only21mil Date: Sat, 31 Oct 2026 22:41:48 +0000 Subject: [PATCH 2/3] refactor(server): share search trigger recreation and the search_text rule - recreateMessageSearchTriggers replaces the drop-and-recreate block that migrations 74 and 77 each wrote out. - readableSearchText is the one rule for what search_text holds, used by the projection and the backfill. - The update trigger compares the text the index stores, so it has one shape before and after migration 77 and still re-indexes the backfill. - The backfill checks the raw payload for a citation link before parsing its JSON. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/ProjectionStore.ts | 6 +- .../src/orchestration-v2/ThreadSearch.test.ts | 11 +--- .../orchestration-v2/messageSearchIndex.ts | 56 ++++++++++++------- ...ExcludeAttachmentPlaceholdersFromSearch.ts | 12 +--- .../077_ProjectionMessageSearchText.test.ts | 44 +++++---------- .../077_ProjectionMessageSearchText.ts | 33 +++++------ 6 files changed, 69 insertions(+), 93 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index adebc66223e58..0c400b39b91a1 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -90,9 +90,9 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import type * as Statement from "effect/sql/Statement"; -import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp"; import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts"; +import { readableSearchText } from "./messageSearchIndex.ts"; import { isThreadHistoryUserTurn, isConversationHistoryItem, @@ -2417,8 +2417,6 @@ export const layer: Layer.Layer = case "message.updated": { const payloadJson = yield* encodeMessagePayload(event.payload); const payload = parseEncodedPayload(payloadJson); - // Thread search matches this readable text; NULL when it equals the stored text. - const searchText = assistantCitationsToPlainText(event.payload.text); yield* sql` INSERT INTO orchestration_v2_projection_messages ( message_id, @@ -2442,7 +2440,7 @@ export const layer: Layer.Layer = ${stringField(payload, "createdAt")}, ${stringField(payload, "updatedAt")}, ${payloadJson}, - ${searchText === event.payload.text ? null : searchText} + ${readableSearchText(event.payload.text)} ) ON CONFLICT(message_id) DO UPDATE SET diff --git a/apps/server/src/orchestration-v2/ThreadSearch.test.ts b/apps/server/src/orchestration-v2/ThreadSearch.test.ts index da95e781937a6..8ad59695ffa2f 100644 --- a/apps/server/src/orchestration-v2/ThreadSearch.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSearch.test.ts @@ -1239,15 +1239,8 @@ it.effect("finds quote text after projections replay from events", () => }).pipe( Effect.provide( ProjectionMaintenance.layer.pipe( - Layer.provideMerge( - Layer.mergeAll( - ThreadSearch.layer, - ProjectionStore.layer, - ProjectStore.layer, - EventStore.layer, - ), - ), - Layer.provideMerge(SqlitePersistence.layerMemory), + Layer.provideMerge(EventStore.layer), + Layer.provideMerge(layerTest), ), ), ), diff --git a/apps/server/src/orchestration-v2/messageSearchIndex.ts b/apps/server/src/orchestration-v2/messageSearchIndex.ts index 2f84078f6cf15..7ed8f3bbaf469 100644 --- a/apps/server/src/orchestration-v2/messageSearchIndex.ts +++ b/apps/server/src/orchestration-v2/messageSearchIndex.ts @@ -1,3 +1,4 @@ +import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; import * as Effect from "effect/Effect"; import type * as SqlClient from "effect/sql/SqlClient"; @@ -48,12 +49,18 @@ const readableText = (row: string) => `COALESCE(${row}.search_text, json_extract(${row}.payload_json, '$.text'))`; /** - * A message's text as it reads: migration 77's search_text where a quote's - * percent-encoded link reads differently, else the stored text. `row` names - * the message row. + * What search_text holds for a message: its text as it reads, a quote's + * percent-encoded link replaced by its words and comment. NULL when that is + * the stored text, so most messages store nothing extra. */ -export const messageSearchText = (sql: SqlClient.SqlClient, row = "messages") => - sql.literal(readableText(row)); +export const readableSearchText = (text: string) => { + const readable = assistantCitationsToPlainText(text); + return readable === text ? null : readable; +}; + +/** A message's text as it reads, for thread search. */ +export const messageSearchText = (sql: SqlClient.SqlClient) => + sql.literal(readableText("messages")); /** * A message's stored text with apostrophes folded, what the index held before @@ -74,14 +81,9 @@ const indexedText = Effect.fn("MessageSearchIndex.indexedText")(function* ( SELECT 1 FROM pragma_table_info('orchestration_v2_projection_messages') WHERE name = 'search_text' `; - const readable = column.length > 0; - return { - readable, - of: (row: string) => - readable - ? sql.literal(foldApostrophes(readableText(row))) - : indexedMessageText(sql, `${row}.payload_json`), - }; + return column.length > 0 + ? (row: string) => sql.literal(foldApostrophes(readableText(row))) + : (row: string) => indexedMessageText(sql, `${row}.payload_json`); }); export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createTriggers")( @@ -93,31 +95,29 @@ export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createT WHEN ${indexedMessageCondition(sql, "new")} BEGIN INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT new.rowid, ${text.of("new")} + SELECT new.rowid, ${text("new")} WHERE json_extract(new.payload_json, '$.text') IS NOT NULL; END `; // Streaming deltas rewrite the row many times a second; skip them until the // message settles. Upserts rewrite every column, so re-index only when what - // the index reads changed, search_text included for migration 77's backfill. + // the index stores changed; migration 77's backfill writes only search_text. // Deleting a rowid the index never held is a no-op. yield* sql` CREATE TRIGGER orchestration_v2_message_search_update - AFTER UPDATE OF payload_json, streaming, role${sql.literal(text.readable ? ", search_text" : "")} - ON orchestration_v2_projection_messages + AFTER UPDATE ON orchestration_v2_projection_messages WHEN (old.streaming = 0 OR new.streaming = 0) AND ( old.streaming IS NOT new.streaming OR old.role IS NOT new.role - OR json_extract(old.payload_json, '$.text') IS NOT json_extract(new.payload_json, '$.text') + OR ${text("old")} IS NOT ${text("new")} OR json_extract(old.payload_json, '$.createdBy') IS NOT json_extract(new.payload_json, '$.createdBy') - ${sql.literal(text.readable ? "OR old.search_text IS NOT new.search_text" : "")} ) BEGIN DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid AND old.streaming = 0; INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT new.rowid, ${text.of("new")} + SELECT new.rowid, ${text("new")} WHERE ${indexedMessageCondition(sql, "new")} AND json_extract(new.payload_json, '$.text') IS NOT NULL; END @@ -133,6 +133,20 @@ export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createT }, ); +/** Replaces the search triggers with their current definition, where the index exists. */ +export const recreateMessageSearchTriggers = Effect.fn("MessageSearchIndex.recreateTriggers")( + function* (sql: SqlClient.SqlClient) { + const index = yield* sql` + SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' + `; + if (index.length === 0) return; + yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; + yield* createMessageSearchTriggers(sql); + }, +); + /** * Refills the search index from the messages table. The index is keyed by * message rowid, so anything that may renumber or copy rows (VACUUM, a dev @@ -157,7 +171,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func `; yield* sql` INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT messages.rowid, ${text.of("messages")} + SELECT messages.rowid, ${text("messages")} FROM orchestration_v2_projection_messages AS messages WHERE ${indexedMessageCondition(sql)} AND json_extract(messages.payload_json, '$.text') IS NOT NULL diff --git a/apps/server/src/persistence/Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts b/apps/server/src/persistence/Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts index 6ec95d108f306..98b2d816cc2fe 100644 --- a/apps/server/src/persistence/Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts +++ b/apps/server/src/persistence/Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts @@ -1,18 +1,10 @@ import * as Effect from "effect/Effect"; import * as SqlClient from "effect/sql/SqlClient"; -import { createMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; +import { recreateMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; export default Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - const index = yield* sql` - SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' - `; - if (index.length === 0) return; - // Legacy placeholders stay indexed until rebuild; FTS deletes can overflow last_insert_rowid at commit. - yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; - yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; - yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; - yield* createMessageSearchTriggers(sql); + yield* recreateMessageSearchTriggers(sql); }); diff --git a/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.test.ts b/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.test.ts index 2a70d4624fd0d..89b178837ed10 100644 --- a/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.test.ts +++ b/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.test.ts @@ -5,7 +5,7 @@ import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Effect from "effect/Effect"; import * as SqlClient from "effect/sql/SqlClient"; -import { createMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; +import { recreateMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; import { runMigrations } from "../Migrations.ts"; const createdAt = "2026-10-01T00:00:00.000Z"; @@ -23,6 +23,14 @@ const quote = serializeAssistantCitation({ }); const readable = "A lighthouse guides ships safely through the darkness.\nComment: what are you talking about"; +const indexed = (sql: SqlClient.SqlClient, phrase: string) => + sql<{ readonly message_id: string }>` + SELECT messages.message_id + FROM orchestration_v2_message_search AS search + INNER JOIN orchestration_v2_projection_messages AS messages + ON messages.rowid = search.rowid + WHERE orchestration_v2_message_search MATCH ${phrase} + `.pipe(Effect.map((rows) => rows.map((row) => row.message_id))); it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))( "077_ProjectionMessageSearchText", @@ -79,18 +87,10 @@ it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))( assert.equal(searchText.get("page-500"), readable); // Messages indexed as stored are indexed again as they read. - const indexed = (phrase: string) => - sql<{ readonly message_id: string }>` - SELECT messages.message_id - FROM orchestration_v2_message_search AS search - INNER JOIN orchestration_v2_projection_messages AS messages - ON messages.rowid = search.rowid - WHERE orchestration_v2_message_search MATCH ${phrase} - `.pipe(Effect.map((rows) => rows.map((row) => row.message_id))); - assert.deepEqual(yield* indexed('"résumé"'), ["accented"]); - assert.deepEqual(yield* indexed(`"kittiwake's"`), ["accented"]); - assert.deepEqual(yield* indexed('"beacon01"'), []); - assert.lengthOf(yield* indexed('"what are you talking about"'), 502); + assert.deepEqual(yield* indexed(sql, '"résumé"'), ["accented"]); + assert.deepEqual(yield* indexed(sql, `"kittiwake's"`), ["accented"]); + assert.deepEqual(yield* indexed(sql, '"beacon01"'), []); + assert.lengthOf(yield* indexed(sql, '"what are you talking about"'), 502); }), ); @@ -98,11 +98,7 @@ it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))( Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; yield* runMigrations(); - // As migration 74 does. - yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; - yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; - yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; - yield* createMessageSearchTriggers(sql); + yield* recreateMessageSearchTriggers(sql); yield* sql` INSERT INTO orchestration_v2_projection_messages ( @@ -114,17 +110,7 @@ it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))( ${`${readable} lantern recreated`} ) `; - const indexed = yield* sql<{ readonly message_id: string }>` - SELECT messages.message_id - FROM orchestration_v2_message_search AS search - INNER JOIN orchestration_v2_projection_messages AS messages - ON messages.rowid = search.rowid - WHERE orchestration_v2_message_search MATCH '"talking about lantern recreated"' - `; - assert.deepEqual( - indexed.map((row) => row.message_id), - ["recreated"], - ); + assert.deepEqual(yield* indexed(sql, '"talking about lantern recreated"'), ["recreated"]); }), ); }, diff --git a/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.ts b/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.ts index 0d60f4c57e54f..4bbcec4d61822 100644 --- a/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.ts +++ b/apps/server/src/persistence/Migrations/077_ProjectionMessageSearchText.ts @@ -1,29 +1,22 @@ -import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; import * as Effect from "effect/Effect"; import * as SqlClient from "effect/sql/SqlClient"; -import { createMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; +import { + readableSearchText, + recreateMessageSearchTriggers, +} from "../../orchestration-v2/messageSearchIndex.ts"; export default Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - // Readable text thread search matches; NULL when it equals the stored text. yield* sql`ALTER TABLE orchestration_v2_projection_messages ADD COLUMN search_text TEXT`; - // The triggers from migration 74 index the stored text, where a quote's - // words are percent-encoded. Recreated now that search_text exists, they - // index it instead, and the backfill below re-indexes, through the update - // trigger, every message that reads differently. - const index = yield* sql` - SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' - `; - if (index.length > 0) { - yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; - yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; - yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; - yield* createMessageSearchTriggers(sql); - } + // Migration 74's triggers index the stored text, where a quote's words are + // percent-encoded. Recreated now that search_text exists, they index it, and + // the backfill below re-indexes every message that reads differently. + yield* recreateMessageSearchTriggers(sql); - // Only messages holding a citation link read differently from their stored text. + // Only messages holding a citation link read differently. The raw check + // first skips parsing the JSON of every other message. const pageSize = 500; let afterMessageId = ""; while (true) { @@ -31,14 +24,14 @@ export default Effect.gen(function* () { SELECT message_id, json_extract(payload_json, '$.text') AS text FROM orchestration_v2_projection_messages WHERE message_id > ${afterMessageId} + AND payload_json LIKE '%t3-citation://%' AND json_type(payload_json, '$.text') = 'text' - AND json_extract(payload_json, '$.text') LIKE '%t3-citation://%' ORDER BY message_id LIMIT ${pageSize} `; for (const row of rows) { - const searchText = assistantCitationsToPlainText(row.text); - if (searchText === row.text) continue; + const searchText = readableSearchText(row.text); + if (searchText === null) continue; yield* sql` UPDATE orchestration_v2_projection_messages SET search_text = ${searchText} From 7c15aaf30eafd1d7542a56c2bd07b2794ebbfadf Mon Sep 17 00:00:00 2001 From: only21mil Date: Sat, 31 Oct 2026 23:35:15 +0000 Subject: [PATCH 3/3] fix(server): leave migration 074 as shipped Migration 074 called the trigger helper directly; only migration 77 needs the shared recreateMessageSearchTriggers. Co-Authored-By: Claude Opus 5.5 --- .../074_ExcludeAttachmentPlaceholdersFromSearch.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/server/src/persistence/Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts b/apps/server/src/persistence/Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts index 98b2d816cc2fe..6ec95d108f306 100644 --- a/apps/server/src/persistence/Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts +++ b/apps/server/src/persistence/Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts @@ -1,10 +1,18 @@ import * as Effect from "effect/Effect"; import * as SqlClient from "effect/sql/SqlClient"; -import { recreateMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; +import { createMessageSearchTriggers } from "../../orchestration-v2/messageSearchIndex.ts"; export default Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const index = yield* sql` + SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' + `; + if (index.length === 0) return; + // Legacy placeholders stay indexed until rebuild; FTS deletes can overflow last_insert_rowid at commit. - yield* recreateMessageSearchTriggers(sql); + yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; + yield* createMessageSearchTriggers(sql); });