fix(server): thread search finds phrases inside quotes and their comments

apps/server/src/orchestration-v2/ProjectionStore.ts

@@ -92,6 +92,7 @@ import type * as Statement from "effect/sql/Statement";
 
 import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp";
 import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts";
+import { readableSearchText } from "./messageSearchIndex.ts";
 import {
   isThreadHistoryUserTurn,
   isConversationHistoryItem,
@@ -2426,7 +2427,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
                 streaming,
                 created_at,
                 updated_at,
-                payload_json
+                payload_json,
+                search_text
               )
               VALUES (
                 ${event.payload.id},
@@ -2437,7 +2439,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
                 ${booleanInt(event.payload.streaming)},
                 ${stringField(payload, "createdAt")},
                 ${stringField(payload, "updatedAt")},
-                ${payloadJson}
+                ${payloadJson},
+                ${readableSearchText(event.payload.text)}
               )
               ON CONFLICT(message_id)
               DO UPDATE SET
@@ -2448,7 +2451,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
                 streaming = excluded.streaming,
                 created_at = excluded.created_at,
                 updated_at = excluded.updated_at,
-                payload_json = excluded.payload_json
+                payload_json = excluded.payload_json,
+                search_text = excluded.search_text
             `;
               break;
             }

apps/server/src/orchestration-v2/ThreadSearch.test.ts

@@ -1,5 +1,6 @@
 import { assert, it } from "@effect/vitest";
 import {
+  EnvironmentId,
   EventId,
   MessageId,
   ProjectId,
@@ -18,9 +19,13 @@ import * as Fiber from "effect/Fiber";
 import * as Layer from "effect/Layer";
 import * as Path from "effect/Path";
 import * as SqlClient from "effect/sql/SqlClient";
+import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
 import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
 
 import * as SqlitePersistence from "../persistence/Sqlite.ts";
+import * as EventStore from "./EventStore.ts";
+import { rebuildMessageSearch } from "./messageSearchIndex.ts";
+import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
 import * as ProjectionStore from "./ProjectionStore.ts";
 import * as ProjectStore from "./ProjectStore.ts";
 import * as ThreadSearch from "./ThreadSearch.ts";
@@ -150,6 +155,20 @@ const message = (
   },
 });
 
+// The prefix and suffix are hidden selection context, not text the reader sees.
+const quote = serializeAssistantCitation({
+  version: 1,
+  environmentId: EnvironmentId.make("environment:local"),
+  threadId: ThreadId.make("thread:quoted"),
+  messageId: MessageId.make("quoted-answer"),
+  text: "A lighthouse guides ships safely through the darkness.",
+  comment: "what are you talking about",
+  start: 0,
+  end: 54,
+  prefix: "beacon01",
+  suffix: "harbor",
+});
+
 it.effect.each([true, false])(
   "ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)",
   (indexed) =>
@@ -349,6 +368,89 @@ it.layer(layerTest)("ThreadSearch", (it) => {
     }),
   );
 
+  it.effect("matches the readable text of quotes and their comments", () =>
+    Effect.gen(function* () {
+      const projections = yield* ProjectionStore.ProjectionStoreV2;
+      const search = yield* ThreadSearch.ThreadSearch;
+      const project = ProjectId.make("project:search-citation");
+      const threadId = ThreadId.make("thread:citation");
+      yield* createProject(project);
+      yield* Effect.forEach(
+        [
+          thread(threadId, project),
+          message(threadId, "citation", "user", `${quote} quiet silver lantern`),
+        ],
+        projections.apply,
+        { discard: true },
+      );
+      const snippets = (query: string) =>
+        Effect.map(search.search({ query }), (result) =>
+          result.matches
+            .filter((match) => match.threadId === threadId)
+            .map((match) => match.snippet),
+        );
+
+      const readable =
+        "A lighthouse guides ships safely through the darkness. Comment: what are you talking about quiet silver lantern";
+      assert.deepEqual(yield* snippets("what are you talking about"), [
+        `…${readable.slice(readable.indexOf("the darkness"))}`,
+      ]);
+      assert.deepEqual(yield* snippets("ships safely through"), [readable]);
+      assert.deepEqual(yield* snippets("beacon01"), []);
+      assert.deepEqual(yield* snippets("harbor"), []);
+
+      yield* projections.apply(message(threadId, "citation", "user", "quiet silver lantern"));
+      assert.deepEqual(yield* snippets("what are you talking about"), []);
+      assert.deepEqual(yield* snippets("lantern"), ["quiet silver lantern"]);
+    }),
+  );
+
+  it.effect("finds accented, CJK, and apostrophe words of a quote through the index", () =>
+    Effect.gen(function* () {
+      const sql = yield* SqlClient.SqlClient;
+      const projections = yield* ProjectionStore.ProjectionStoreV2;
+      const search = yield* ThreadSearch.ThreadSearch;
+      const project = ProjectId.make("project:search-citation-index");
+      const threadId = ThreadId.make("thread:citation-index");
+      // The stored link percent-encodes these words; the index must hold them as they read.
+      const encodedQuote = serializeAssistantCitation({
+        version: 1,
+        environmentId: EnvironmentId.make("environment:local"),
+        threadId: ThreadId.make("thread:quoted"),
+        messageId: MessageId.make("quoted-answer"),
+        text: "The kittiwake's résumé mentions 灯台の光 twice.",
+        comment: "naïve question",
+        start: 0,
+        end: 44,
+        prefix: "",
+        suffix: "",
+      });
+      yield* createProject(project);
+      yield* Effect.forEach(
+        [thread(threadId, project), message(threadId, "citation-index", "user", encodedQuote)],
+        projections.apply,
+        { discard: true },
+      );
+      const index = yield* sql`
+        SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
+      `;
+      assert.lengthOf(index, 1);
+      const threadIds = (query: string) =>
+        search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
+
+      for (const query of ["kittiwake's", "kittiwake’s", "résumé", "灯台の光", "naïve"]) {
+        assert.deepEqual(yield* threadIds(query), [threadId], query);
+      }
+      // `t3 doctor` rebuilds the index after VACUUM; quotes must still read.
+      yield* rebuildMessageSearch(sql);
+      assert.deepEqual(yield* threadIds("résumé"), [threadId]);
+
+      yield* projections.apply(message(threadId, "citation-index", "user", "plain words"));
+      assert.deepEqual(yield* threadIds("résumé"), []);
+      assert.deepEqual(yield* threadIds("plain"), [threadId]);
+    }),
+  );
+
   it.effect("indexes a message when it finishes streaming and follows later edits", () =>
     Effect.gen(function* () {
       const projections = yield* ProjectionStore.ProjectionStoreV2;
@@ -1115,6 +1217,35 @@ it.effect("starts snippets a few words before the match so one line shows it", (
   }),
 );
 
+it.effect("finds quote text after projections replay from events", () =>
+  Effect.gen(function* () {
+    const eventStore = yield* EventStore.EventStoreV2;
+    const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
+    const search = yield* ThreadSearch.ThreadSearch;
+    const project = ProjectId.make("project:search-replay");
+    const threadId = ThreadId.make("thread:replay");
+    yield* createProject(project);
+    yield* eventStore.append({
+      events: [thread(threadId, project), message(threadId, "replay", "user", quote)],
+    });
+
+    assert.isTrue((yield* maintenance.rebuild).valid);
+    assert.deepEqual(
+      (yield* search.search({ query: "what are you talking about" })).matches.map(
+        (match) => match.threadId,
+      ),
+      [threadId],
+    );
+  }).pipe(
+    Effect.provide(
+      ProjectionMaintenance.layer.pipe(
+        Layer.provideMerge(EventStore.layer),
+        Layer.provideMerge(layerTest),
+      ),
+    ),
+  ),
+);
+
 it.effect("searches on the reader worker while the writer is busy", () =>
   Effect.gen(function* () {
     const fileSystem = yield* FileSystem.FileSystem;

apps/server/src/orchestration-v2/ThreadSearch.ts

@@ -33,7 +33,11 @@ import {
   type ThreadSearchSubject,
 } from "@t3tools/shared/paletteSearch";
 import { threadSearchPullRequests } from "@t3tools/shared/threadPullRequests";
-import { indexedMessageCondition, nonPlaceholderMessageCondition } from "./messageSearchIndex.ts";
+import {
+  indexedMessageCondition,
+  messageSearchText,
+  nonPlaceholderMessageCondition,
+} from "./messageSearchIndex.ts";
 import { loadOlderSettledThreadsPage } from "./ShellStream.ts";
 
 // When the user last wrote in the thread aliased `threads`, as the clients
@@ -185,7 +189,7 @@ export const messageHitsQuery = (
             messages.thread_id AS "threadId",
             eligible.project_id AS "projectId",
             messages.role AS source,
-            json_extract(messages.payload_json, '$.text') AS text,
+            ${messageSearchText(sql)} AS text,
             messages.created_at AS "messageCreatedAt"
           FROM orchestration_v2_projection_messages AS messages
           INNER JOIN eligible ON eligible.thread_id = messages.thread_id
@@ -202,7 +206,7 @@ export const messageHitsQuery = (
             messages.thread_id AS "threadId",
             eligible.project_id AS "projectId",
             messages.role AS source,
-            json_extract(messages.payload_json, '$.text') AS text,
+            ${messageSearchText(sql)} AS text,
             messages.created_at AS "messageCreatedAt"
           FROM orchestration_v2_message_search
           INNER JOIN orchestration_v2_projection_messages AS messages
@@ -223,7 +227,7 @@ export const messageHitsQuery = (
       AND messages.message_id NOT LIKE 'async-answer:%'
       AND NOT EXISTS (
         SELECT 1 FROM json_each(${substrings}) AS token
-        WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
+        WHERE instr(lower(${messageSearchText(sql)}), token.value) = 0
       )
     -- The index's own rowid, so FTS5 returns hits newest first and stops at
     -- the limit; ordering by the messages table's rowid sorts every hit.

apps/server/src/orchestration-v2/messageSearchIndex.ts

@@ -1,3 +1,4 @@
+import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations";
 import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
 import * as Effect from "effect/Effect";
 import type * as SqlClient from "effect/sql/SqlClient";
@@ -40,46 +41,83 @@ export const indexedMessageCondition = (sql: SqlClient.SqlClient, row = "message
   AND ${nonPlaceholderMessageCondition(sql, row)}
 `;
 
+// Curly apostrophes straight, as search folds a query, since iOS keyboards
+// type "’" and others type "'".
+const foldApostrophes = (text: string) => `replace(replace(${text}, '’', ''''), '‘', '''')`;
+
+const readableText = (row: string) =>
+  `COALESCE(${row}.search_text, json_extract(${row}.payload_json, '$.text'))`;
+
+/**
+ * What search_text holds for a message: its text as it reads, a quote's
+ * percent-encoded link replaced by its words and comment. NULL when that is
+ * the stored text, so most messages store nothing extra.
+ */
+export const readableSearchText = (text: string) => {
+  const readable = assistantCitationsToPlainText(text);
+  return readable === text ? null : readable;
+};
+
+/** A message's text as it reads, for thread search. */
+export const messageSearchText = (sql: SqlClient.SqlClient) =>
+  sql.literal(readableText("messages"));
+
 /**
- * What the index stores for a message: its text with curly apostrophes
- * straight, as search folds a query, since iOS keyboards type "’" and others
- * type "'". Migration 69's triggers use it too. `payload` is the message's
+ * A message's stored text with apostrophes folded, what the index held before
+ * migration 77. Migration 69's backfill uses it. `payload` is the message's
  * JSON column.
  */
 export const indexedMessageText = (sql: SqlClient.SqlClient, payload: string) =>
-  sql.literal(`replace(replace(json_extract(${payload}, '$.text'), '’', ''''), '‘', '''')`);
+  sql.literal(foldApostrophes(`json_extract(${payload}, '$.text')`));
+
+// What the index stores: a message's readable text with apostrophes folded.
+// Migrations 69 and 74 create the triggers before migration 77 adds
+// search_text, and `t3 doctor` can rebuild a database that has not migrated,
+// so those fall back to the stored text.
+const indexedText = Effect.fn("MessageSearchIndex.indexedText")(function* (
+  sql: SqlClient.SqlClient,
+) {
+  const column = yield* sql`
+    SELECT 1 FROM pragma_table_info('orchestration_v2_projection_messages')
+    WHERE name = 'search_text'
+  `;
+  return column.length > 0
+    ? (row: string) => sql.literal(foldApostrophes(readableText(row)))
+    : (row: string) => indexedMessageText(sql, `${row}.payload_json`);
+});
 
 export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createTriggers")(
   function* (sql: SqlClient.SqlClient) {
+    const text = yield* indexedText(sql);
     yield* sql`
       CREATE TRIGGER orchestration_v2_message_search_insert
       AFTER INSERT ON orchestration_v2_projection_messages
       WHEN ${indexedMessageCondition(sql, "new")}
       BEGIN
         INSERT INTO orchestration_v2_message_search (rowid, text)
-        SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")}
+        SELECT new.rowid, ${text("new")}
         WHERE json_extract(new.payload_json, '$.text') IS NOT NULL;
       END
     `;
     // Streaming deltas rewrite the row many times a second; skip them until the
-    // message settles. Upserts rewrite every column, so a finished message
-    // re-indexes only when what the index reads changed. Deleting a rowid the
-    // index never held is a no-op.
+    // message settles. Upserts rewrite every column, so re-index only when what
+    // the index stores changed; migration 77's backfill writes only search_text.
+    // Deleting a rowid the index never held is a no-op.
     yield* sql`
       CREATE TRIGGER orchestration_v2_message_search_update
-      AFTER UPDATE OF payload_json, streaming, role ON orchestration_v2_projection_messages
+      AFTER UPDATE ON orchestration_v2_projection_messages
       WHEN (old.streaming = 0 OR new.streaming = 0)
         AND (
           old.streaming IS NOT new.streaming
           OR old.role IS NOT new.role
-          OR json_extract(old.payload_json, '$.text') IS NOT json_extract(new.payload_json, '$.text')
+          OR ${text("old")} IS NOT ${text("new")}
           OR json_extract(old.payload_json, '$.createdBy')
             IS NOT json_extract(new.payload_json, '$.createdBy')
         )
       BEGIN
         DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid AND old.streaming = 0;
         INSERT INTO orchestration_v2_message_search (rowid, text)
-        SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")}
+        SELECT new.rowid, ${text("new")}
         WHERE ${indexedMessageCondition(sql, "new")}
           AND json_extract(new.payload_json, '$.text') IS NOT NULL;
       END
@@ -95,6 +133,20 @@ export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createT
   },
 );
 
+/** Replaces the search triggers with their current definition, where the index exists. */
+export const recreateMessageSearchTriggers = Effect.fn("MessageSearchIndex.recreateTriggers")(
+  function* (sql: SqlClient.SqlClient) {
+    const index = yield* sql`
+      SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
+    `;
+    if (index.length === 0) return;
+    yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
+    yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
+    yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
+    yield* createMessageSearchTriggers(sql);
+  },
+);
+
 /**
  * Refills the search index from the messages table. The index is keyed by
  * message rowid, so anything that may renumber or copy rows (VACUUM, a dev
@@ -108,6 +160,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func
     SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
   `;
   if (index.length === 0) return;
+  const text = yield* indexedText(sql);
   // One transaction: an interrupt between clear and refill would leave an
   // empty index that nothing repairs, since search only checks the table exists.
   yield* sql.withTransaction(
@@ -118,7 +171,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func
       `;
       yield* sql`
         INSERT INTO orchestration_v2_message_search (rowid, text)
-        SELECT messages.rowid, ${indexedMessageText(sql, "messages.payload_json")}
+        SELECT messages.rowid, ${text("messages")}
         FROM orchestration_v2_projection_messages AS messages
         WHERE ${indexedMessageCondition(sql)}
           AND json_extract(messages.payload_json, '$.text') IS NOT NULL

apps/server/src/persistence/Migrations.ts

@@ -79,6 +79,7 @@ import Migration0073 from "./Migrations/073_ActiveTurnItemIndexes.ts";
 import Migration0074 from "./Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts";
 import Migration0075 from "./Migrations/075_SubagentDeliveryState.ts";
 import Migration0076 from "./Migrations/076_ProjectionProjectsPinArchive.ts";
+import Migration0077 from "./Migrations/077_ProjectionMessageSearchText.ts";
 
 const migrationEntries = [
   [1, "OrchestrationEvents", Migration0001],
@@ -158,6 +159,7 @@ const migrationEntries = [
   [74, "ExcludeAttachmentPlaceholdersFromSearch", Migration0074],
   [75, "SubagentDeliveryState", Migration0075],
   [76, "ProjectionProjectsPinArchive", Migration0076],
+  [77, "ProjectionMessageSearchText", Migration0077],
 ] as const;
 
 export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);