From c18b9c4e7312dce2b1b0091fdf85164203d808f8 Mon Sep 17 00:00:00 2001
From: MarcoDotIO <Marcus.Arnett10@gmail.com>
Date: Thu, 29 Oct 2026 19:52:57 -0400
Subject: [PATCH] fix(server): message search without the index finds words
 typed with a curly apostrophe
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The search index stores message text with curly apostrophes straightened, and
queries are straightened the same way. The substring check that the scan path
(no FTS5 trigram index, or a two-character CJK word) and short tokens rely on
read the raw text instead, so "gannet's" never matched "the gannet’s nest".
Check the same straightened text the index stores.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../src/orchestration-v2/ThreadSearch.test.ts | 29 +++++++++++++++++++
 .../src/orchestration-v2/ThreadSearch.ts      |  4 +--
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/apps/server/src/orchestration-v2/ThreadSearch.test.ts b/apps/server/src/orchestration-v2/ThreadSearch.test.ts
index 399943c7b1b92..72febe8ff06e3 100644
--- a/apps/server/src/orchestration-v2/ThreadSearch.test.ts
+++ b/apps/server/src/orchestration-v2/ThreadSearch.test.ts
@@ -668,6 +668,35 @@ it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) =>
   );
 });
 
+it.layer(layerTest)("ThreadSearch without the index", (it) => {
+  it.effect("finds a curly apostrophe for a straight one and back", () =>
+    Effect.gen(function* () {
+      const projections = yield* ProjectionStore.ProjectionStoreV2;
+      const search = yield* ThreadSearch.ThreadSearch;
+      const sql = yield* SqlClient.SqlClient;
+      const project = ProjectId.make("project:scan-apostrophes");
+      const curly = ThreadId.make("thread:scan-apostrophe-curly");
+      const straight = ThreadId.make("thread:scan-apostrophe-straight");
+      yield* createProject(project);
+      yield* Effect.forEach(
+        [
+          thread(curly, project),
+          message(curly, "scan-apostrophe-curly", "user", "the gannet’s nest"),
+          thread(straight, project),
+          message(straight, "scan-apostrophe-straight", "user", "a kittiwake's nest"),
+        ],
+        projections.apply,
+        { discard: true },
+      );
+      yield* sql`DROP TABLE orchestration_v2_message_search`;
+      const threadIds = (query: string) =>
+        search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
+      assert.deepEqual(yield* threadIds("gannet's"), [curly]);
+      assert.deepEqual(yield* threadIds("kittiwake’s"), [straight]);
+    }),
+  );
+});
+
 it.layer(layerTest)("ThreadSearch.searchOlderSettled", (it) => {
   it.effect("returns older settled title matches in scope, then older message hits", () =>
     Effect.gen(function* () {
diff --git a/apps/server/src/orchestration-v2/ThreadSearch.ts b/apps/server/src/orchestration-v2/ThreadSearch.ts
index 37a1b9e726f9b..1452748f716d3 100644
--- a/apps/server/src/orchestration-v2/ThreadSearch.ts
+++ b/apps/server/src/orchestration-v2/ThreadSearch.ts
@@ -33,7 +33,7 @@ import {
   type ThreadSearchSubject,
 } from "@t3tools/shared/paletteSearch";
 import { threadSearchPullRequests } from "@t3tools/shared/threadPullRequests";
-import { indexedMessageCondition } from "./messageSearchIndex.ts";
+import { indexedMessageCondition, indexedMessageText } from "./messageSearchIndex.ts";
 import { loadOlderSettledThreadsPage } from "./ShellStream.ts";
 
 // When the user last wrote in the thread aliased `threads`, as the clients
@@ -221,7 +221,7 @@ export const messageHitsQuery = (
       AND messages.message_id NOT LIKE 'async-answer:%'
       AND NOT EXISTS (
         SELECT 1 FROM json_each(${substrings}) AS token
-        WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
+        WHERE instr(lower(${indexedMessageText(sql, "messages.payload_json")}), token.value) = 0
       )
     -- The index's own rowid, so FTS5 returns hits newest first and stops at
     -- the limit; ordering by the messages table's rowid sorts every hit.