From 181f60b434ae806dd959011e1a8ce9820a7bdbc0 Mon Sep 17 00:00:00 2001
From: ysyneu <ysyneu@gmail.com>
Date: Mon, 21 Sep 2026 03:02:02 -0700
Subject: [PATCH 1/2] fix(search-index): index every page in full

upload.sh cut each page's cleaned text at 6000 characters (bytes on the
ubuntu runner), so keyword search never saw the rest of a long page: 129 of
279 zh pages and 144 of 281 en pages were cut, losing 44% and 47% of their
text. Each Meilisearch document now holds the whole cleaned page; ids are
unchanged, so the next sync upserts in place.

The content reaches jq through stdin: as a --arg, a page over 128 KiB would
exceed Linux's limit on a single command-line argument.

The index embedder must bound its own input, e.g. a documentTemplate of
"A document named {{doc.title}} with content {{doc.content | truncate: 6000}}":
Meilisearch 1.39 sends the full rendered template to a REST embedder
regardless of documentTemplateMaxBytes, and text-embedding-v4 rejects a long
page, which fails the whole indexing batch.
---
 scripts/upload.sh | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/scripts/upload.sh b/scripts/upload.sh
index 91dffae0..42069751 100644
--- a/scripts/upload.sh
+++ b/scripts/upload.sh
@@ -25,9 +25,6 @@ SCRIPT_NAME=$(basename "$0")
 BASE_URL="${BASE_URL:-https://docs.flashduty.com}"
 DRY_RUN=false
 BATCH_SIZE=5
-# DashScope text-embedding-v4 accepts up to 8192 tokens per input.
-# Truncate content to stay safely within this limit after cleanup.
-MAX_CONTENT_CHARS=6000
 LIST_PAGE_SIZE=1000
 
 usage() {
@@ -133,17 +130,19 @@ dir_for_file() {
   fi
 }
 
-# Clean raw MDX content for embedding: strip frontmatter, import statements and
-# HTML/MDX tags, collapse whitespace, then truncate. Tags are stripped after
-# lines are joined so a tag whose attributes span several lines goes too; a tag
-# must start with a letter or '/', which keeps comparisons like "a < b" intact.
+# Clean raw MDX content for indexing: strip frontmatter, import statements and
+# HTML/MDX tags, collapse whitespace. Tags are stripped after lines are joined
+# so a tag whose attributes span several lines goes too; a tag must start with
+# a letter or '/', which keeps comparisons like "a < b" intact.
+# The whole page is indexed so keyword search sees all of it. The index's
+# embedder reads only its first documentTemplateMaxBytes, so nothing here has
+# to fit the embedding model's input limit.
 clean_content() {
   local file=$1
   awk 'BEGIN{skip=0} NR==1 && /^---$/{skip=1;next} skip && /^---$/{skip=0;next} !skip' "$file" \
     | grep -v '^import ' \
     | tr '\n' ' ' \
-    | sed -E 's/<[A-Za-z/][^<>]*>//g; s/ +/ /g' \
-    | cut -c1-"$MAX_CONTENT_CHARS"
+    | sed -E 's/<[A-Za-z/][^<>]*>//g; s/ +/ /g'
 }
 
 # Build a JSON document for a single file
@@ -157,13 +156,14 @@ build_doc_json() {
   id=$(file_to_id "$file")
   content=$(clean_content "$file")
 
-  jq -n \
+  # The content goes through stdin: as a --arg it would hit the kernel's
+  # 128 KiB limit on a single command-line argument for a long page.
+  printf '%s' "$content" | jq -Rs \
     --arg id "$id" \
     --arg title "$title" \
-    --arg content "$content" \
     --arg locale "$locale" \
     --arg url "$doc_url" \
-    '{id: $id, title: $title, content: $content, locale: $locale, url: $url}' 2>/dev/null
+    '{id: $id, title: $title, content: ., locale: $locale, url: $url}' 2>/dev/null
 }
 
 # Write "id<TAB>url" for every document in the index to $1. Fails unless the

From 4b7f925f8eccbcc5c0db7a17aa7107cc65f8526e Mon Sep 17 00:00:00 2001
From: ysyneu <ysyneu@gmail.com>
Date: Mon, 21 Sep 2026 08:27:22 -0700
Subject: [PATCH 2/2] docs(search-index): say where the embedder input is cut

The comment claimed the embedder reads only the first
documentTemplateMaxBytes of a document. Meilisearch does not apply that
setting to REST embedders; the cut happens in the index's documentTemplate
with the truncate filter.
---
 scripts/upload.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/upload.sh b/scripts/upload.sh
index 42069751..8879c942 100644
--- a/scripts/upload.sh
+++ b/scripts/upload.sh
@@ -135,8 +135,9 @@ dir_for_file() {
 # so a tag whose attributes span several lines goes too; a tag must start with
 # a letter or '/', which keeps comparisons like "a < b" intact.
 # The whole page is indexed so keyword search sees all of it. The index's
-# embedder reads only its first documentTemplateMaxBytes, so nothing here has
-# to fit the embedding model's input limit.
+# embedder cuts its own input in its documentTemplate
+# ({{doc.content | truncate: 6000}}), so nothing here has to fit the embedding
+# model's input limit.
 clean_content() {
   local file=$1
   awk 'BEGIN{skip=0} NR==1 && /^---$/{skip=1;next} skip && /^---$/{skip=0;next} !skip' "$file" \