diff --git a/scripts/build_index_docs.py b/scripts/build_index_docs.py new file mode 100644 index 00000000..e4783576 --- /dev/null +++ b/scripts/build_index_docs.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Build the Meilisearch documents for the docs Q&A index. + +Every page under zh/ and en/ (its frontmatter description, then its body) is +packed into chunks of whole headings and paragraphs, so that the whole page is +searchable and each chunk fits the index embedder's documentTemplateMaxBytes +(4000) together with the page title. A paragraph longer than a chunk is split +at line boundaries, never inside a UTF-8 character. A chunk starts with a +breadcrumb line ("Page title > Section > Subsection") naming where it starts, +so it stays understandable on its own. + +Output: one JSON document per line on stdout, fields id/title/content/locale/url. +Document ids are "-", so a page that shrinks +leaves stale ids behind for upload.sh's reconcile step to delete. + +Usage: python3 scripts/build_index_docs.py [FILE ...] (default: all of zh/ and en/) +Env: BASE_URL docs base URL (default: https://docs.flashduty.com) +""" + +import hashlib +import json +import os +import re +import sys +from pathlib import Path + +BASE_URL = os.environ.get("BASE_URL", "https://docs.flashduty.com").rstrip("/") + +# Content bytes per chunk. The embedder template adds the page title and ~30 +# bytes of fixed text, and the whole rendered template must stay under 4000. +MAX_CHUNK_BYTES = 3000 + +FENCE_RE = re.compile(r"^\s*(```|~~~)") +HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$") +# MDX components are CamelCase (, ), HTML tags are lowercase +# (
, ). ALL_CAPS placeholders such as are content. +TAG_RE = re.compile(r"]*)?)/?>") +TITLE_ATTR_RE = re.compile(r"""\btitle=(?:"([^"]*)"|'([^']*)'|\{["']([^"']*)["']\})""") + + +def page_id(path): + rel = re.sub(r"\.mdx?$", "", path) + return hashlib.md5(rel.encode("utf-8")).hexdigest() + + +def split_frontmatter(text): + lines = text.split("\n") + if not lines or lines[0].strip() != "---": + return {}, text + for i in range(1, len(lines)): + if lines[i].strip() == "---": + meta = {} + for line in lines[1:i]: + m = re.match(r"^([A-Za-z_]+):\s*(.*)$", line) + if m: + meta[m.group(1)] = m.group(2).strip().strip("\"'") + return meta, "\n".join(lines[i + 1 :]) + return {}, text + + +def page_title(meta, path): + if meta.get("title"): + return meta["title"] + stem = re.sub(r"\.mdx?$", "", os.path.basename(path)) + return re.sub(r"^[0-9.]*\s*", "", stem) + + +def page_url(meta, path): + url = meta.get("url", "") + if url.startswith("/"): + return BASE_URL + url + if url: + return url + return BASE_URL + "/" + re.sub(r"\.mdx?$", "", path) + + +def strip_tag(m): + name = m.group(1) + if name.isupper() or "_" in name: + return m.group(0) + if not (name[0].isupper() or name.islower()): + return m.group(0) + if m.group(0).startswith(" limit: + head = line.encode("utf-8")[:limit].decode("utf-8", "ignore") + if cur: + pieces.append(cur) + cur = "" + pieces.append(head) + line = line[len(head) :] + candidate = line if not cur else cur + "\n" + line + if utf8_len(candidate) > limit: + pieces.append(cur) + cur = line + else: + cur = candidate + if cur: + pieces.append(cur) + return pieces + + +def chunk_page(title, body): + """Return the list of chunk contents for one page.""" + chunks = [] + h2 = h3 = "" + # The open chunk: its breadcrumb, its parts, and the bytes they take + # including the blank lines between them. It fits while size <= budget. + crumb, parts, size, budget = "", [], 0, 0 + + def breadcrumb(): + return " > ".join(p for p in (title, h2, h3) if p) + + for level, text in blocks(body): + if level: + if level <= 2: + h2, h3 = (text if level == 2 else ""), "" + elif level == 3: + h3 = text + text = "#" * level + " " + text + # Size pieces for a chunk that would start here, the tightest case. + limit = MAX_CHUNK_BYTES - utf8_len(breadcrumb()) - 2 + for piece in split_oversized(text, limit) if utf8_len(text) > limit else [text]: + if parts and size + 2 + utf8_len(piece) > budget: + chunks.append(crumb + "\n\n" + "\n\n".join(parts)) + parts = [] + if not parts and 0 < level <= 3: + continue # the breadcrumb of the chunk that starts here names it + if parts: + size += 2 + utf8_len(piece) + else: + crumb = breadcrumb() + budget = MAX_CHUNK_BYTES - utf8_len(crumb) - 2 + size = utf8_len(piece) + parts.append(piece) + if parts: + chunks.append(crumb + "\n\n" + "\n\n".join(parts)) + return chunks + + +def build(path): + locale = "zh-CN" if path.startswith("zh/") else "en-US" + meta, body = split_frontmatter(Path(path).read_text(encoding="utf-8")) + title = page_title(meta, path) + url = page_url(meta, path) + pid = page_id(path) + body = clean_body(body) + if meta.get("description"): + body = meta["description"] + "\n\n" + body + for i, content in enumerate(chunk_page(title, body)): + yield { + "id": f"{pid}-{i:03d}", + "title": title, + "content": content, + "locale": locale, + "url": url, + } + + +def doc_files(): + files = [] + for root in ("zh", "en"): + for p in Path(root).rglob("*"): + if p.suffix in (".md", ".mdx") and p.stem != "index" and p.is_file(): + files.append(p.as_posix()) + return sorted(files) + + +def main(argv): + files = argv[1:] or doc_files() + for path in files: + for doc in build(path): + sys.stdout.write(json.dumps(doc, ensure_ascii=False) + "\n") + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/upload.sh b/scripts/upload.sh index 91dffae0..a78037ef 100644 --- a/scripts/upload.sh +++ b/scripts/upload.sh @@ -2,9 +2,10 @@ # # Sync documentation to Meilisearch for AI Q&A bot indexing. # -# Every run makes the index match the docs in zh/ and en/: all docs are -# upserted (Meilisearch only re-embeds documents whose content changed) and -# index documents whose source file no longer exists are deleted. +# Every run makes the index match the docs in zh/ and en/: every page is split +# into chunks by scripts/build_index_docs.py, all chunks are upserted +# (Meilisearch only re-embeds documents whose content changed), and index +# documents that are no longer produced are deleted. # # Required env vars: MEILI_ENDPOINT, MEILI_API_KEY, MEILI_INDEX # Optional: @@ -24,10 +25,7 @@ cd "$REPO_ROOT" SCRIPT_NAME=$(basename "$0") BASE_URL="${BASE_URL:-https://docs.flashduty.com}" DRY_RUN=false -BATCH_SIZE=5 -# DashScope text-embedding-v4 accepts up to 8192 tokens per input. -# Truncate content to stay safely within this limit after cleanup. -MAX_CONTENT_CHARS=6000 +BATCH_SIZE=20 LIST_PAGE_SIZE=1000 usage() { @@ -35,7 +33,7 @@ usage() { Usage: $SCRIPT_NAME [OPTIONS] Sync documentation (zh/, en/) to Meilisearch for AI Q&A bot indexing: -upload every doc and delete index documents whose source file is gone. +upload every doc chunk and delete index documents no longer produced. Options: --dry-run List what would be uploaded and deleted without writing @@ -62,109 +60,14 @@ if [[ -z "${MEILI_ENDPOINT:-}" || -z "${MEILI_API_KEY:-}" || -z "${MEILI_INDEX:- exit 1 fi -if ! command -v jq &> /dev/null; then - echo "Error: jq is required. Install it first (e.g. brew install jq on macOS)." >&2 - exit 1 -fi - -# --- Helpers --- - -# Generate a stable document ID from a file path (relative to repo root) -file_to_id() { - local file=$1 - local rel="${file%.mdx}" - rel="${rel%.md}" - echo -n "$rel" | openssl md5 | awk '{print $NF}' -} - -extract_title() { - local file=$1 - local title - title=$(grep -m 1 '^title:' "$file" 2>/dev/null | sed -n 's/title: *"\(.*\)"/\1/p') || true - if [[ -z "${title:-}" ]]; then - title=$(grep -m 1 '^title:' "$file" 2>/dev/null | sed -n 's/title: *\(.*\)$/\1/p' | xargs) || true - fi - if [[ -z "${title:-}" ]]; then - local base - base=$(basename "$file") - title="${base%.mdx}" - title="${title%.md}" - title=$(echo "$title" | sed 's/^[0-9.]*[[:space:]]*//') - fi - echo "$title" -} - -extract_url() { - local file=$1 - local dir=$2 - local locale=$3 - local doc_url - doc_url=$(grep -m 1 '^url:' "$file" 2>/dev/null | sed -n 's/url: *"\(.*\)"/\1/p') || true - if [[ -z "${doc_url:-}" ]]; then - doc_url=$(grep -m 1 '^url:' "$file" 2>/dev/null | sed -n 's/url: *\(.*\)$/\1/p' | xargs) || true - fi - if [[ -z "${doc_url:-}" ]]; then - local rel_path - rel_path="${file#$dir/}" - rel_path="${rel_path%.mdx}" - rel_path="${rel_path%.md}" - local locale_prefix - [[ "$locale" == "zh-CN" ]] && locale_prefix="zh" || locale_prefix="en" - doc_url="${BASE_URL}/${locale_prefix}/${rel_path}" +for tool in jq python3; do + if ! command -v "$tool" &> /dev/null; then + echo "Error: $tool is required. Install it first." >&2 + exit 1 fi - echo "$doc_url" -} - -locale_for_file() { - local file=$1 - if [[ "$file" == zh/* ]]; then - echo "zh-CN" - else - echo "en-US" - fi -} - -dir_for_file() { - local file=$1 - if [[ "$file" == zh/* ]]; then - echo "zh" - else - echo "en" - fi -} - -# Clean raw MDX content for embedding: strip frontmatter, import statements and -# HTML/MDX tags, collapse whitespace, then truncate. Tags are stripped after -# lines are joined so a tag whose attributes span several lines goes too; a tag -# must start with a letter or '/', which keeps comparisons like "a < b" intact. -clean_content() { - local file=$1 - awk 'BEGIN{skip=0} NR==1 && /^---$/{skip=1;next} skip && /^---$/{skip=0;next} !skip' "$file" \ - | grep -v '^import ' \ - | tr '\n' ' ' \ - | sed -E 's/<[A-Za-z/][^<>]*>//g; s/ +/ /g' \ - | cut -c1-"$MAX_CONTENT_CHARS" -} +done -# Build a JSON document for a single file -build_doc_json() { - local file=$1 - local dir locale title doc_url id content - dir=$(dir_for_file "$file") - locale=$(locale_for_file "$file") - title=$(extract_title "$file") - doc_url=$(extract_url "$file" "$dir" "$locale") - id=$(file_to_id "$file") - content=$(clean_content "$file") - - jq -n \ - --arg id "$id" \ - --arg title "$title" \ - --arg content "$content" \ - --arg locale "$locale" \ - --arg url "$doc_url" \ - '{id: $id, title: $title, content: $content, locale: $locale, url: $url}' 2>/dev/null -} +# --- Helpers --- # Write "idurl" for every document in the index to $1. Fails unless the # pages add up to exactly the total the index reports, so an incomplete @@ -269,55 +172,29 @@ delete_documents() { fi } -# Collect files into batched JSON arrays and upload -upload_files() { - local file_list=$1 - local total_files=0 - local total_success=0 - local batch_json="[" - local batch_count=0 - - while IFS= read -r file; do - [[ -z "$file" ]] && continue - total_files=$((total_files + 1)) - - local doc_json - if ! doc_json=$(build_doc_json "$file"); then - echo "JSON error: $file" >&2 - continue +# Upload the documents of a JSON-lines file in batches +upload_docs() { + local docs=$1 + local total=0 + local uploaded=0 + local batch count + + split -l "$BATCH_SIZE" "$docs" "$work_dir/batch_" + for batch in "$work_dir"/batch_*; do + count=$(wc -l < "$batch" | xargs) + total=$((total + count)) + if upload_batch "$(jq -cs . "$batch")" "$count"; then + uploaded=$((uploaded + count)) fi - - if [[ $batch_count -gt 0 ]]; then - batch_json="${batch_json}," - fi - batch_json="${batch_json}${doc_json}" - batch_count=$((batch_count + 1)) - - if [[ $batch_count -ge $BATCH_SIZE ]]; then - batch_json="${batch_json}]" - if upload_batch "$batch_json" "$batch_count"; then - total_success=$((total_success + batch_count)) - fi - batch_json="[" - batch_count=0 - fi - done < "$file_list" - - # Upload remaining - if [[ $batch_count -gt 0 ]]; then - batch_json="${batch_json}]" - if upload_batch "$batch_json" "$batch_count"; then - total_success=$((total_success + batch_count)) - fi - fi + done echo "" echo "=== Upload Summary ===" - echo "Total files: $total_files" - echo "Uploaded: $total_success" - echo "Failed: $((total_files - total_success))" + echo "Total documents: $total" + echo "Uploaded: $uploaded" + echo "Failed: $((total - uploaded))" - [[ $total_success -eq $total_files ]] || return 1 + [[ $uploaded -eq $total ]] || return 1 } # --- Main --- @@ -331,19 +208,20 @@ echo "" work_dir=$(mktemp -d) trap 'rm -rf "$work_dir"' EXIT -echo "Scanning all documentation files..." -find zh en -type f \( -name "*.md" -o -name "*.mdx" \) ! -name "index.md" ! -name "index.mdx" \ - | sort > "$work_dir/files" -file_count=$(wc -l < "$work_dir/files" | xargs) -echo "Found $file_count documentation files" -if [[ $file_count -eq 0 ]]; then - echo "Error: no documentation files found; refusing to sync an empty doc set." >&2 +echo "Building index documents..." +BASE_URL="$BASE_URL" python3 scripts/build_index_docs.py > "$work_dir/docs.jsonl" +doc_count=$(wc -l < "$work_dir/docs.jsonl" | xargs) +echo "Built $doc_count documents from $(jq -r '.url' "$work_dir/docs.jsonl" | sort -u | wc -l | xargs) pages" +if [[ $doc_count -eq 0 ]]; then + echo "Error: no documents built; refusing to sync an empty doc set." >&2 exit 1 fi -while IFS= read -r file; do - file_to_id "$file" -done < "$work_dir/files" | LC_ALL=C sort -u > "$work_dir/current_ids" +jq -r '.id' "$work_dir/docs.jsonl" | LC_ALL=C sort -u > "$work_dir/current_ids" +if [[ $(wc -l < "$work_dir/current_ids" | xargs) -ne $doc_count ]]; then + echo "Error: duplicate document ids in the build output." >&2 + exit 1 +fi echo "Listing documents in index..." if ! list_index_docs "$work_dir/index_docs"; then @@ -353,7 +231,7 @@ fi echo "Index has $(wc -l < "$work_dir/index_docs" | xargs) documents" echo "" -# Index documents with no source file in the current doc set +# Index documents the current build no longer produces LC_ALL=C sort "$work_dir/index_docs" \ | LC_ALL=C join -t $'\t' -v 1 - "$work_dir/current_ids" > "$work_dir/stale_docs" stale_count=$(wc -l < "$work_dir/stale_docs" | xargs) @@ -361,10 +239,10 @@ stale_count=$(wc -l < "$work_dir/stale_docs" | xargs) result=0 echo "--- Uploading documents ---" -upload_files "$work_dir/files" || result=1 +upload_docs "$work_dir/docs.jsonl" || result=1 echo "" -echo "--- Removing documents with no source file ---" +echo "--- Removing documents no longer produced ---" if [[ $stale_count -eq 0 ]]; then echo "None." else diff --git a/tests/test_build_index_docs.py b/tests/test_build_index_docs.py new file mode 100644 index 00000000..7a9e7251 --- /dev/null +++ b/tests/test_build_index_docs.py @@ -0,0 +1,95 @@ +import importlib.util +import os +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).parents[1] / "scripts/build_index_docs.py" +SPEC = importlib.util.spec_from_file_location("build_index_docs", SCRIPT_PATH) +BUILD = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(BUILD) + + +def build_page(rel_path, text): + """Write text at rel_path under a temp repo root and build its documents.""" + with tempfile.TemporaryDirectory() as root: + path = Path(root) / rel_path + path.parent.mkdir(parents=True) + path.write_text(text, encoding="utf-8") + cwd = os.getcwd() + os.chdir(root) + try: + return list(BUILD.build(rel_path)) + finally: + os.chdir(cwd) + + +class BuildIndexDocsTest(unittest.TestCase): + def test_long_chinese_page_is_fully_indexed_within_byte_limit(self): + paragraphs = [f"第{i}段:分派策略按顺序匹配,命中后通知值班人员。" * 12 for i in range(60)] + text = "---\ntitle: 分派策略\n---\n\n## 配置要素\n\n" + "\n\n".join(paragraphs) + docs = build_page("zh/on-call/escalation.mdx", text) + + self.assertGreater(len(docs), 1) + for doc in docs: + self.assertLessEqual(len(doc["content"].encode("utf-8")), BUILD.MAX_CHUNK_BYTES) + self.assertNotIn("�", doc["content"]) + joined = "\n".join(doc["content"] for doc in docs) + for paragraph in paragraphs: + self.assertIn(paragraph, joined) + + def test_single_oversized_line_splits_on_character_boundaries(self): + line = "告警" * 3000 + docs = build_page("zh/a.mdx", "---\ntitle: T\n---\n\n" + line) + for doc in docs: + self.assertLessEqual(len(doc["content"].encode("utf-8")), BUILD.MAX_CHUNK_BYTES) + body = "".join(doc["content"].split("\n\n", 1)[1] for doc in docs) + self.assertEqual(body, line) + + def test_tags_are_stripped_but_titles_placeholders_and_code_survive(self): + text = ( + "---\ntitle: Web SDK\n---\n\n" + "\n\nReplace first.\n\n\n\n" + "Keep it secret.\n\n" + "```html\n\n```\n" + ) + content = build_page("en/rum/web.mdx", text)[0]["content"] + + self.assertIn("Create the app", content) + self.assertIn("", content) + self.assertIn('', content) + self.assertIn("Keep it secret.", content) + self.assertNotIn("", content) + self.assertNotIn(" Rules > Conditions\n\n")) + self.assertTrue(docs[-1]["content"].startswith("Routing > Rules > Conditions\n\n")) + + def test_ids_locale_and_urls(self): + docs = build_page("zh/on-call/a.mdx", "---\ntitle: A\n---\n\n" + "\n\n".join(["y" * 2000] * 3)) + pid = BUILD.page_id("zh/on-call/a.mdx") + self.assertEqual([d["id"] for d in docs], [f"{pid}-{i:03d}" for i in range(len(docs))]) + self.assertEqual({d["locale"] for d in docs}, {"zh-CN"}) + self.assertEqual(docs[0]["url"], BUILD.BASE_URL + "/zh/on-call/a") + + relative = build_page("zh/openapi.mdx", "---\ntitle: API\ndescription: Open API\nurl: /zh/openapi/introduction\n---\n") + self.assertEqual(relative[0]["url"], BUILD.BASE_URL + "/zh/openapi/introduction") + + def test_link_only_page_is_indexed_by_its_description(self): + text = '---\ntitle: Terraform Provider\ndescription: "Manage Flashduty resources as code"\nurl: https://registry.terraform.io/providers/flashcatcloud/flashduty\n---\n' + docs = build_page("en/developer/terraform.mdx", text) + + self.assertEqual(len(docs), 1) + self.assertEqual(docs[0]["content"], "Terraform Provider\n\nManage Flashduty resources as code") + self.assertEqual(docs[0]["url"], "https://registry.terraform.io/providers/flashcatcloud/flashduty") + + +if __name__ == "__main__": + unittest.main()