From 6ab86258835e7ba8baefb6e07715b27ecaf9473e Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:06:04 +0800 Subject: [PATCH 01/10] =?UTF-8?q?feat:=20add=20sync=20pipeline=20script=20?= =?UTF-8?q?(embedded-docs=20=E2=86=92=20Comfy-Org/docs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add scripts/sync_to_docs.py, the script that generates the built-in-nodes/* .mdx pages in Comfy-Org/docs from the doc sources in this repo (en.md + zh/ja/ko translations), and updates docs.json navigation with case-corrected slugs. Key behaviors: - Concrete SEO descriptions extracted from each node's en.md overview (first sentence) instead of a templated string — the GEO improvement previously only possible by hand-editing individual .mdx files - Per-locale published-name resolution (macOS-safe, case-sensitive) - MDX safety: code blocks preserved verbatim, Mintlify components (//...) kept raw, unknown tags escaped in pairs - Assets copied to images/built-in-nodes// This makes the pipeline that owns these pages visible and maintainable in the same repo as the content, so fixes (e.g. slug casing, MDX escaping, description quality) land at the source instead of being overwritten by the next sync. --- scripts/README.md | 65 +++ scripts/sync_to_docs.py | 1183 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 1248 insertions(+) create mode 100644 scripts/README.md create mode 100644 scripts/sync_to_docs.py diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..7d291c49f --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,65 @@ +# Sync pipeline: embedded-docs → Comfy-Org/docs + +This directory contains the scripts that generate the `built-in-nodes/*` pages on +[docs.comfy.org](https://docs.comfy.org) from the documentation sources in this +repository (`comfyui_embedded_docs/docs//{en,zh,ja,ko}.md`). + +## `sync_to_docs.py` + +Converts every node's `en.md` (+ `zh.md` / `ja.md` / `ko.md` when present) into an +`.mdx` page in a checkout of [Comfy-Org/docs](https://github.com/Comfy-Org/docs), +and updates the `docs.json` navigation (slug casing, category groups, locale tabs). + +### Usage + +```bash +# Point at a Comfy-Org/docs checkout (defaults to ../docs relative to this repo) +export TARGET_DOCS=/path/to/comfy/docs + +# Dry-run (no files written) +python3 scripts/sync_to_docs.py --node Canny --dry-run + +# Sync a single node +python3 scripts/sync_to_docs.py --node Canny + +# Sync everything (all nodes with en.md) +python3 scripts/sync_to_docs.py --mode all +``` + +Optional env vars: + +| Var | Purpose | +|-----|---------| +| `TARGET_DOCS` | Comfy-Org/docs checkout root (default: `../docs` next to this repo) | +| `COMFYUI_PATH` | ComfyUI source checkout, used only to extract node categories when scanner output is absent | +| `ALL_NODES_INFO` | Path to scanner output JSON (`{nodes: {name: {category, ...}}}`), enables category lookup without ComfyUI source | + +### What it generates + +- **Per-locale `.mdx`**: `built-in-nodes/X.mdx`, `zh/built-in-nodes/X.mdx`, `ja/...`, `ko/...` +- **Frontmatter**: title + a **concrete SEO description** extracted from the node's + `en.md` overview first sentence (not a templated string), `sidebarTitle`, icon, wide mode +- **`docs.json` nav**: adds/updates the node slug under the right category group for + all 4 locales, with case-corrected slugs matching the on-disk files +- **Assets**: copies referenced images to `images/built-in-nodes//` + +### MDX safety + +`_normalize_mdx_content()` makes the Markdown source safe for Mintlify's MDX parser: + +- Fenced code blocks are preserved byte-for-byte (never escaped) +- Whitelisted HTML tags (`video`, `source`, `p`, `br`, ...) stay raw +- Paired Mintlify components (`...`, ``, ``, ...) stay raw +- Unknown tags (`` API syntax examples) are escaped to `<bbox>` in pairs +- Orphaned closing tags are escaped (avoids acorn "Unexpected closing slash" errors) +- Comparison operators in prose (`<= 3840`) are escaped + +### Notes + +- Node slugs in `docs.json` must match the on-disk `.mdx` filename **exactly** + (Mintlify routing is case-sensitive). `published_node_name()` resolves the + published name per locale against real directory entries — important on macOS, + where `Path.is_file()` cannot distinguish `CLIPTextEncodeControlnet.mdx` from + `ClipTextEncodeControlnet.mdx` (case-insensitive APFS). +- After syncing, a PR against Comfy-Org/docs is opened separately (the pipeline + itself only writes files and updates `docs.json` locally). diff --git a/scripts/sync_to_docs.py b/scripts/sync_to_docs.py new file mode 100644 index 000000000..5ffe15f9d --- /dev/null +++ b/scripts/sync_to_docs.py @@ -0,0 +1,1183 @@ +#!/usr/bin/env python3 +""" +Sync embedded-docs (en.md / zh.md / ja.md / ko.md + assets) to comfy/docs as NodeName.mdx. +Copies images to docs/images/built-in-nodes// and updates docs.json nav. + +Env: + EMBEDDED_DOCS_PATH - repo root (default: script parent's parent) + COMFYUI_PATH - ComfyUI repo for category parsing + TARGET_DOCS - comfy/docs root (e.g. /path/to/comfy/docs) +""" + +import argparse +import json +import os +import re +import shutil +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +# ALL_NODES_INFO_PATH: optional scanner output (node_name -> { category, ... }). +# When absent, category lookup falls back to ComfyUI source extraction. +ALL_NODES_INFO_PATH = Path(os.getenv("ALL_NODES_INFO", "")) if os.getenv("ALL_NODES_INFO") else Path("") +_nodes_info_cache: Optional[Dict[str, Dict[str, Any]]] = None + + +def _load_all_nodes_info() -> Dict[str, Dict[str, Any]]: + """Load all_nodes_info.json from scanner (node_name -> { file, category?, ... }).""" + global _nodes_info_cache + if _nodes_info_cache is not None: + return _nodes_info_cache + if not ALL_NODES_INFO_PATH.exists(): + _nodes_info_cache = {} + return _nodes_info_cache + try: + with open(ALL_NODES_INFO_PATH, "r", encoding="utf-8") as f: + data = json.load(f) + _nodes_info_cache = data.get("nodes", {}) + return _nodes_info_cache + except Exception: + _nodes_info_cache = {} + return _nodes_info_cache + +# --- Self-contained path resolution ------------------------------------- +# This script is maintained inside the embedded-docs repo (scripts/sync_to_docs.py), +# so the source docs live next to it. Target (Comfy-Org/docs checkout) and +# ComfyUI source are provided via env vars, mirroring the local pipeline setup. +_SCRIPT_DIR = Path(__file__).resolve().parent.parent +EMBEDDED_DOCS_PATH = _SCRIPT_DIR +COMFYUI_PATH = Path(os.getenv("COMFYUI_PATH", "")) +TARGET_DOCS = Path(os.getenv("TARGET_DOCS", _SCRIPT_DIR / ".." / "docs")) + +DOCS_SOURCE = EMBEDDED_DOCS_PATH / "comfyui_embedded_docs" / "docs" +BUILTIN_EN = TARGET_DOCS / "built-in-nodes" +BUILTIN_ZH = TARGET_DOCS / "zh" / "built-in-nodes" +BUILTIN_JA = TARGET_DOCS / "ja" / "built-in-nodes" +BUILTIN_KO = TARGET_DOCS / "ko" / "built-in-nodes" +IMAGES_TARGET = TARGET_DOCS / "images" / "built-in-nodes" +DOCS_JSON = TARGET_DOCS / "docs.json" + +# Locale sync + docs.json navigation (language code -> config) +LOCALE_CONFIGS: List[Dict[str, Any]] = [ + { + "code": "en", + "md_file": "en.md", + "builtin_dir": BUILTIN_EN, + "page_prefix": "built-in-nodes", + "tab": "Built-in Nodes", + "wrapper": "Nodes", + "default_group": "Advanced", + "lang_idx": 0, + }, + { + "code": "zh", + "md_file": "zh.md", + "builtin_dir": BUILTIN_ZH, + "page_prefix": "zh/built-in-nodes", + "tab": "内置节点", + "wrapper": "节点", + "default_group": "高级", + "lang_idx": 1, + }, + { + "code": "ja", + "md_file": "ja.md", + "builtin_dir": BUILTIN_JA, + "page_prefix": "ja/built-in-nodes", + "tab": "組み込みノード", + "wrapper": "ノード", + "default_group": "上級", + "lang_idx": 2, + }, + { + "code": "ko", + "md_file": "ko.md", + "builtin_dir": BUILTIN_KO, + "page_prefix": "ko/built-in-nodes", + "tab": "내장 노드 (Built-in Nodes)", + "wrapper": "노드", + "default_group": "고급", + "lang_idx": 3, + }, +] + +SCAN_PATHS = [ + COMFYUI_PATH / "nodes.py", + COMFYUI_PATH / "comfy_extras", + COMFYUI_PATH / "comfy_api_nodes", +] + +# ComfyUI category (first segment) -> (EN, zh, ja, ko group labels) for docs.json +# If a category is not in this map, a new group is created with the category name (EN); other locales use EN when no translation. +CATEGORY_TO_GROUP = { + "conditioning": ("Conditioning", "条件", "条件付け", "컨디셔닝"), + "loaders": ("Loader", "加载器", "ローダー", "로더"), + "image": ("Image", "图像", "画像", "이미지"), + "latent": ("Latent", "潜变量", "潜在変数", "잠재 변수"), + "sampling": ("Sampling", "采样", "サンプリング", "샘플링"), + "3d": ("3D", "3D", "3D", "3D"), + "3D": ("3D", "3D", "3D", "3D"), + "advanced": ("Advanced", "高级", "上級", "고급"), + "utils": ("Utils", "实用工具", "ユーティリティ", "유틸리티"), + "utility": ("Utils", "实用工具", "ユーティリティ", "유틸리티"), + "util": ("Utils", "实用工具", "ユーティリティ", "유틸리티"), + "_for_testing": ("Advanced", "高级", "上級", "고급"), + "api": ("API", "API", "API", "API"), + "api node": ("API Node", "API Node", "API Node", "API Node"), + "model_patches": ("Model Patches", "模型补丁", "モデルパッチ", "모델 패치"), + "dataset": ("Image", "图像", "画像", "이미지"), + "audio": ("Audio", "Audio", "Audio", "Audio"), + "basics": ("Basics", "Basics", "Basics", "Basics"), + "camera": ("Camera", "Camera", "Camera", "Camera"), + "context": ("Context", "Context", "Context", "Context"), + "image generation": ("Image", "图像", "画像", "이미지"), + "image tools": ("Image", "图像", "画像", "이미지"), + "logic": ("Logic", "Logic", "Logic", "Logic"), + "mask": ("Mask", "Mask", "Mask", "Mask"), + "textgen": ("Textgen", "Textgen", "Textgen", "Textgen"), + "training": ("Training", "Training", "Training", "Training"), + "transform": ("Transform", "Transform", "Transform", "Transform"), + "guidance": ("Sampling", "采样", "サンプリング", "샘플링"), +} +DEFAULT_GROUP_EN = "Advanced" +DEFAULT_GROUP_ZH = "高级" +DEFAULT_GROUP_JA = "上級" +DEFAULT_GROUP_KO = "고급" + +# Hardcoded category fallback for nodes that the scanner cannot resolve: +# - replacement nodes in nodes_replacements.py (no category field) +# - deprecated / aliased nodes not found in current source +# - partner API nodes with flat MDX files but no scanner entry +_FALLBACK_CATEGORY: Dict[str, str] = { + # Replacement nodes (nodes_replacements.py) — inherit from their original + "BatchImagesNode": "image", + "ConditioningAverage": "conditioning", + "ControlNetLoader": "loaders", + "HunyuanRefinerLatent": "conditioning", + "HunyuanVideo15SuperResolution": "loaders", + "ImageBatch": "image", + "ImageScaleBy": "image/upscaling", + "Load3D": "3d", + "Load3DAnimation": "3d", + "Preview3D": "3d", + "Preview3DAnimation": "3d", + "ResizeImageMaskNode": "image", + "SVD_img2vid_Conditioning": "conditioning/video_models", + "SDV_img2vid_Conditioning": "conditioning/video_models", + "T2IAdapterLoader": "loaders", + "wanBlockSwap": "utils", + # Model merging + "CLIPAdd": "advanced/model_merging", + "CLIPSubtract": "advanced/model_merging", + "SaveLoRANode": "advanced/model_merging", + # Deprecated loaders + "DeprecatedCheckpointLoader": "advanced/loaders", + "DeprecatedDiffusersLoader": "advanced/loaders", + # Model patches + "EpsilonScaling": "model_patches/unet", + # Partner API nodes (flat MDX files) + "FluxProCannyNode": "api node/image/BFL", + "FluxProDepthNode": "api node/image/BFL", + "FluxProImageNode": "api node/image/BFL", + "ByteDanceImageEditNode": "api node/image/ByteDance", + "PikaImageToVideoNode2_2": "api node/video/Pika", + "PikaScenesV2_2": "api node/video/Pika", + "PikaStartEndFrameNode2_2": "api node/video/Pika", + "PikaTextToVideoNode2_2": "api node/video/Pika", + "Pikadditions": "api node/video/Pika", + "Pikaffects": "api node/video/Pika", + "Pikaswaps": "api node/video/Pika", + # Image / dataset + "LoadImageSetFromFolderNode": "image", + "LoadImageSetNode": "image", + "LoadImageTextSetFromFolderNode": "image", + # Utils + "MarkdownNote": "utils", + "Note": "utils", + "Reroute": "utils", + "TerminalLog": "utils", + # Sampling (renamed aliases) + "SamplerDpmpp2mSde": "sampling/custom_sampling/samplers", + "SamplerDpmppSde": "sampling/custom_sampling/samplers", + # Conditioning (deprecated aliases) + "Sd4xupscaleConditioning": "conditioning", + "Stablezero123Conditioning": "conditioning/video_models", + "Stablezero123ConditioningBatched": "conditioning/video_models", + "SvdImg2vidConditioning": "conditioning/video_models", + # _for_testing nodes with no sub-path (scanner returns just "_for_testing") + "DifferentialDiffusion": "sampling", + "FreSca": "advanced", + "LatentBlend": "latent", + "LoadLatent": "loaders", + "LoraSave": "advanced/model_merging", + "Mahiro": "utils", + "PerpNeg": "conditioning", + "PerpNegGuider": "sampling", + "SamplerEulerCFGpp": "sampling/custom_sampling/samplers", + "SaveLatent": "latent", + "SelfAttentionGuidance": "sampling", + "TorchCompileModel": "advanced", + "VAEDecodeTiled": "latent", + "VAEEncodeTiled": "latent", +} + +# Reverse map: group label (EN or ZH) -> ComfyUI category root string +# Used by _restructure_group_by_category to look up which category root each group corresponds to. +_GROUP_LABEL_TO_CATEGORY_ROOT: Dict[str, str] = {} +def _build_reverse_map() -> None: + seen: Dict[str, str] = {} + for cat_key, labels in CATEGORY_TO_GROUP.items(): + for label in labels: + if label not in seen: + seen[label] = cat_key + _GROUP_LABEL_TO_CATEGORY_ROOT.update(seen) +_build_reverse_map() + + +def _default_group_for_lang(lang_idx: int) -> str: + return (DEFAULT_GROUP_EN, DEFAULT_GROUP_ZH, DEFAULT_GROUP_JA, DEFAULT_GROUP_KO)[lang_idx] + + +def _group_label_for_category(first_segment: str, lang_idx: int) -> str: + first_lower = first_segment.lower() + if first_lower in CATEGORY_TO_GROUP: + return CATEGORY_TO_GROUP[first_lower][lang_idx] + return _seg_to_label(first_segment) + + +def _find_lang_entry(nav: Dict[str, Any], lang_code: str) -> Optional[Dict[str, Any]]: + for entry in nav.get("navigation", {}).get("languages", []): + if entry.get("language") == lang_code: + return entry + return None + + +def _find_tab_pages(lang_entry: Dict[str, Any], tab_name: str) -> Optional[List[Any]]: + for tab in lang_entry.get("tabs", []): + if tab.get("tab") == tab_name and "pages" in tab: + return tab["pages"] + return None + + +def _seg_to_label(seg: str) -> str: + """Convert a path segment like 'custom_sampling' to a display label 'Custom Sampling'.""" + return seg.replace("_", " ").replace("-", " ").title() + + +def _category_to_group_and_sub(full_category_path: str, lang_idx: int = 0) -> Tuple[str, Optional[str]]: + """Return (group label, sub-group label or None) for the given locale index. + + Always uses the FIRST segment to determine the top-level group so that the + hierarchy mirrors ComfyUI's node menu exactly. + """ + if not full_category_path or not full_category_path.strip(): + return _default_group_for_lang(lang_idx), None + raw = full_category_path.strip() + segments = [s.strip() for s in raw.split("/") if s.strip()] + if not segments: + return _default_group_for_lang(lang_idx), None + group_label = _group_label_for_category(segments[0], lang_idx) + if len(segments) > 1: + return group_label, _seg_to_label(segments[1]) + return group_label, None + +# Regex for local image/asset refs and disclaimer +MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)]+)\)") +HTML_SRC_RE = re.compile(r'<(?:img|video|audio|source)[^>]+src=["\']([^"\'>]+)["\']', re.IGNORECASE) +def is_local_link(href: str) -> bool: + href = href.strip().split("#")[0].split("?")[0] + return bool(href) and not ( + href.startswith("http://") + or href.startswith("https://") + or href.startswith("data:") + ) + + +def _class_name_variants(node_name: str) -> List[str]: + """Return possible class names in source (e.g. ClipTextEncode <-> CLIPTextEncode).""" + variants = [node_name] + if node_name.startswith("Clip") and len(node_name) > 4: + variants.append("CLIP" + node_name[4:]) + elif node_name.startswith("CLIP") and len(node_name) > 4: + variants.append("Clip" + node_name[4:]) + return variants + + +def _category_from_class_block(text: str, node_name: str, first_segment_only: bool = True) -> Optional[str]: + """Extract category from a class block. If first_segment_only, return first path segment.""" + for class_name in _class_name_variants(node_name): + class_pattern = re.compile( + r"^class\s+" + re.escape(class_name) + r"\s*[:(].*?(?=^class\s|\Z)", + re.MULTILINE | re.DOTALL, + ) + m = class_pattern.search(text) + if not m: + continue + block = m.group(0) + for pattern in (r"CATEGORY\s*=\s*[\"']([^\"']+)[\"']", r"category\s*=\s*[\"']([^\"']+)[\"']"): + c = re.search(pattern, block) + if c: + raw = c.group(1).strip() + return raw.split("/")[0] if first_segment_only else raw + return None + + +def _category_from_schema_node_id(text: str, node_name: str, first_segment_only: bool = True) -> Optional[str]: + """Extract category from Schema that contains node_id='NodeName'.""" + escaped = re.escape(node_name) + idx = re.search(r'node_id\s*=\s*["\']' + escaped + r'["\']', text) + if not idx: + return None + start = max(0, idx.start() - 400) + end = min(len(text), idx.end() + 400) + window = text[start:end] + c = re.search(r'category\s*=\s*["\']([^"\']+)["\']', window) + if c: + raw = c.group(1).strip() + return raw.split("/")[0] if first_segment_only else raw + return None + + +def extract_category_full_from_comfyui(node_name: str) -> Optional[str]: + """Extract full category string from ComfyUI source (e.g. 'api node/image/ByteDance').""" + for base in SCAN_PATHS: + if not base.exists(): + continue + files = [base] if base.is_file() else list(base.rglob("*.py")) + for path in files: + if path.suffix != ".py": + continue + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except Exception: + continue + if node_name not in text and not any(v in text for v in _class_name_variants(node_name)): + continue + cat = _category_from_class_block(text, node_name, first_segment_only=False) + if cat: + return cat + cat = _category_from_schema_node_id(text, node_name, first_segment_only=False) + if cat: + return cat + return None + + +def _resolve_node_info_key(nodes: Dict[str, Dict], node_name: str) -> Optional[Dict[str, Any]]: + """Get node info from all_nodes_info by node_name, or by class variant, or by node_id.""" + if node_name in nodes: + return nodes[node_name] + for variant in _class_name_variants(node_name): + if variant in nodes: + return nodes[variant] + for key, info in nodes.items(): + if info.get("node_id") == node_name: + return info + return None + + +def scanner_node_key(node_name: str) -> Optional[str]: + """Return the all_nodes_info.json dict key for this node, if known to the scanner.""" + nodes = _load_all_nodes_info() + if node_name in nodes: + return node_name + for variant in _class_name_variants(node_name): + if variant in nodes: + return variant + for key, info in nodes.items(): + if info.get("node_id") == node_name: + return key + lower = node_name.lower() + for key in nodes: + if key.lower() == lower: + return key + for key, info in nodes.items(): + class_name = (info.get("class_name") or "").strip() + if class_name and class_name.lower() == lower: + return key + return None + + +def canonical_node_name(node_name: str) -> str: + """Return scanner node key (ComfyUI class name); falls back to node_name if not in scanner.""" + return scanner_node_key(node_name) or node_name + + +def node_name_nav_aliases(node_name: str) -> Set[str]: + """All page-key basename variants that should collapse to the same scanner node.""" + canonical = canonical_node_name(node_name) + aliases: Set[str] = {node_name, canonical} + for variant in _class_name_variants(node_name): + aliases.add(variant) + for variant in _class_name_variants(canonical): + aliases.add(variant) + lower = canonical.lower() + for key in _load_all_nodes_info(): + if key.lower() == lower: + aliases.add(key) + return aliases + + +def _locale_code_for_page_key(page_key: str) -> str: + """Infer locale code from a page key prefix (e.g. zh/built-in-nodes/X -> zh).""" + for cfg in LOCALE_CONFIGS: + if page_key.startswith(cfg["page_prefix"] + "/"): + return cfg["code"] + return "en" + + +def published_node_name(node_name: str, locale_code: Optional[str] = None) -> str: + """MDX filename and docs.json page basename — keep existing on-disk name when already published. + + macOS APFS is case-insensitive, so Path.is_file() cannot distinguish + CLIPTextEncodeControlnet.mdx from ClipTextEncodeControlnet.mdx (same inode). + Instead we match against the real on-disk directory entries (os.listdir names), + which preserve the true casing/spelling committed to git. Historical spellings + (e.g. HunyuanDit vs HunyuanDiT, Sdxl vs SDXL) are handled via casefold fallback. + + Per-locale: en/zh/ja on-disk files are uppercase CLIP..., ko is lowercase Clip..., + so the published name must be resolved against each locale's own directory. + """ + scanner = canonical_node_name(node_name) + locales = [c for c in LOCALE_CONFIGS if locale_code is None or c["code"] == locale_code] + aliases = sorted(node_name_nav_aliases(scanner), key=len, reverse=True) + for locale in locales: + d = locale["builtin_dir"] + if not d.is_dir(): + continue + try: + entries = os.listdir(d) + except OSError: + continue + mdx_names = {e[:-4] for e in entries if e.endswith(".mdx")} + # 1) exact (case-sensitive) alias match against real on-disk names + for alias in aliases: + if alias in mdx_names: + return alias + # 2) casefold match: return the real on-disk spelling (covers DiT/Dit, SDXL/Sdxl, CLIP/Clip) + folded = {n.casefold(): n for n in mdx_names} + for alias in aliases: + if alias.casefold() in folded: + return folded[alias.casefold()] + return scanner + + +def canonical_page_key(page_key: str) -> str: + """Normalize a docs.json page key to the published on-disk MDX basename.""" + parts = page_key.split("/") + if not parts: + return page_key + published = published_node_name(parts[-1], _locale_code_for_page_key(page_key)) + if parts[-1] == published: + return page_key + return "/".join([*parts[:-1], published]) + + +def resolve_source_node(node_name: str) -> Tuple[str, Path]: + """Return (scanner canonical name, embedded-docs dir) for reading en.md / zh.md / ja.md.""" + canonical = canonical_node_name(node_name) + for candidate in node_name_nav_aliases(node_name): + node_dir = DOCS_SOURCE / candidate + if (node_dir / "en.md").exists(): + return canonical, node_dir + if DOCS_SOURCE.exists(): + lower = canonical.lower() + for node_dir in DOCS_SOURCE.iterdir(): + if node_dir.is_dir() and node_dir.name.lower() == lower and (node_dir / "en.md").exists(): + return canonical, node_dir + return canonical, DOCS_SOURCE / canonical + + +def list_nodes_with_en_md() -> List[str]: + """List embedded-docs nodes deduped by scanner canonical name.""" + seen: Set[str] = set() + nodes: List[str] = [] + for node_dir in sorted(DOCS_SOURCE.iterdir()): + if not node_dir.is_dir() or not (node_dir / "en.md").exists(): + continue + canonical = canonical_node_name(node_dir.name) + if canonical in seen: + continue + seen.add(canonical) + nodes.append(canonical) + return nodes + + +def get_full_category_for_node(node_name: str) -> Optional[str]: + """Get full category string for node: prefer scanner all_nodes_info.json (look up by name/variant/node_id), else extract from source.""" + nodes = _load_all_nodes_info() + info = _resolve_node_info_key(nodes, node_name) + if info and info.get("category"): + raw = info["category"].strip() + return raw if raw else None + return extract_category_full_from_comfyui(node_name) + + +def get_category_for_node(node_name: str) -> Optional[str]: + """Get category (first segment) for node: prefer scanner all_nodes_info.json, else extract from ComfyUI source.""" + full = get_full_category_for_node(node_name) + if full: + return full.split("/")[0] + return extract_category_from_comfyui(node_name) + + +def extract_category_from_comfyui(node_name: str) -> Optional[str]: + """Find node in ComfyUI source and extract CATEGORY/category for this node only. Returns first segment (e.g. sampling from sampling/custom_sampling).""" + for base in SCAN_PATHS: + if not base.exists(): + continue + files = [base] if base.is_file() else list(base.rglob("*.py")) + for path in files: + if path.suffix != ".py": + continue + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except Exception: + continue + if node_name not in text and not any(v in text for v in _class_name_variants(node_name)): + continue + # Prefer: category from the class block that defines this node + cat = _category_from_class_block(text, node_name) + if cat: + return cat + # Fallback: Schema with node_id (e.g. node_id="AddNoise" ... category="...") + cat = _category_from_schema_node_id(text, node_name) + if cat: + return cat + return None + + +def collect_page_keys(pages: List[Any], prefix: str = "") -> Set[str]: + """Recursively collect all page string keys from a tab's pages.""" + out: Set[str] = set() + for item in pages: + if isinstance(item, str): + out.add(item) + elif isinstance(item, dict): + if "pages" in item: + out |= collect_page_keys(item["pages"], prefix) + return out + + +def flatten_builtin_pages(pages: List[Any], key_prefix: str) -> List[str]: + """ + Recursively collect built-in-nodes page keys (e.g. built-in-nodes/NodeName or zh/built-in-nodes/NodeName) + and return a sorted flat list. Used for flat/collapsed sidebar (no groups). + """ + keys: Set[str] = set() + for item in pages: + if isinstance(item, str) and (item == key_prefix or item.startswith(key_prefix + "/")): + keys.add(item) + elif isinstance(item, dict) and "pages" in item: + keys |= set(flatten_builtin_pages(item["pages"], key_prefix)) + return sorted(keys) + + +def find_group_in_pages(pages: List[Any], group_label: str) -> Optional[List[Any]]: + """Find the top-level group with 'group' == group_label and return its 'pages' list.""" + for item in pages: + if isinstance(item, dict) and item.get("group") == group_label and "pages" in item: + return item["pages"] + return None + + +def find_or_create_group_in_pages(pages: List[Any], group_label: str) -> List[Any]: + """Find group with group_label in pages, or create it and append. Return that group's 'pages' list.""" + for item in pages: + if isinstance(item, dict) and item.get("group") == group_label and "pages" in item: + return item["pages"] + new_group: Dict[str, Any] = {"group": group_label, "pages": []} + pages.append(new_group) + return new_group["pages"] + + +def remove_page_from_pages(pages: List[Any], page_key: str) -> None: + """Remove page_key from pages tree in place (recursive).""" + i = 0 + while i < len(pages): + item = pages[i] + if isinstance(item, str): + if item == page_key: + pages.pop(i) + continue + elif isinstance(item, dict) and "pages" in item: + remove_page_from_pages(item["pages"], page_key) + i += 1 + + +def _remove_group_from_pages(pages: List[Any], group_label: str) -> None: + """Remove the first top-level group with given label from pages (in place).""" + for i, item in enumerate(pages): + if isinstance(item, dict) and item.get("group") == group_label: + pages.pop(i) + return + + +def _sort_pages_alphabetically(pages: List[Any], groups_first: bool = True) -> None: + """Sort pages array in place: recursively sort nested 'pages', then sort this level. + + groups_first=True → sub-groups before flat page strings (used inside category groups + so collapsible folders appear above the flat node list). + groups_first=False → flat page strings before sub-groups (used at tab level so + standalone pages like 'overview' stay at the very top). + Both labels are compared case-insensitively. + """ + for item in pages: + if isinstance(item, dict) and "pages" in item: + # Inner levels always put sub-groups first + _sort_pages_alphabetically(item["pages"], groups_first=True) + if groups_first: + # groups (0) before flat pages (1) + pages.sort(key=lambda x: (0, x.get("group", "").lower()) if isinstance(x, dict) else (1, x.lower())) + else: + # flat pages (0) before groups (1) — keeps 'overview' at top of tab + pages.sort(key=lambda x: (1, x.get("group", "").lower()) if isinstance(x, dict) else (0, x.lower())) + + +def _rebuild_wrapper_groups( + wrapper_pages: List[Any], + node_cat_map: Dict[str, str], + lang_idx: int = 0, +) -> None: + """Completely rebuild all non-API groups inside wrapper_pages from scratch. + + Collects every page key (except those in API Node), then re-places each one + using the first-segment rule that mirrors ComfyUI's node menu hierarchy. + This corrects any historical mis-placements without requiring a full re-sync. + + lang_idx: 0 = EN, 1 = zh, 2 = ja, 3 = ko labels from CATEGORY_TO_GROUP. + """ + # Preserve the API Node group as-is + api_node_item: Optional[Dict[str, Any]] = None + for item in wrapper_pages: + if isinstance(item, dict) and item.get("group") == "API Node": + api_node_item = item + break + + api_keys: Set[str] = set(collect_page_keys(api_node_item["pages"])) if api_node_item else set() + all_keys: Set[str] = set(collect_page_keys(wrapper_pages)) + non_api_keys = {canonical_page_key(k) for k in (all_keys - api_keys)} + + # Build case-insensitive lookup for node_cat_map (handles ClipLoader → CLIPLoader mismatches) + lower_cat_map: Dict[str, str] = {k.lower(): v for k, v in node_cat_map.items()} + # Build case-insensitive lookup for fallback map + lower_fallback: Dict[str, str] = {k.lower(): v for k, v in _FALLBACK_CATEGORY.items()} + + # Rebuild wrapper: clear everything, re-add API Node, then re-place all other keys + wrapper_pages.clear() + if api_node_item is not None: + wrapper_pages.append(api_node_item) + + for key in sorted(non_api_keys): + key_parts = Path(key).parts # e.g. ('built-in-nodes', 'conditioning', 'video-models', 'wan-vace-to-video') + node_name = published_node_name(key_parts[-1]) + if key_parts[-1] != node_name: + key = "/".join([*key_parts[:-1], node_name]) + + # If the page key has intermediate path segments (nested MDX), derive category from path + # e.g. built-in-nodes/conditioning/video-models/foo → conditioning/video_models + prefix_dirs = tuple(cfg["page_prefix"] for cfg in LOCALE_CONFIGS) + path_derived_cat = "" + for pfx in prefix_dirs: + pfx_parts = Path(pfx).parts + if key_parts[: len(pfx_parts)] == pfx_parts and len(key_parts) > len(pfx_parts) + 1: + # Middle segments are the category + mid = key_parts[len(pfx_parts) : -1] + path_derived_cat = "/".join(mid).replace("-", "_") + break + + # Step 1: get raw category from scanner (case-insensitive fallback included) + scanner_cat = ( + node_cat_map.get(node_name) + or lower_cat_map.get(node_name.lower()) + or "" + ).strip() + + # Step 2: get explicit fallback (always takes priority over _for_testing scanner result) + fallback_cat = ( + _FALLBACK_CATEGORY.get(node_name) + or lower_fallback.get(node_name.lower()) + or "" + ).strip() + + # Step 3: choose final category + if path_derived_cat: + # Nested MDX path (e.g. conditioning/video-models/...) — use path directly + full_cat = path_derived_cat + elif scanner_cat and not scanner_cat.startswith("_for_testing"): + # Scanner returned a clean category — use it + full_cat = scanner_cat + elif fallback_cat: + # Explicit fallback always wins over _for_testing scanner result + full_cat = fallback_cat + elif scanner_cat.startswith("_for_testing"): + # Remap _for_testing/* to proper top-level paths + rest = scanner_cat[len("_for_testing"):].lstrip("/") + if not rest: + full_cat = "" + elif rest.startswith("custom_sampling"): + full_cat = "sampling/" + rest + elif rest.startswith("conditioning"): + full_cat = rest + elif rest.startswith("stable_cascade"): + full_cat = "conditioning/" + rest + else: + full_cat = "advanced/" + rest + else: + full_cat = "" + + # Route nodes whose category starts with "api node" into the API Node group + if full_cat.lower().startswith("api node"): + if api_node_item is None: + api_node_item = {"group": "API Node", "pages": []} + wrapper_pages.insert(0, api_node_item) + # Build sub-path inside API Node: segs after "api node" + segs = [s.strip() for s in full_cat.split("/") if s.strip()][1:] + target = api_node_item["pages"] + for seg in segs: + target = find_or_create_group_in_pages(target, _seg_to_label(seg)) + if key not in target: + target.append(key) + continue + + segs = [s.strip() for s in full_cat.split("/") if s.strip()] + + if not segs: + group_label = _default_group_for_lang(lang_idx) + else: + group_label = _group_label_for_category(segs[0], lang_idx) + + target = find_or_create_group_in_pages(wrapper_pages, group_label) + for seg in segs[1:]: + target = find_or_create_group_in_pages(target, _seg_to_label(seg)) + if key not in target: + target.append(key) + + +def _remove_empty_groups(pages: List[Any]) -> None: + """Remove groups with empty 'pages' in place (recursive, bottom-up).""" + i = 0 + while i < len(pages): + item = pages[i] + if isinstance(item, dict) and "pages" in item: + _remove_empty_groups(item["pages"]) + if len(item["pages"]) == 0: + pages.pop(i) + continue + i += 1 + + +def _migrate_toplevel_groups_to_wrapper(pages: List[Any], wrapper_label: str) -> None: + """Move any top-level dict groups (other than wrapper_label) inside the wrapper group. + + This ensures previously added top-level groups (3D, API Node, etc.) become nested + inside the wrapper so Mintlify renders them as collapsible entries. + """ + orphans: List[Dict[str, Any]] = [] + i = 0 + while i < len(pages): + item = pages[i] + if isinstance(item, dict) and "pages" in item and item.get("group") != wrapper_label: + orphans.append(item) + pages.pop(i) + else: + i += 1 + if not orphans: + return + wrapper = find_or_create_group_in_pages(pages, wrapper_label) + for orphan in orphans: + existing = find_group_in_pages(wrapper, orphan["group"]) + if existing is not None: + # Merge pages; avoid duplicates + for p in orphan["pages"]: + if p not in existing: + existing.append(p) + else: + wrapper.append(orphan) + + +def get_description_from_content(content: str) -> str: + """Extract the first sentence from the first content paragraph. + + Used as the seed for the SEO meta description (see build_seo_description). + Skips AI-generated disclaimer blockquote lines and headings. + """ + lines = content.split("\n") + first_para: List[str] = [] + for line in lines: + line_stripped = line.strip() + if not line_stripped: + if first_para: + break + continue + if line_stripped.startswith("##") or line_stripped.startswith("#"): + break + if line_stripped.startswith("> ") and ( + "AI-generated" in line_stripped + or "AI 生成" in line_stripped + or "AI によって生成" in line_stripped + or "AI에 의해 생성" in line_stripped + ): + continue + first_para.append(line_stripped) + paragraph = " ".join(first_para) if first_para else "" + # Return only the first sentence (up to first ". " or end of paragraph) + for sep in (". ", "。"): + idx = paragraph.find(sep) + if idx != -1: + return paragraph[: idx + 1] + return paragraph[:160] if paragraph else "" + + +def find_local_asset_refs(content: str, doc_dir: Path) -> List[Tuple[str, Path]]: + """Return list of (original_ref, resolved_absolute_path) for local assets.""" + refs: List[Tuple[str, Path]] = [] + seen: Set[str] = set() + for m in MD_IMAGE_RE.finditer(content): + href = m.group(1).strip() + if not is_local_link(href): + continue + path = (doc_dir / href.split("#")[0].split("?")[0]).resolve() + if path.exists() and path.is_file() and path not in [p for _, p in refs]: + refs.append((href, path)) + for m in HTML_SRC_RE.finditer(content): + href = m.group(1).strip() + if not is_local_link(href): + continue + path = (doc_dir / href.split("#")[0].split("?")[0]).resolve() + if path.exists() and path.is_file() and path not in [p for _, p in refs]: + refs.append((href, path)) + return refs + + +def copy_assets_and_rewrite( + content: str, + doc_dir: Path, + node_name: str, + images_out_dir: Path, + dry_run: bool, +) -> str: + """Copy referenced assets to images_out_dir and rewrite refs to /images/built-in-nodes/NodeName/xxx.""" + refs = find_local_asset_refs(content, doc_dir) + out = content + for orig, abs_path in refs: + filename = abs_path.name + new_ref = f"/images/built-in-nodes/{node_name}/{filename}" + if not dry_run: + images_out_dir.mkdir(parents=True, exist_ok=True) + dest = images_out_dir / filename + if abs_path != dest: + shutil.copy2(abs_path, dest) + # Replace in content (use orig as-is to avoid re-escaping) + out = out.replace(orig, new_ref) + return out + + +def _escape_frontmatter_description(description: str) -> str: + """Escape backslashes and double-quotes for YAML frontmatter (avoids parsing errors).""" + return description.replace("\\", "\\\\").replace('"', '\\"') + + +def _normalize_mdx_content(content: str) -> str: + """Apply MDX-safe normalizations: strip H1 title, self-closing tags, etc. + + Strips the leading H1 (# Title) line — the sidebar title in frontmatter + already provides the heading, so a duplicate H1 is redundant on the page. + """ + # Strip leading H1 (# Title) and any blank lines after it + content = re.sub(r'^#\s+[^\n]*\n?\n*', '', content, count=1) + + # Protect fenced code blocks from ALL escaping below: inside ``` blocks the + # content must stay byte-identical (CommonMark renders code blocks verbatim, + # so <= would show literally instead of <=). + _code_blocks: List[str] = [] + def _stash_code(m: "re.Match[str]") -> str: + _code_blocks.append(m.group(0)) + return f"\x00CODEBLOCK{len(_code_blocks) - 1}\x00" + content = re.sub(r"```.*?```", _stash_code, content, flags=re.DOTALL) + + content = content.replace("
", "
") + content = re.sub(r"(]+)>", r"\1 />", content) + content = re.sub(r"<(https?://[^>\s]+)>", r"[\1](\1)", content) + # Escape comparison-style angle brackets like <1.0, <100, <= 3840 that are NOT HTML/JSX tags. + # Must escape <= first, then already handled above). + content = re.sub(r"<=", r"<=", content) + content = re.sub(r"<(\d)", r"<\1", content) + # Escape any remaining bare < that is not an HTML/JSX tag (e.g. and are intentional paired components + # (e.g. Mintlify's ..., , ) — keep them raw. + # Only orphaned tags (opening without closing, or closing without opening) get escaped. + _MDX_HTML_TAGS = ("br", "source", "img", "video", "audio", "a", "p", "div", "span", "table", "tr", "td", "th", "ul", "ol", "li", "code", "pre", "strong", "em", "b", "i") + # Mintlify built-in components — must stay raw, otherwise the site renders + # literal <Note> text instead of the Note callout. + _MINTLIFY_COMPONENTS = ( + "Accordion", "AccordionGroup", "Badge", "Card", "CardGroup", "CodeGroup", + "FileTree", "Frame", "Icon", "Info", "Note", "Param", "RequestExample", + "ResponseExample", "SettingsMenuContext", "Step", "Steps", "Tab", "Tabs", + "Tip", "Update", "UpdateReminder", "Warning", "ReqHint", + ) + _keep_raw = set(_MDX_HTML_TAGS) + # Mintlify components stay raw ONLY when paired (opening + closing both present). + # Orphaned (e.g. from an older escaped ) must still be escaped, + # otherwise MDX acorn fails with "Unexpected closing slash". + # Unknown tags (e.g. API syntax examples) are NEVER kept raw. + _open_tags = set(re.findall(r"<([a-zA-Z_][a-zA-Z0-9]*)(?:\s|>)", content)) + _close_tags = set(re.findall(r"", content)) + _keep_raw |= set(_MINTLIFY_COMPONENTS) & _open_tags & _close_tags + # Escape closing tags -> </Note> when not kept raw (orphaned closing tag). + # Must run BEFORE the opening-tag rule so pairs stay consistent. + content = re.sub( + r"", + lambda m: m.group(0) if m.group(1) in _keep_raw else "</" + m.group(1) + ">", + content, + ) + content = re.sub(r"<([a-zA-Z_][a-zA-Z0-9]*)", lambda m: m.group(0) if m.group(1) in _keep_raw else "<" + m.group(1), content) + + # Restore code blocks verbatim + def _restore_code(m: "re.Match[str]") -> str: + return _code_blocks[int(m.group(1))] + content = re.sub(r"\x00CODEBLOCK(\d+)\x00", _restore_code, content) + return content + + +def build_frontmatter(node_name: str, description: str) -> str: + """Frontmatter with a concrete, node-specific SEO description. + + `description` is the first sentence extracted from the node's en.md + overview (see get_description_from_content). Using it instead of a + templated string gives every node page a real, searchable summary — + the GEO improvement that previously could only be done by hand-editing + individual .mdx files (e.g. Comfy-Org/docs#1216). + """ + seo_desc = (description or "").strip() + if not seo_desc: + seo_desc = f"Complete documentation for the {node_name} node in ComfyUI. Learn its inputs, outputs, parameters and usage." + # Keep meta descriptions reasonably short; strip markdown/backticks noise + seo_desc = re.sub(r"[`*_#>]", "", seo_desc) + seo_desc = seo_desc[:180] + seo_desc_escaped = _escape_frontmatter_description(seo_desc) + return f"""--- +title: "{node_name} - ComfyUI Built-in Node Documentation" +description: "{seo_desc_escaped}" +sidebarTitle: "{node_name}" +icon: "circle" +mode: wide +--- + +""" + + +def _normalize_category(raw: Optional[str]) -> str: + raw = (raw or "").strip() + raw_lower = raw.lower() + if raw_lower.startswith("partner node"): + raw = "api node" + raw[len("partner node"):] + return raw + + +def _purge_noncanonical_nav_pages(tab_pages: List[Any]) -> None: + """Remove duplicate docs.json page keys (e.g. CLIPLoader when ClipLoader.mdx is already published).""" + for key in sorted(collect_page_keys(tab_pages)): + parts = key.split("/") + if not parts: + continue + locale_code = _locale_code_for_page_key(key) + if parts[-1] != published_node_name(parts[-1], locale_code): + remove_page_from_pages(tab_pages, key) + + +def _place_page_in_nav( + tab_pages: List[Any], + page_key: str, + full_category: str, + locale: Dict[str, Any], +) -> None: + """Insert page_key under the correct Built-in Nodes group for one locale.""" + page_key = canonical_page_key(page_key) + raw = _normalize_category(full_category) + raw_lower = raw.lower() + lang_idx = locale["lang_idx"] + wrapper = find_or_create_group_in_pages(tab_pages, locale["wrapper"]) + + if raw_lower.startswith("api node"): + parts = [p.strip() for p in raw.split("/") if p.strip()] + type_label = _seg_to_label(parts[1]) if len(parts) >= 2 else "Other" + provider_label = _seg_to_label(parts[2]) if len(parts) >= 3 else None + api_pages = find_or_create_group_in_pages(wrapper, "API Node") + if type_label: + type_pages = find_or_create_group_in_pages(api_pages, type_label) + if provider_label: + provider_pages = find_or_create_group_in_pages(type_pages, provider_label) + if page_key not in provider_pages: + provider_pages.append(page_key) + elif page_key not in type_pages: + type_pages.append(page_key) + elif page_key not in api_pages: + api_pages.append(page_key) + return + + group_label, sub_label = _category_to_group_and_sub(raw, lang_idx) + gp = find_or_create_group_in_pages(wrapper, group_label) + if sub_label: + sub_gp = find_or_create_group_in_pages(gp, sub_label) + if page_key not in sub_gp: + sub_gp.append(page_key) + elif page_key not in gp: + gp.append(page_key) + + +def sync_node( + node_name: str, + dry_run: bool, +) -> Tuple[bool, Optional[str], List[str]]: + """Sync one node: en.md (+ zh.md / ja.md when present) -> MDX and copy assets.""" + scanner_name, node_dir = resolve_source_node(node_name) + en_md = node_dir / "en.md" + if not en_md.exists(): + print(f" Skip {node_name}: no en.md") + return False, None, [] + + published_en = published_node_name(scanner_name, "en") + images_out = IMAGES_TARGET / published_en + content_en = en_md.read_text(encoding="utf-8") + description = get_description_from_content(content_en) + synced_locales: List[str] = [] + + for locale in LOCALE_CONFIGS: + md_path = node_dir / locale["md_file"] + if locale["code"] != "en" and not md_path.exists(): + continue + + # Per-locale published name: en/zh/ja on-disk files are uppercase CLIP..., + # ko is lowercase Clip... — resolve against this locale's own directory. + published = published_en if locale["code"] == "en" else published_node_name(scanner_name, locale["code"]) + + content = md_path.read_text(encoding="utf-8") + content = copy_assets_and_rewrite(content, node_dir, published_en, images_out, dry_run) + content = _normalize_mdx_content(content) + mdx = build_frontmatter(scanner_name, description or f"Documentation for {scanner_name} node.") + content + + target_mdx = locale["builtin_dir"] / f"{published}.mdx" + if not dry_run: + locale["builtin_dir"].mkdir(parents=True, exist_ok=True) + target_mdx.write_text(mdx, encoding="utf-8") + print(f" {locale['code'].upper()}: {locale['page_prefix']}/{published}.mdx") + synced_locales.append(locale["code"]) + + full_category = get_full_category_for_node(scanner_name) if not dry_run else None + return True, full_category, synced_locales + + +def main(): + parser = argparse.ArgumentParser(description="Sync embedded-docs to comfy/docs (built-in-nodes + docs.json)") + parser.add_argument("--node", type=str, help="Sync only this node") + parser.add_argument("--mode", choices=("all", "test"), default="test", help="all = every node with en.md; test = first N") + parser.add_argument("--count", type=int, default=10, help="N for test mode") + parser.add_argument("--dry-run", action="store_true", help="Do not write files") + parser.add_argument("--no-docs-json", action="store_true", help="Do not update docs.json") + args = parser.parse_args() + + if not DOCS_SOURCE.exists(): + print(f"ERROR: DOCS_SOURCE not found: {DOCS_SOURCE}") + sys.exit(1) + if not TARGET_DOCS.exists() and not args.dry_run: + print(f"ERROR: TARGET_DOCS not found: {TARGET_DOCS}") + sys.exit(1) + + if args.node: + _canonical, _src = resolve_source_node(args.node) + nodes = [_canonical] if (_src / "en.md").exists() else [] + else: + nodes = list_nodes_with_en_md() + if args.mode == "test": + nodes = nodes[: args.count] + + update_docs_json = not args.no_docs_json and not args.dry_run + print(f"Syncing {len(nodes)} nodes to {TARGET_DOCS} (dry_run={args.dry_run}, update_docs_json={update_docs_json})") + if update_docs_json: + print(f" docs.json path: {DOCS_JSON}") + if not DOCS_JSON.exists(): + print(f" WARNING: docs.json not found at above path; navigation will not be updated.") + synced: List[Tuple[str, Optional[str], List[str]]] = [] + for node_name in nodes: + ok, category, synced_locales = sync_node(node_name, args.dry_run) + if ok: + synced.append((node_name, category, synced_locales)) + + if synced and not args.dry_run and not args.no_docs_json: + if not DOCS_JSON.exists(): + print("docs.json: skipped (file not found).") + else: + with open(DOCS_JSON, "r", encoding="utf-8") as f: + nav = json.load(f) + added: Dict[str, List[str]] = {cfg["code"]: [] for cfg in LOCALE_CONFIGS} + for node_name, full_category, synced_locales in synced: + scanner_name = canonical_node_name(node_name) + for locale in LOCALE_CONFIGS: + if locale["code"] not in synced_locales: + continue + published = published_node_name(scanner_name, locale["code"]) + page_key = f"{locale['page_prefix']}/{published}" + lang_entry = _find_lang_entry(nav, locale["code"]) + if lang_entry is None: + continue + tab_pages = _find_tab_pages(lang_entry, locale["tab"]) + if tab_pages is None: + continue + for alias in node_name_nav_aliases(scanner_name): + remove_page_from_pages(tab_pages, f"{locale['page_prefix']}/{alias}") + _place_page_in_nav(tab_pages, page_key, full_category or "", locale) + added[locale["code"]].append(page_key) + + for locale in LOCALE_CONFIGS: + lang_entry = _find_lang_entry(nav, locale["code"]) + if lang_entry is None: + continue + tab_pages = _find_tab_pages(lang_entry, locale["tab"]) + if tab_pages is not None: + _purge_noncanonical_nav_pages(tab_pages) + _migrate_toplevel_groups_to_wrapper(tab_pages, locale["wrapper"]) + + node_cat_map: Dict[str, str] = { + name: info.get("category", "") + for name, info in _load_all_nodes_info().items() + } + for locale in LOCALE_CONFIGS: + lang_entry = _find_lang_entry(nav, locale["code"]) + if lang_entry is None: + continue + tab_pages = _find_tab_pages(lang_entry, locale["tab"]) + if tab_pages is None: + continue + wrapper = find_group_in_pages(tab_pages, locale["wrapper"]) + if wrapper is not None: + _rebuild_wrapper_groups(wrapper, node_cat_map, lang_idx=locale["lang_idx"]) + + for locale in LOCALE_CONFIGS: + lang_entry = _find_lang_entry(nav, locale["code"]) + if lang_entry is None: + continue + tab_pages = _find_tab_pages(lang_entry, locale["tab"]) + if tab_pages is None: + continue + _purge_noncanonical_nav_pages(tab_pages) + _remove_empty_groups(tab_pages) + _sort_pages_alphabetically(tab_pages, groups_first=False) + + with open(DOCS_JSON, "w", encoding="utf-8") as f: + json.dump(nav, f, indent=2, ensure_ascii=False) + any_added = any(added[code] for code in added) + if any_added: + print(f"docs.json: updated {DOCS_JSON}") + for locale in LOCALE_CONFIGS: + for key in added[locale["code"]]: + print(f" + {locale['code'].upper()}: {key}") + else: + print("docs.json: no new entries (all synced nodes already in nav).") + + print("Done.") + + +if __name__ == "__main__": + main() From 6a24c3da4ef8a36d66607b1327f40dc1e900cfcd Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:07:48 +0800 Subject: [PATCH 02/10] fix: get_description_from_content skips H1 title instead of stopping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overview first-sentence extraction treated the page title (# H1) as the end of the overview, returning empty for every node whose en.md starts with a title line — so build_frontmatter fell back to the templated description. H1 is now skipped (continue) and only H2+ sections end the overview. Verified: Canny, GLSLShader, OpenAIDalle2, ClipTextEncodeSdxlRefiner now get real descriptions. --- scripts/sync_to_docs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/sync_to_docs.py b/scripts/sync_to_docs.py index 5ffe15f9d..18760f357 100644 --- a/scripts/sync_to_docs.py +++ b/scripts/sync_to_docs.py @@ -810,8 +810,12 @@ def get_description_from_content(content: str) -> str: if first_para: break continue - if line_stripped.startswith("##") or line_stripped.startswith("#"): + # H1 (# Title) is the page title — skip it and keep looking for the + # first content paragraph. H2+ (## Section) ends the overview. + if line_stripped.startswith("##"): break + if line_stripped.startswith("#"): + continue if line_stripped.startswith("> ") and ( "AI-generated" in line_stripped or "AI 生成" in line_stripped From 27804fc03e6786bbef8f9fd808ff84dea7843275 Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:10:41 +0800 Subject: [PATCH 03/10] feat: add complete documentation pipeline (scan/generate/translate/sync) Move the full node-docs automation pipeline into this repo under pipeline/, so the code that maintains these docs lives with the content it maintains: - scan_missing_nodes.py: scan ComfyUI source, detect new/changed nodes - prepare_ai_input.py + batch_generate_docs.py: LLM doc generation - batch_translate_docs.py: 11-language translation - update_param_translations.py + sync_frontend_translations.py: reconcile parameter names with the ComfyUI frontend i18n - sync_to_comfy_docs.py: generate built-in-nodes/*.mdx + docs.json nav in Comfy-Org/docs (concrete SEO descriptions, per-locale case-safe slug resolution, MDX-safe normalization) - version_tracker.py: per-node source hash tracking - lib/ + config/: shared modules and generation/translation rules - tests/: unit tests Path config via env (see env.example); no local/user-specific paths. data/ and ai_input/ are gitignored generated outputs. --- pipeline/.gitignore | 26 + pipeline/README.md | 107 ++ pipeline/config/doc_rules.txt | 153 ++ pipeline/config/translation_config.json | 90 ++ pipeline/config/translation_rules.txt | 122 ++ pipeline/env.example | 28 + pipeline/lib/__init__.py | 1 + pipeline/lib/doc_disclaimer.py | 180 +++ pipeline/lib/doc_title.py | 227 +++ pipeline/lib/hash_footer.py | 88 ++ pipeline/lib/node_source_extract.py | 811 +++++++++++ pipeline/lib/paths.py | 53 + pipeline/main.py | 1287 +++++++++++++++++ pipeline/requirements.txt | 3 + pipeline/scripts/batch_generate_docs.py | 442 ++++++ pipeline/scripts/batch_translate_docs.py | 499 +++++++ pipeline/scripts/check_config.py | 133 ++ pipeline/scripts/check_md_links.py | 212 +++ pipeline/scripts/check_outputs.py | 20 + pipeline/scripts/cleanup_duplicate_hashes.py | 89 ++ pipeline/scripts/fix_doc_titles.py | 206 +++ pipeline/scripts/fix_translations.py | 77 + pipeline/scripts/generate_docs.py | 570 ++++++++ pipeline/scripts/migrate_docs_format.py | 313 ++++ pipeline/scripts/prepare_ai_input.py | 474 ++++++ pipeline/scripts/prepare_translation.py | 220 +++ pipeline/scripts/replace_placeholders.py | 135 ++ pipeline/scripts/runtime.py | 8 + pipeline/scripts/scan_missing_nodes.py | 485 +++++++ .../scripts/sync_frontend_translations.py | 144 ++ .../scripts/sync_to_comfy_docs.py | 19 +- pipeline/scripts/update_param_translations.py | 250 ++++ pipeline/scripts/update_translation_status.py | 89 ++ pipeline/scripts/version_tracker.py | 228 +++ pipeline/tests/test_doc_title.py | 101 ++ pipeline/tests/test_sync_to_comfy_docs.sh | 32 + scripts/README.md | 65 - 37 files changed, 7913 insertions(+), 74 deletions(-) create mode 100644 pipeline/.gitignore create mode 100644 pipeline/README.md create mode 100644 pipeline/config/doc_rules.txt create mode 100644 pipeline/config/translation_config.json create mode 100644 pipeline/config/translation_rules.txt create mode 100644 pipeline/env.example create mode 100644 pipeline/lib/__init__.py create mode 100644 pipeline/lib/doc_disclaimer.py create mode 100644 pipeline/lib/doc_title.py create mode 100644 pipeline/lib/hash_footer.py create mode 100644 pipeline/lib/node_source_extract.py create mode 100644 pipeline/lib/paths.py create mode 100644 pipeline/main.py create mode 100644 pipeline/requirements.txt create mode 100644 pipeline/scripts/batch_generate_docs.py create mode 100644 pipeline/scripts/batch_translate_docs.py create mode 100644 pipeline/scripts/check_config.py create mode 100644 pipeline/scripts/check_md_links.py create mode 100644 pipeline/scripts/check_outputs.py create mode 100644 pipeline/scripts/cleanup_duplicate_hashes.py create mode 100644 pipeline/scripts/fix_doc_titles.py create mode 100644 pipeline/scripts/fix_translations.py create mode 100644 pipeline/scripts/generate_docs.py create mode 100644 pipeline/scripts/migrate_docs_format.py create mode 100644 pipeline/scripts/prepare_ai_input.py create mode 100644 pipeline/scripts/prepare_translation.py create mode 100644 pipeline/scripts/replace_placeholders.py create mode 100644 pipeline/scripts/runtime.py create mode 100644 pipeline/scripts/scan_missing_nodes.py create mode 100644 pipeline/scripts/sync_frontend_translations.py rename scripts/sync_to_docs.py => pipeline/scripts/sync_to_comfy_docs.py (98%) create mode 100644 pipeline/scripts/update_param_translations.py create mode 100644 pipeline/scripts/update_translation_status.py create mode 100644 pipeline/scripts/version_tracker.py create mode 100644 pipeline/tests/test_doc_title.py create mode 100644 pipeline/tests/test_sync_to_comfy_docs.sh delete mode 100644 scripts/README.md diff --git a/pipeline/.gitignore b/pipeline/.gitignore new file mode 100644 index 000000000..4335925da --- /dev/null +++ b/pipeline/.gitignore @@ -0,0 +1,26 @@ +# Environment +.env +*.env.local + +# Python +.venv/ +venv/ +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +dist/ + +# OS / editor +.DS_Store +.ref/ +.cursorrules +.gemini/ +.claude/ + +# Generated / local workflow outputs (never commit) +logs/ +ai_input/ +translation_batches/ +data/ +*.backup diff --git a/pipeline/README.md b/pipeline/README.md new file mode 100644 index 000000000..77301a034 --- /dev/null +++ b/pipeline/README.md @@ -0,0 +1,107 @@ +# ComfyUI Embedded Docs — Documentation Pipeline + +This directory contains the automation pipeline that keeps +[Comfy-Org/embedded-docs](https://github.com/Comfy-Org/embedded-docs) and the +`built-in-nodes/*` pages on [docs.comfy.org](https://docs.comfy.org) in sync with +the ComfyUI source code. + +The pipeline scans the ComfyUI codebase for new/changed nodes, generates and +translates node documentation into 11 languages, and publishes it. + +## Layout + +``` +pipeline/ +├── main.py # CLI entry point (full workflows) +├── scripts/ +│ ├── scan_missing_nodes.py # Scan ComfyUI source: find new/changed nodes +│ ├── prepare_ai_input.py # Build AI input bundles (source + meta + prompt) +│ ├── batch_generate_docs.py # Generate en.md via LLM +│ ├── batch_translate_docs.py # Translate en.md → 11 languages via LLM +│ ├── update_param_translations.py # Sync parameter names from frontend i18n +│ ├── sync_to_comfy_docs.py # embedded-docs → Comfy-Org/docs (.mdx + docs.json) +│ ├── sync_frontend_translations.py # Export frontend param translations +│ ├── version_tracker.py # Node version hash tracking +│ └── ... # maintenance / fixup helpers +├── lib/ # shared modules (paths, extract, titles, hashes) +├── config/ # generation rules + translation prompts +├── data/ # scan results, version DB (gitignored, generated) +└── ai_input/ # AI input bundles (gitignored, generated) +``` + +## Setup + +```bash +cd pipeline +cp env.example .env +# edit .env: COMFYUI_PATH, DEEPSEEK_API_KEY, etc. +pip install -r requirements.txt +``` + +## Weekly workflow + +```bash +# 1. Pull latest ComfyUI source first (required!) +cd /path/to/ComfyUI && git fetch origin master && git rebase origin/master + +# 2. Scan + regenerate changed node docs (never skip, even if scan says "no changes") +cd /path/to/embedded-docs/pipeline +python3 main.py --mode changed + +# 3. Generate docs for new nodes +python3 main.py --mode all --force + +# 4. Translate all languages (11 locales) + update param translations +python3 main.py --translate --all-languages --mode all + +# 5. Sync to Comfy-Org/docs (built-in-nodes .mdx + docs.json nav) +TARGET_DOCS=/path/to/comfy/docs python3 scripts/sync_to_comfy_docs.py --mode all + +# 6. Commit in embedded-docs, open PR; commit in docs, open PR +``` + +## Scripts overview + +| Script | Purpose | +|--------|---------| +| `scan_missing_nodes.py` | Scan ComfyUI source; report new nodes, changed nodes (source hash), possibly deprecated docs. Outputs to `data/`. | +| `prepare_ai_input.py` | Build AI input bundles (source code + metadata + prompt) for new/changed nodes. | +| `batch_generate_docs.py` | Generate `en.md` from AI input bundles via the configured LLM (OpenAI-compatible API). | +| `batch_translate_docs.py` | Translate `en.md` into 11 languages (zh, zh-TW, es, fr, ja, ko, ru, ar, tr, pt-BR, fa). | +| `update_param_translations.py` | Reconcile parameter/output name translations against the ComfyUI frontend i18n. | +| `sync_frontend_translations.py` | Export the frontend's parameter translations for use by the above. | +| `sync_to_comfy_docs.py` | Generate `built-in-nodes/*.mdx` + update `docs.json` navigation in a Comfy-Org/docs checkout. | +| `version_tracker.py` | Track per-node source SHA-256 hashes; detects changed nodes. | + +## Sync details (`sync_to_comfy_docs.py`) + +- Generates per-locale `.mdx` (`built-in-nodes/X.mdx`, `zh/...`, `ja/...`, `ko/...`) +- Frontmatter `description` is a **concrete summary extracted from the node's + `en.md` overview first sentence**, not a templated string +- Node slugs in `docs.json` are resolved per-locale against real on-disk file + names (case-sensitive) — prevents the case-mismatch 404s that occurred when + macOS's case-insensitive filesystem hid slug/filename differences +- MDX-safe normalization: code blocks preserved verbatim, whitelisted HTML and + paired Mintlify components (``/``/...) kept raw, unknown tags and + orphaned closing tags escaped + +## Env vars + +See `env.example`. Key ones: + +| Var | Required | Purpose | +|-----|----------|---------| +| `COMFYUI_PATH` | yes (scan/generate) | ComfyUI source checkout | +| `DEEPSEEK_API_KEY` | yes (LLM steps) | OpenAI-compatible API key | +| `API_BASE_URL` / `API_MODEL` | no | Defaults: DeepSeek | +| `EMBEDDED_DOCS_PATH` | no | embedded-docs repo root (defaults to repo root) | +| `TARGET_DOCS` | sync step | Comfy-Org/docs checkout | +| `COMFYUI_FRONTEND_PATH` | param-translation step | ComfyUI frontend repo | + +## Notes + +- **Never commit `.env`**, `data/`, or `ai_input/` (gitignored). +- The AI content pipeline uses an OpenAI-compatible chat API; configure provider + via `API_BASE_URL` / `API_MODEL` / key. +- Replacement nodes (aliases in `nodes_replacements.py`) have no standalone + class; the scanner reports them but they need no docs. diff --git a/pipeline/config/doc_rules.txt b/pipeline/config/doc_rules.txt new file mode 100644 index 000000000..d1b8abd6e --- /dev/null +++ b/pipeline/config/doc_rules.txt @@ -0,0 +1,153 @@ +# ComfyUI Node Documentation Rules + +## Document Structure + +Required sections in order: + +0. **Title** (level-one heading `# Display Name`) + - **NOT generated by AI** — the build pipeline prepends this from frontend `nodeDefs.json` `display_name`. + - Fallback order: target locale → English `display_name` → ComfyUI class name. + - Do **not** include a level-one heading in AI output; start with the overview paragraph. + +1. **Overview** (first paragraph, no heading) + - 1-3 sentences concisely explaining what the node does and how it works + - Use simple, easy-to-understand everyday vocabulary + - Avoid technical jargon + - Extract from class docstring if available + - Focus on what the node does, not lengthy explanations + +2. **Inputs** (## Inputs) + - The bundle may contain: (a) `# --- preceding context (same-file, AST-filtered unless PREAMBLE_MODE=full) ---` with only imports/helpers/constants referenced by this node class; (b) optional `# --- cross-file context` snippets with resolved callee source from `COMFYUI_PATH`; (c) `# --- node class ---` with the node's class definition. Use preamble and resolved snippets only to clarify behavior (parameters, loaders, checkpoints), not unrelated modules. + + - Complete parameter table with format: + | Parameter | Description | Data Type | Required | Range | + |-----------|-------------|-----------|----------|-------| + + - Parameter names use backticks: `parameter_name` + - Data types in PLAIN TEXT (no backticks): IMAGE, STRING, INT, FLOAT, MODEL, etc. + - Required column: "Yes" for required parameters, "No" for optional parameters + - Extract all information from source code including tooltips + - Description should be factual and based on tooltips/source code + - If a parameter has a default value, mention it in the Description + + **Special Formatting for COMBO Parameters:** + - For parameters with multiple options (COMBO type): + * List ALL available options in the Range column + * Use `
` tags to separate each option for better readability + * In Description column, explain what the parameter does and mention default if applicable + * Example Range: `"UPPERCASE"
"lowercase"
"Capitalize"
"Title Case"` + * Example Description: The case conversion mode to apply (default: "UPPERCASE") + + - If there are many options (>5), you can either: + * List all with `
` if they're important + * Or mention "Multiple options available" and explain in Description + + - Extract option values from the source code: + * Look for `options=[...]` in INPUT_TYPES + * Look for `options=[...]` in comfy_io.Combo.Input + * Include all options exactly as they appear in code + + **IMPORTANT - Parameter Constraints:** + - Analyze the source code for parameter constraints and limitations + - If multiple images/inputs are allowed, specify the maximum count + - If parameters have dependencies (e.g., param A requires param B), mention this clearly + - If certain parameter combinations are required together, explain this + - If there are mutual exclusions (e.g., can't use A and B together), note this + - Look for validation logic in the code (e.g., "if image and mask", "max=8", etc.) + - Include these constraints in the parameter descriptions or add a note after the table + +3. **Outputs** (## Outputs) + - Output table with format: + | Output Name | Description | Data Type | + |-------------|-------------|-----------| + + - Output names use backticks: `OUTPUT_NAME` + - Data types in PLAIN TEXT + - Description should explain what data is returned + +## Document Footer (added by automation — do NOT include in AI output) + +The build pipeline appends an AI disclaimer blockquote at the **bottom** of each file (after Overview / Inputs / Outputs). Do **not** include this disclaimer in generated markdown — it is injected automatically with the correct GitHub link. + +## Data Type Rules + +**CRITICAL: Do NOT translate data types** + +Always keep these in English: +- IMAGE +- FLOAT +- INT +- STRING +- MODEL +- CONDITIONING +- LATENT +- MASK +- CLIP +- VAE +- CONTROL_NET +- etc. + +## Language Style + +- **Simple and easy to understand**: Assume reader is non-technical +- **Clear and concise**: Avoid long complex sentences +- **Accurate and consistent**: Use terminology consistently +- **Clear structure**: Logical hierarchy, easy to reference + +## What to Avoid + +❌ Do NOT use emojis +❌ Do NOT include source code snippets in documentation +❌ Do NOT use overly technical terminology +❌ Do NOT translate data type names +❌ Do NOT include a level-one heading (# title) in AI output — it is injected automatically from frontend display names +❌ Do NOT add speculative usage tips or suggestions beyond what's in the source code +❌ Do NOT make assumptions about how users should use the node + +## What to Include + +✅ Extract information from class docstrings +✅ Use tooltip text exactly as provided for parameter descriptions +✅ Include default values and ranges from source code +✅ Stick to factual descriptions based on the source code +✅ Use simple analogies ONLY when they directly clarify the node's function + +## Special Notes + +- If the node has a docstring, use it as foundation for the function description +- Extract ALL parameter information including tooltips, defaults, min/max values +- For COMBO type parameters, include the list of available options in the description +- Keep descriptions factual - base them on tooltips and source code comments +- Avoid speculation about best practices or usage scenarios +- Match the professional yet accessible tone of existing documentation +- Focus on objective functionality rather than subjective recommendations + +## Critical: Parameter Constraints and Limitations + +**Analyze the source code carefully for:** + +1. **Numeric Limits**: + - Maximum/minimum values (e.g., `max=8` means "maximum 8 images") + - Batch size limitations + - Array/list length restrictions + +2. **Parameter Dependencies**: + - Required combinations (e.g., "image and mask must both be provided") + - Conditional requirements (e.g., "if mode=X, then param Y is required") + - Mutual exclusions (e.g., "cannot use both A and B") + +3. **Validation Logic**: + - Look for `if` statements that validate parameters + - Check for exceptions/errors raised (e.g., "raises Exception if X and Y...") + - Note any size/dimension matching requirements + +4. **Mode Switching**: + - If different parameter combinations trigger different behaviors + - Example: "When image and mask are provided, switches to editing mode" + +**How to Document:** +- Add constraints directly in the parameter's Description column +- If multiple parameters have related constraints, add a note section after the table +- Use clear, factual language: "Required when...", "Must match...", "Maximum of..." +- Base everything on actual code logic, not assumptions + diff --git a/pipeline/config/translation_config.json b/pipeline/config/translation_config.json new file mode 100644 index 000000000..72c948e49 --- /dev/null +++ b/pipeline/config/translation_config.json @@ -0,0 +1,90 @@ +{ + "zh": { + "name": "简体中文", + "heading_overview": "概述", + "heading_inputs": "输入", + "heading_outputs": "输出", + "disclaimer": "本文档由 AI 生成。如果您发现任何错误或有改进建议，欢迎贡献！", + "prompt_template": "你是一位专业的技术文档翻译专家，专门负责将 ComfyUI 节点文档从英文翻译成简体中文。\n\n## 翻译规则\n\n1. **必须保持不变的内容：**\n - 参数名必须保持英文，用反引号包裹：`image`、`seed`、`model`\n - 数据类型必须保持英文大写：IMAGE、STRING、INT、FLOAT、MODEL、CONDITIONING 等\n - Range 列中的值保持不变：数字、\"auto\"、选项名称\n - 不要翻译任何代码、文件路径\n\n2. **需要翻译的内容：**\n - 章节标题翻译为：{heading_overview}、{heading_inputs}、{heading_outputs}\n - 所有描述性文字和说明\n - 参数描述\n\n3. **翻译质量要求：**\n - 使用自然流畅的简体中文\n - 保持专业但易懂的语气\n - 确保技术准确性\n - 使用中文技术文档的标准术语\n\n4. **格式要求：**\n - 保持所有 Markdown 格式不变\n - 保持表格结构完整\n - 不要添加免责声明（系统将自动添加到文档底部）\n\n## 翻译示例\n\n**英文：**\nThis node combines two CLIP models by adding the second model to the first.\n\n**中文：**\n此节点通过将第二个 CLIP 模型添加到第一个模型来组合两个 CLIP 模型。\n\n请将以下英文文档翻译成简体中文，不要包含免责声明：\n\n5. **关键规则 - 输出名称：** 输出表（Outputs）的第一列是输出名（如 `positive`、`negative`、`latent`、`image`、`model`、`conditioning`），**必须保持英文不变**。翻译输出名会导致输出表中出现重复名称，破坏文档结构。" + }, + "es": { + "name": "Español", + "heading_overview": "Descripción general", + "heading_inputs": "Entradas", + "heading_outputs": "Salidas", + "disclaimer": "Esta documentación fue generada por IA. Si encuentra algún error o tiene sugerencias de mejora, ¡no dude en contribuir!", + "prompt_template": "Eres un experto en traducción técnica especializado en documentación de nodos ComfyUI del inglés al español.\n\n## Reglas de Traducción\n\n1. **Contenido que NO debe traducirse:**\n - Nombres de parámetros entre comillas invertidas: `image`, `seed`, `model`\n - Tipos de datos en MAYÚSCULAS: IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING, etc.\n - Valores en columna Range: números, \"auto\", nombres de opciones\n - Código, rutas de archivos\n\n2. **Contenido que SÍ debe traducirse:**\n - Títulos de secciones: {heading_overview}, {heading_inputs}, {heading_outputs}\n - Todo el texto descriptivo y explicativo\n - Descripciones de parámetros\n\n3. **Calidad de traducción:**\n - Usar español estándar y neutral\n - Mantener tono profesional pero accesible\n - Asegurar precisión técnica\n - Usar terminología técnica estándar en español\n\n4. **Formato:**\n - Mantener todo el formato Markdown\n - Preservar estructura de tablas\n - No agregar ninguna nota o enlace al inicio del documento (será agregado automáticamente)\n\nPor favor traduce la siguiente documentación al español, sin incluir la aviso de IA:\n\n5. **CRÍTICO - Nombres de salida:** La primera columna de la tabla de Salidas contiene nombres de salida (ej. `positive`, `negative`, `latent`, `image`, `model`, `conditioning`) que DEBEN permanecer en inglés. Traducir estos nombres crea entradas duplicadas y rompe la estructura del documento." + }, + "fr": { + "name": "Français", + "heading_overview": "Aperçu général", + "heading_inputs": "Entrées", + "heading_outputs": "Sorties", + "disclaimer": "Cette documentation a été générée par IA. Si vous trouvez des erreurs ou avez des suggestions d'amélioration, n'hésitez pas à contribuer !", + "prompt_template": "Vous êtes un expert en traduction technique spécialisé dans la documentation des nœuds ComfyUI de l'anglais vers le français.\n\n## Règles de Traduction\n\n1. **Contenu à NE PAS traduire:**\n - Noms de paramètres entre backticks: `image`, `seed`, `model`\n - Types de données en MAJUSCULES: IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING, etc.\n - Valeurs dans la colonne Range: nombres, \"auto\", noms d'options\n - Code, chemins de fichiers\n\n2. **Contenu à traduire:**\n - Titres de sections: {heading_overview}, {heading_inputs}, {heading_outputs}\n - Tout le texte descriptif et explicatif\n - Descriptions des paramètres\n\n3. **Qualité de traduction:**\n - Utiliser le français standard\n - Maintenir un ton professionnel mais accessible\n - Assurer la précision technique\n - Utiliser la terminologie technique standard en français\n\n4. **Format:**\n - Conserver tout le formatage Markdown\n - Préserver la structure des tableaux\n - Ne pas ajouter de note ou lien au début du document (sera ajouté automatiquement)\n\nVeuillez traduire la documentation suivante en français, sans inclure la avertissement IA:\n\n5. **CRITIQUE - Noms de sortie:** La première colonne du tableau des Sorties contient des noms de sortie (ex: `positive`, `negative`, `latent`, `image`, `model`, `conditioning`) qui DOIVENT rester en anglais. Traduire ces noms crée des doublons et casse la structure du document." + }, + "ja": { + "name": "日本語", + "heading_overview": "概要", + "heading_inputs": "入力", + "heading_outputs": "出力", + "disclaimer": "このドキュメントは AI によって生成されました。エラーを見つけた場合や改善のご提案がある場合は、ぜひ貢献してください！", + "prompt_template": "あなたは ComfyUI ノードドキュメントを英語から日本語に翻訳する技術翻訳の専門家です。\n\n## 翻訳ルール\n\n1. **翻訳してはいけない内容：**\n - バッククォートで囲まれたパラメータ名：`image`、`seed`、`model`\n - 大文字のデータ型：IMAGE、STRING、INT、FLOAT、MODEL、CONDITIONING など\n - Range列の値：数値、\"auto\"、オプション名\n - コード、ファイルパス\n\n2. **翻訳する内容：**\n - セクション見出し：{heading_overview}、{heading_inputs}、{heading_outputs}\n - すべての説明文\n - パラメータの説明\n\n3. **翻訳品質：**\n - 丁寧語（です・ます体）を使用\n - 専門的でありながらわかりやすい表現\n - 技術的な正確性を保つ\n - 標準的な日本語技術用語を使用\n\n4. **フォーマット：**\n - すべての Markdown フォーマットを保持\n - 表の構造を維持\n - 免責事項は追加しないでください（システムが文書末尾に自動追加します）\n\n以下の英語ドキュメントを日本語に翻訳してください（免責事項は含めないでください）：\n\n5. **重要ルール - 出力名：** 出力テーブルの最初の列には出力名（例：`positive`、`negative`、`latent`、`image`、`model`、`conditioning`）が含まれています。これらは**英語のままにしてください**。出力名を翻訳すると重複エントリが発生し、文書構造が壊れます。" + }, + "ko": { + "name": "한국어", + "heading_overview": "개요", + "heading_inputs": "입력", + "heading_outputs": "출력", + "disclaimer": "이 문서는 AI에 의해 생성되었습니다. 오류를 발견하거나 개선 제안이 있으시면 기여해 주세요!", + "prompt_template": "당신은 ComfyUI 노드 문서를 영어에서 한국어로 번역하는 기술 번역 전문가입니다.\n\n## 번역 규칙\n\n1. **번역하지 말아야 할 내용:**\n - 백틱으로 둘러싸인 매개변수 이름: `image`, `seed`, `model`\n - 대문자 데이터 타입: IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING 등\n - Range 열의 값: 숫자, \"auto\", 옵션 이름\n - 코드, 파일 경로\n\n2. **번역해야 할 내용:**\n - 섹션 제목: {heading_overview}, {heading_inputs}, {heading_outputs}\n - 모든 설명 텍스트\n - 매개변수 설명\n\n3. **번역 품질:**\n - 정중한 격식체(합니다체) 사용\n - 전문적이면서도 이해하기 쉬운 표현\n - 기술적 정확성 유지\n - 표준 한국어 기술 용어 사용\n\n4. **형식:**\n - 모든 Markdown 형식 유지\n - 표 구조 보존\n - 문서 시작 부분에 메모나 링크를 추가하지 마세요 (자동으로 추가됩니다)\n\n다음 영어 문서를 한국어로 번역해주세요 (문서 시작 부분의 메모는 포함하지 마세요):\n\n5. **중요 규칙 - 출력 이름:** 출력 테이블의 첫 번째 열에는 출력 이름(예: `positive`, `negative`, `latent`, `image`, `model`, `conditioning`)이 포함되어 있으며, **영어로 유지해야 합니다**. 출력 이름을 번역하면 중복 항목이 생성되고 문서 구조가 손상됩니다." + }, + "ru": { + "name": "Русский", + "heading_overview": "Обзор", + "heading_inputs": "Входы", + "heading_outputs": "Выходы", + "disclaimer": "Эта документация была создана с помощью ИИ. Если вы обнаружите ошибки или у вас есть предложения по улучшению, пожалуйста, внесите свой вклад!", + "prompt_template": "Вы эксперт по техническому переводу документации узлов ComfyUI с английского на русский язык.\n\n## Правила Перевода\n\n1. **Что НЕ переводить:**\n - Имена параметров в обратных кавычках: `image`, `seed`, `model`\n - Типы данных ЗАГЛАВНЫМИ буквами: IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING и т.д.\n - Значения в колонке Range: числа, \"auto\", названия опций\n - Код, пути к файлам\n\n2. **Что переводить:**\n - Заголовки разделов: {heading_overview}, {heading_inputs}, {heading_outputs}\n - Весь описательный текст\n - Описания параметров\n\n3. **Качество перевода:**\n - Использовать стандартный русский язык\n - Поддерживать профессиональный, но доступный тон\n - Обеспечить техническую точность\n - Использовать стандартную техническую терминологию на русском\n\n4. **Формат:**\n - Сохранить всё форматирование Markdown\n - Сохранить структуру таблиц\n - Не добавляйте отказ от ответственности (он будет добавлен автоматически в конце документа)\n\nПожалуйста, переведите следующую документацию на русский язык (не включайте отказ от ответственности):\n\n5. **ВАЖНО - Имена выходов:** Первый столбец таблицы выходов содержит имена выходов (например, `positive`, `negative`, `latent`, `image`, `model`, `conditioning`), которые ДОЛЖНЫ оставаться на английском языке. Перевод этих имен создает дубликаты и нарушает структуру документа." + }, + "zh-TW": { + "name": "繁體中文", + "heading_overview": "概述", + "heading_inputs": "輸入", + "heading_outputs": "輸出", + "disclaimer": "本文檔由 AI 生成。如果您發現任何錯誤或有改進建議，歡迎貢獻！", + "prompt_template": "你是一位專業的技術文檔翻譯專家，專門負責將 ComfyUI 節點文檔從英文翻譯成繁體中文。\n\n## 翻譯規則\n\n1. **必須保持不變的內容：**\n - 參數名必須保持英文，用反引號包裹：`image`、`seed`、`model`\n - 資料類型必須保持英文大寫：IMAGE、STRING、INT、FLOAT、MODEL、CONDITIONING 等\n - Range 列中的值保持不變：數字、\"auto\"、選項名稱\n - 不要翻譯任何程式碼、檔案路徑\n\n2. **需要翻譯的內容：**\n - 章節標題翻譯為：{heading_overview}、{heading_inputs}、{heading_outputs}\n - 所有描述性文字和說明\n - 參數描述\n\n3. **翻譯質量要求：**\n - 使用自然流暢的繁體中文\n - 保持專業但易懂的語氣\n - 確保技術準確性\n - 使用繁體中文技術文檔的標準術語\n\n4. **格式要求：**\n - 保持所有 Markdown 格式不變\n - 保持表格結構完整\n - 不要添加免責聲明（系統將自動添加到文檔底部）\n\n## 翻譯示例\n\n**英文：**\nThis node combines two CLIP models by adding the second model to the first.\n\n**繁體中文：**\n此節點透過將第二個 CLIP 模型添加到第一個模型來組合兩個 CLIP 模型。\n\n請將以下英文文檔翻譯成繁體中文，不要包含免責聲明：\n\n5. **關鍵規則 - 輸出名稱：** 輸出表（Outputs）的第一列是輸出名（如 `positive`、`negative`、`latent`、`image`、`model`、`conditioning`），**必須保持英文不變**。翻譯輸出名會導致輸出表中出現重複名稱，破壞文檔結構。" + }, + "ar": { + "name": "العربية", + "heading_overview": "نظرة عامة", + "heading_inputs": "المدخلات", + "heading_outputs": "المخرجات", + "disclaimer": "تم إنشاء هذه الوثيقة بواسطة الذكاء الاصطناعي. إذا وجدت أي أخطاء أو لديك اقتراحات للتحسين، فلا تتردد في المساهمة!", + "prompt_template": "أنت خبير في الترجمة التقنية متخصص في توثيق عُقد ComfyUI من الإنجليزية إلى العربية.\n\n## قواعد الترجمة\n\n1. **المحتوى الذي يجب عدم ترجمته:**\n - أسماء المعاملات بين علامات الاقتباس الخلفية: `image`, `seed`, `model`\n - أنواع البيانات بالأحرف الكبيرة: IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING, إلخ\n - القيم في عمود Range: الأرقام، \"auto\"، أسماء الخيارات\n - الكود، مسارات الملفات\n\n2. **المحتوى الذي يجب ترجمته:**\n - عناوين الأقسام: {heading_overview}, {heading_inputs}, {heading_outputs}\n - جميع النصوص الوصفية والتوضيحية\n - أوصاف المعاملات\n\n3. **جودة الترجمة:**\n - استخدام اللغة العربية الفصحى المعاصرة\n - الحفاظ على نبرة احترافية ولكن سهلة الفهم\n - ضمان الدقة التقنية\n - استخدام المصطلحات التقنية العربية القياسية\n\n4. **التنسيق:**\n - الحفاظ على جميع تنسيقات Markdown\n - الحفاظ على بنية الجداول\n - عدم إضافة أي ملاحظة أو رابط في بداية الوثيقة (سيتم إضافتها تلقائيًا)\n\nالرجاء ترجمة الوثيقة التالية إلى العربية، دون تضمين الملاحظة الأولية للوثيقة:\n\n5. **قاعدة حاسمة - أسماء المخرجات:** العمود الأول من جدول المخرجات يحتوي على أسماء مخرجات (مثل `positive`، `negative`، `latent`، `image`، `model`، `conditioning`) التي يجب أن تبقى بالإنجليزية. ترجمة هذه الأسماء يؤدي إلى تكرار الإدخالات وكسر بنية المستند." + }, + "tr": { + "name": "Türkçe", + "heading_overview": "Genel Bakış", + "heading_inputs": "Girdiler", + "heading_outputs": "Çıktılar", + "disclaimer": "Bu belge yapay zeka tarafından oluşturulmuştur. Herhangi bir hata bulursanız veya iyileştirme önerileriniz varsa, katkıda bulunmaktan çekinmeyin!", + "prompt_template": "ComfyUI düğüm belgelerini İngilizceden Türkçeye çevirmede uzmanlaşmış teknik çeviri uzmanısınız.\n\n## Çeviri Kuralları\n\n1. **Çevrilmemesi gereken içerik:**\n - Ters tırnak içindeki parametre adları: `image`, `seed`, `model`\n - BÜYÜK harflerle veri türleri: IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING, vb.\n - Range sütunundaki değerler: sayılar, \"auto\", seçenek adları\n - Kod, dosya yolları\n\n2. **Çevrilmesi gereken içerik:**\n - Bölüm başlıkları: {heading_overview}, {heading_inputs}, {heading_outputs}\n - Tüm açıklayıcı metinler\n - Parametre açıklamaları\n\n3. **Çeviri kalitesi:**\n - Standart Türkçe kullanın\n - Profesyonel ama anlaşılır bir üslup koruyun\n - Teknik doğruluğu sağlayın\n - Standart Türkçe teknik terminolojiyi kullanın\n\n4. **Format:**\n - Tüm Markdown biçimlendirmesini koruyun\n - Tablo yapısını koruyun\n - Belgenin başına herhangi bir not veya bağlantı eklemeyin (otomatik olarak eklenecektir)\n\nLütfen aşağıdaki belgeyi Türkçeye çevirin (belgenin başlangıç notunu dahil etmeyin):\n\n5. **KRİTİK KURAL - Çıktı Adları:** Çıktılar tablosunun ilk sütunu, İngilizce kalması ZORUNLU olan çıktı adlarını içerir (örn. `positive`, `negative`, `latent`, `image`, `model`, `conditioning`). Çıktı adlarını çevirmek yinelenen girdilere ve belge yapısının bozulmasına neden olur." + }, + "pt-BR": { + "name": "Português (BR)", + "heading_overview": "Visão Geral", + "heading_inputs": "Entradas", + "heading_outputs": "Saídas", + "disclaimer": "Esta documentação foi gerada por IA. Se você encontrar erros ou tiver sugestões de melhoria, sinta-se à vontade para contribuir!", + "prompt_template": "Você é um especialista em tradução técnica especializado em documentação de nós ComfyUI do inglês para português brasileiro.\n\n## Regras de Tradução\n\n1. **Conteúdo que NÃO deve ser traduzido:**\n - Nomes de parâmetros entre crases: `image`, `seed`, `model`\n - Tipos de dados em MAIÚSCULAS: IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING, etc.\n - Valores na coluna Range: números, \"auto\", nomes de opções\n - Código, caminhos de arquivos\n\n2. **Conteúdo que DEVE ser traduzido:**\n - Títulos de seções: {heading_overview}, {heading_inputs}, {heading_outputs}\n - Todo o texto descritivo e explicativo\n - Descrições de parâmetros\n\n3. **Qualidade da tradução:**\n - Use português brasileiro padrão\n - Mantenha um tom profissional mas acessível\n - Garanta precisão técnica\n - Use terminologia técnica padrão em português brasileiro\n\n4. **Formato:**\n - Mantenha toda a formatação Markdown\n - Preserve a estrutura das tabelas\n - Não adicione nenhuma nota ou link no início do documento (será adicionado automaticamente)\n\nPor favor, traduza a seguinte documentação para português brasileiro, sem incluir a nota inicial do documento:\n\n5. **CRÍTICO - Nomes de saída:** A primeira coluna da tabela de Saídas contém nomes de saída (ex: `positive`, `negative`, `latent`, `image`, `model`, `conditioning`) que DEVEM permanecer em inglês. Traduzir esses nomes cria entradas duplicadas e quebra a estrutura do documento." + }, + "fa": { + "name": "فارسی", + "heading_overview": "نمای کلی", + "heading_inputs": "ورودی‌ها", + "heading_outputs": "خروجی‌ها", + "disclaimer": "این مستند با هوش مصنوعی تهیه شده است. اگر خطایی دیدید یا پیشنهادی برای بهبود دارید، خوشحال می‌شویم مشارکت کنید!", + "prompt_template": "شما یک مترجم فنی متخصص هستید که اسناد گره‌های ComfyUI را از انگلیسی به فارسی برمی‌گردانید.\n\n## قوانین ترجمه\n\n1. **محتوایی که نباید ترجمه شود:**\n - نام پارامترها در بک‌تیک: `image`, `seed`, `model`\n - نوع‌های داده با حروف بزرگ انگلیسی: IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING و غیره\n - مقادیر ستون Range: اعداد، \"auto\", نام گزینه‌ها\n - کد، مسیر فایل\n\n2. **محتوایی که باید ترجمه شود:**\n - عناوین بخش‌ها: {heading_overview}, {heading_inputs}, {heading_outputs}\n - تمام متن توضیحی\n - توضیحات پارامترها\n\n3. **کیفیت ترجمه:**\n - فارسی رسمی معاصر و روان بنویسید\n - لحن حرفه‌ای ولی قابل‌فهم\n - دقت فنی را حفظ کنید\n\n4. **قالب:**\n - قالب Markdown و ساختار جدول دست‌نخورده بماند\n - توضیح یا لینک در ابتدای سند اضافه نکنید (به‌صورت خودکار افزوده می‌شود)\n\nلطفاً سند انگلیسی زیر را به فارسی برگردانید؛ یادداشت ابتدای سند را در خروجی نگنجانید:\n\n5. **قانون حیاتی - نام‌های خروجی:** ستون اول جدول خروجی‌ها شامل نام‌های خروجی است (مانند `positive`، `negative`، `latent`، `image`، `model`، `conditioning`) که باید به انگلیسی باقی بمانند. ترجمه این نام‌ها باعث ایجاد ورودی‌های تکراری و شکستن ساختار سند می‌شود." + } +} \ No newline at end of file diff --git a/pipeline/config/translation_rules.txt b/pipeline/config/translation_rules.txt new file mode 100644 index 000000000..41e8f17a1 --- /dev/null +++ b/pipeline/config/translation_rules.txt @@ -0,0 +1,122 @@ +# ComfyUI Node Documentation Translation Rules + +## Translation Principles + +1. **Accuracy First** + - Maintain technical accuracy + - Preserve all parameter names in backticks (`) + - Keep data types in ENGLISH (IMAGE, STRING, INT, FLOAT, etc.) + - Do not translate code, node names, or technical identifiers + +2. **Natural Language** + - Use natural, fluent expressions in target language + - Adapt metaphors and examples to local culture when appropriate + - Maintain professional but accessible tone + +3. **Consistency** + - Use consistent terminology throughout + - Follow language-specific conventions for technical documentation + - Maintain the same structure as the English version + +4. **Preserve Formatting** + - Keep all Markdown formatting intact + - Preserve table structures + - Maintain line breaks and spacing + - Keep backticks around parameter names + +## What to Translate + +✅ **DO translate:** +- Section headings (Overview, Inputs, Outputs) +- Parameter descriptions +- Explanatory text +- Usage notes and tips +- Constraint descriptions + +❌ **DO NOT translate:** +- Parameter names (keep in backticks: `image`, `seed`, `model`) +- Data types (IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING, etc.) +- Values in Range column (numbers, "auto", option names) +- Code snippets +- File paths +- URLs (except in the disclaimer footer added by automation) + +## Language-Specific Requirements + +### Chinese (zh) +- Use simplified Chinese characters +- Technical terms: use commonly accepted Chinese translations +- Headings: 概述, 输入, 输出 +- Tone: professional but accessible + +### Spanish (es) +- Use standard Spanish (neutral, not regional dialects) +- Headings: Descripción general, Entradas, Salidas +- Maintain formal "usted" form where appropriate + +### French (fr) +- Use standard French +- Headings: Aperçu général, Entrées, Sorties +- Maintain formal tone + +### Japanese (ja) +- Use polite form (です/ます体) +- Headings: 概要, 入力, 出力 +- Technical terms: use katakana for foreign technical terms when appropriate + +### Korean (ko) +- Use formal polite form (합니다체) +- Headings: 개요, 입력, 출력 +- Technical terms: use Hangul or keep English when standard + +### Russian (ru) +- Use standard Russian +- Headings: Обзор, Входы, Выходы +- Maintain formal tone + +## Table Structure + +Maintain the exact table structure: + +**Inputs:** +| Parameter | Description | Data Type | Required | Range | +|-----------|-------------|-----------|----------|-------| + +**Outputs:** +| Output Name | Description | Data Type | +|-------------|-------------|-----------| + +## Example Translation + +**English:** +> This node combines two CLIP models by adding the second model to the first. + +**Chinese:** +> 此节点通过将第二个 CLIP 模型添加到第一个模型来组合两个 CLIP 模型。 + +**Spanish:** +> Este nodo combina dos modelos CLIP añadiendo el segundo modelo al primero. + +**French:** +> Ce nœud combine deux modèles CLIP en ajoutant le second modèle au premier. + +**Japanese:** +> このノードは、2番目のCLIPモデルを1番目に追加することで、2つのCLIPモデルを結合します。 + +**Korean:** +> 이 노드는 두 번째 CLIP 모델을 첫 번째 모델에 추가하여 두 개의 CLIP 모델을 결합합니다。 + +**Russian:** +> Этот узел объединяет две модели CLIP, добавляя вторую модель к первой. + +## Quality Checklist + +Before finalizing translation, verify: +- [ ] All parameter names remain in English within backticks +- [ ] All data types remain in English (uppercase) +- [ ] Table structure is intact +- [ ] Markdown formatting is preserved +- [ ] Technical accuracy is maintained +- [ ] Language sounds natural to native speakers +- [ ] No English text remains except parameter names, data types, and values + diff --git a/pipeline/env.example b/pipeline/env.example new file mode 100644 index 000000000..8c0b78682 --- /dev/null +++ b/pipeline/env.example @@ -0,0 +1,28 @@ +# ============================================================================ +# comfyui-embedded-docs documentation pipeline — environment template +# Copy this file to `.env` and fill in your values. Never commit `.env`. +# ============================================================================ + +# --- Repository paths ------------------------------------------------------- +# Path to a ComfyUI source checkout (required for scan / prepare / changed steps) +COMFYUI_PATH=/path/to/ComfyUI + +# Path to the ComfyUI frontend repo (optional; used by sync_frontend_translations) +COMFYUI_FRONTEND_PATH=/path/to/ComfyUI_frontend + +# Path to the embedded-docs repo root (defaults to the repo this pipeline lives in) +EMBEDDED_DOCS_PATH= + +# Path to a Comfy-Org/docs checkout (used by sync_to_comfy_docs.py) +TARGET_DOCS=/path/to/comfy/docs + +# --- LLM API configuration -------------------------------------------------- +# The pipeline uses an OpenAI-compatible chat API for doc generation & translation. +DEEPSEEK_API_KEY=your_api_key_here +API_BASE_URL=https://api.deepseek.com +API_MODEL=deepseek-chat + +# --- Batch processing ------------------------------------------------------- +BATCH_SIZE=5 +MAX_RETRIES=3 +DELAY_BETWEEN_REQUESTS=2 diff --git a/pipeline/lib/__init__.py b/pipeline/lib/__init__.py new file mode 100644 index 000000000..f39ed0020 --- /dev/null +++ b/pipeline/lib/__init__.py @@ -0,0 +1 @@ +"""Shared libraries for doc_automation.""" diff --git a/pipeline/lib/doc_disclaimer.py b/pipeline/lib/doc_disclaimer.py new file mode 100644 index 000000000..27fd5ee3f --- /dev/null +++ b/pipeline/lib/doc_disclaimer.py @@ -0,0 +1,180 @@ +"""AI disclaimer helpers for generated and translated node documentation.""" + +from __future__ import annotations + +from lib.hash_footer import strip_source_hash_footer + +# Substrings that identify the AI disclaimer blockquote (any supported language). +_DISCLAIMER_MARKERS = ( + "AI-generated", + "AI generated", + "AI 生成", + "AI によって生成", + "AI에 의해 생성", + "generada por IA", + "générée par IA", + "gerada por IA", + "с помощью ИИ", + "الذكاء الاصطناعي", + "yapay zeka", + "هوش مصنوعی", + "Edit on GitHub", + "GitHub で編集", + "GitHub에서 편집", + "在 GitHub 上编辑", + "在 GitHub 上編輯", + "Editar en GitHub", + "Modifier sur GitHub", + "Редактировать на GitHub", +) + +_EDIT_LINK_HINTS = ( + "edit on github", + "github.com", + "github で", + "github에서", + "github 上", + "github'da", +) + + +def is_disclaimer_line(line: str) -> bool: + """True if this line is the AI disclaimer blockquote.""" + s = line.strip() + if not s.startswith(">"): + return False + low = s.lower() + if any(h in low for h in _EDIT_LINK_HINTS): + return True + return any(marker in s for marker in _DISCLAIMER_MARKERS) + + +def strip_ai_disclaimer(content: str) -> str: + """Remove AI disclaimer blockquote(s) from the top or bottom of a markdown body.""" + body = strip_source_hash_footer(content).strip() + lines = body.split("\n") + + i = 0 + while i < len(lines): + if not lines[i].strip(): + i += 1 + continue + if is_disclaimer_line(lines[i]): + i += 1 + while i < len(lines) and not lines[i].strip(): + i += 1 + continue + break + lines = lines[i:] + + while lines: + if not lines[-1].strip(): + lines.pop() + continue + if is_disclaimer_line(lines[-1]): + lines.pop() + while lines and not lines[-1].strip(): + lines.pop() + continue + break + + return "\n".join(lines).rstrip() + + +def create_en_disclaimer(node_name: str) -> str: + github_link = ( + f"https://github.com/Comfy-Org/embedded-docs/blob/main/" + f"comfyui_embedded_docs/docs/{node_name}/en.md" + ) + return ( + "> This documentation was AI-generated. If you find any errors or have suggestions " + f"for improvement, please feel free to contribute! [Edit on GitHub]({github_link})" + ) + + +def create_translated_disclaimer(target_lang: str, node_name: str, lang_config: dict) -> str: + github_link = ( + f"https://github.com/Comfy-Org/embedded-docs/blob/main/" + f"comfyui_embedded_docs/docs/{node_name}/{target_lang}.md" + ) + disclaimer_text = lang_config.get("disclaimer", "This documentation was AI-generated.") + edit_text = { + "zh": "在 GitHub 上编辑", + "es": "Editar en GitHub", + "fr": "Modifier sur GitHub", + "ja": "GitHub で編集", + "ko": "GitHub에서 편집", + "ru": "Редактировать на GitHub", + "zh-TW": "在 GitHub 上編輯", + "ar": "تحرير على GitHub", + "tr": "GitHub'da Düzenle", + "pt-BR": "Editar no GitHub", + "fa": "ویرایش در GitHub", + }.get(target_lang, "Edit on GitHub") + return f"> {disclaimer_text} [{edit_text}]({github_link})" + + +def compose_document(body: str, disclaimer: str, footer: str = "") -> str: + """Assemble markdown: main content, disclaimer at bottom, optional hash footer last.""" + clean = strip_ai_disclaimer(body) + out = f"{clean.rstrip()}\n\n{disclaimer.strip()}" + if footer: + out += footer if footer.startswith("\n") else f"\n{footer}" + if not out.endswith("\n"): + out += "\n" + return out + + +def extract_metadata_suffix(original: str) -> str: + """ + Return trailing disclaimer + SHA footer from *original*, unchanged. + + Used when fixing doc titles without altering hash or disclaimer formatting. + """ + from lib.hash_footer import SOURCE_HASH_FOOTER_RE + + text = original.rstrip() + footer = "" + footer_m = SOURCE_HASH_FOOTER_RE.search(text) + if footer_m: + footer = text[footer_m.start():] + text = text[:footer_m.start()].rstrip() + + lines = text.split("\n") + disc_start = len(lines) + i = len(lines) - 1 + while i >= 0: + if not lines[i].strip(): + i -= 1 + continue + if is_disclaimer_line(lines[i]): + disc_start = i + while disc_start > 0: + prev = lines[disc_start - 1] + if is_disclaimer_line(prev): + disc_start -= 1 + elif not prev.strip(): + disc_start -= 1 + else: + break + break + break + + parts: list[str] = [] + if disc_start < len(lines): + disc = "\n".join(lines[disc_start:]).strip() + if disc: + parts.append(disc) + if footer: + parts.append(footer.lstrip("\n")) + return "\n\n".join(parts) + + +def assemble_document_with_metadata_suffix(body: str, metadata_suffix: str) -> str: + """Join main body with an unchanged disclaimer + footer suffix.""" + out = body.rstrip() + if metadata_suffix.strip(): + out += "\n\n" + metadata_suffix.strip() + if not out.endswith("\n"): + out += "\n" + return out diff --git a/pipeline/lib/doc_title.py b/pipeline/lib/doc_title.py new file mode 100644 index 000000000..183dff8c8 --- /dev/null +++ b/pipeline/lib/doc_title.py @@ -0,0 +1,227 @@ +"""Inject node display-name headings from frontend translations (not AI).""" + +from __future__ import annotations + +import json +import re +from functools import lru_cache +from typing import Any, Dict, Literal, Optional + +HashMode = Literal["preserve", "update"] + +from lib.paths import NODE_TRANSLATIONS + +# Level-1 heading: single `#`, not `##` or deeper. +_H1_RE = re.compile(r"^#(?!#)") + + +@lru_cache(maxsize=1) +def load_node_translations() -> Dict[str, Any]: + """Load exported frontend nodeDefs translations (data/node_translations.json).""" + if not NODE_TRANSLATIONS.is_file(): + return {} + try: + with open(NODE_TRANSLATIONS, encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return {} + + +def _display_name_for_lang( + data: Dict[str, Any], + lang: str, + node_name: str, +) -> str: + lang_data = data.get(lang, {}) + if not isinstance(lang_data, dict): + return "" + node_data = lang_data.get(node_name) + if not isinstance(node_data, dict): + return "" + return str(node_data.get("display_name", "")).strip() + + +def get_node_display_name( + node_name: str, + lang: str = "en", + translations: Optional[Dict[str, Any]] = None, +) -> str: + """ + Return display name from frontend nodeDefs. + + Fallback order: + 1. Target language ``display_name`` + 2. English ``display_name`` (when target lang is missing) + 3. ComfyUI class name (``node_name``) + """ + data = translations if translations is not None else load_node_translations() + + if lang != "en": + localized = _display_name_for_lang(data, lang, node_name) + if localized: + return localized + + english = _display_name_for_lang(data, "en", node_name) + if english: + return english + + return node_name + + +def _is_h1_line(line: str) -> bool: + stripped = line.strip() + if not stripped: + return False + if not _H1_RE.match(stripped): + return False + return bool(re.match(r"^#(?!#)\s*\S", stripped)) + + +def extract_leading_h1_title(content: str) -> Optional[str]: + """Return the text of the first leading level-1 heading, or None.""" + lines = content.split("\n") + i = 0 + while i < len(lines) and not lines[i].strip(): + i += 1 + if i >= len(lines) or not _is_h1_line(lines[i]): + return None + m = re.match(r"^#(?!#)\s*(.+)$", lines[i].strip()) + return m.group(1).strip() if m else None + + +def count_leading_h1_lines(content: str) -> int: + """Count consecutive level-1 headings at the start of the document.""" + lines = content.split("\n") + i = 0 + count = 0 + while True: + while i < len(lines) and not lines[i].strip(): + i += 1 + if i < len(lines) and _is_h1_line(lines[i]): + count += 1 + i += 1 + continue + break + return count + + +def analyze_title_issues( + body: str, + node_name: str, + lang: str = "en", + translations: Optional[Dict[str, Any]] = None, +) -> list[str]: + """ + Detect title problems: missing, duplicate, or mismatch vs frontend display_name. + """ + issues: list[str] = [] + n = count_leading_h1_lines(body) + expected = get_node_display_name(node_name, lang, translations) + + if n == 0: + issues.append("missing") + if n > 1: + issues.append("duplicate") + + current = extract_leading_h1_title(body) + if n >= 1 and current != expected: + issues.append("mismatch") + + return issues + + +def title_needs_fix( + body: str, + node_name: str, + lang: str = "en", + translations: Optional[Dict[str, Any]] = None, +) -> bool: + return bool(analyze_title_issues(body, node_name, lang, translations)) + + +def strip_leading_h1(content: str) -> str: + """Remove all consecutive leading level-1 markdown headings.""" + lines = content.split("\n") + i = 0 + while i < len(lines): + while i < len(lines) and not lines[i].strip(): + i += 1 + if i < len(lines) and _is_h1_line(lines[i]): + i += 1 + continue + break + return "\n".join(lines[i:]).lstrip("\n") + + +def ensure_doc_title( + body: str, + node_name: str, + lang: str = "en", + translations: Optional[Dict[str, Any]] = None, +) -> str: + """ + Prepend ``# {display_name}`` to the document body. + Strips any existing leading H1(s) first so AI-generated titles are replaced. + """ + clean = strip_leading_h1(body.strip()) + title = get_node_display_name(node_name, lang, translations) + if not clean: + return f"# {title}\n" + return f"# {title}\n\n{clean}" + + +def fix_document_title( + original_content: str, + node_name: str, + lang: str, + translations: Optional[Dict[str, Any]] = None, + lang_config: Optional[dict] = None, + hash_mode: HashMode = "preserve", + en_source_hex: Optional[str] = None, +) -> str: + """ + Rebuild a markdown file with corrected H1. + + hash_mode: + - ``preserve``: keep original disclaimer + SHA footer bytes unchanged + - ``update``: rewrite disclaimer and sync SHA from ``en_source_hex`` / en.md / ai_input + """ + from lib.doc_disclaimer import ( + assemble_document_with_metadata_suffix, + compose_document, + create_en_disclaimer, + create_translated_disclaimer, + extract_metadata_suffix, + strip_ai_disclaimer, + ) + from lib.hash_footer import ( + extract_english_source_fingerprint_hex, + format_source_hash_footer, + load_node_source_sha256, + strip_source_hash_footer, + ) + + body = strip_ai_disclaimer(strip_source_hash_footer(original_content)) + fixed_body = ensure_doc_title(body, node_name, lang, translations) + + if hash_mode == "preserve": + suffix = extract_metadata_suffix(original_content) + return assemble_document_with_metadata_suffix(fixed_body, suffix) + + if en_source_hex: + src_fp = en_source_hex + elif lang == "en": + src_fp = ( + load_node_source_sha256(node_name) + or extract_english_source_fingerprint_hex(original_content) + ) + else: + src_fp = None + + if lang == "en": + disclaimer = create_en_disclaimer(node_name) + else: + disclaimer = create_translated_disclaimer(lang, node_name, lang_config or {}) + + footer = format_source_hash_footer(src_fp) if src_fp else "" + return compose_document(fixed_body, disclaimer, footer) diff --git a/pipeline/lib/hash_footer.py b/pipeline/lib/hash_footer.py new file mode 100644 index 000000000..f5c7f055a --- /dev/null +++ b/pipeline/lib/hash_footer.py @@ -0,0 +1,88 @@ +""" +Shared helpers for the optional SHA-256 footer on en.md (node class body only). +Fingerprints match ``NodeVersionTracker`` / ``prepare_ai_input`` after contextual extraction: +the saved ``ai_input/*/source_code.py`` may include preamble + cross-file context, but the +hash is computed on the node class definition alone. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import Optional + +from lib.node_source_extract import class_source_from_contextual_bundle +from lib.paths import AI_INPUT_DIR + +AI_INPUT_PATH = AI_INPUT_DIR + +SOURCE_HASH_FOOTER_RE = re.compile( + r"\n---\s*\n\*\*Source fingerprint \(SHA-256\):\*\*\s*`[a-fA-F0-9]{64}`\s*$", + re.MULTILINE, +) + +# Same label as in ``format_source_hash_footer`` / en.md (keep ASCII so translations stay comparable). +ENGLISH_SOURCE_FINGERPRINT_HEX_RE = re.compile( + r"\*\*Source fingerprint \(SHA-256\):\*\*\s*`([a-fA-F0-9]{64})`", +) + + +def strip_source_hash_footer(markdown_body: str) -> str: + """Remove trailing source fingerprint footer (e.g. before translation).""" + return SOURCE_HASH_FOOTER_RE.sub("", markdown_body.rstrip()).rstrip() + + +def extract_english_source_fingerprint_hex(markdown_body: str) -> Optional[str]: + """Parse the English footer line left by ``batch_generate_docs``; returns lowercase hex or None.""" + m = ENGLISH_SOURCE_FINGERPRINT_HEX_RE.search(markdown_body) + if not m: + return None + h = m.group(1).lower() + if len(h) == 64 and all(c in "0123456789abcdef" for c in h): + return h + return None + + +def strip_trailing_fingerprint_section(markdown_body: str) -> str: + """Drop a trailing ``---`` block that contains a SHA-256 hex in backticks (any localized heading).""" + body = markdown_body.rstrip() + idx = body.rfind("\n---") + if idx == -1: + return markdown_body + tail = body[idx:] + if re.search(r"`[a-fA-F0-9]{64}`", tail): + return body[:idx].rstrip() + return markdown_body + + +def load_node_source_sha256(node_name: str) -> Optional[str]: + """SHA-256 (hex, UTF-8) of the node ``class`` body — same scope as ``basic_info.version_info.source_hash``.""" + + node_dir = AI_INPUT_PATH / node_name + basic = node_dir / "basic_info.json" + if basic.exists(): + try: + with open(basic, encoding="utf-8") as f: + data = json.load(f) + h = (data.get("version_info") or {}).get("source_hash") + if isinstance(h, str) and len(h) == 64: + lc = h.lower() + if all(c in "0123456789abcdef" for c in lc): + return lc + except (json.JSONDecodeError, OSError, TypeError): + pass + src_file = node_dir / "source_code.py" + if src_file.exists(): + try: + text = src_file.read_text(encoding="utf-8") + body = class_source_from_contextual_bundle(text) + return hashlib.sha256(body.encode("utf-8")).hexdigest() + except OSError: + pass + return None + + +def format_source_hash_footer(sha256_hex: str) -> str: + return f"\n\n---\n**Source fingerprint (SHA-256):** `{sha256_hex}`\n" diff --git a/pipeline/lib/node_source_extract.py b/pipeline/lib/node_source_extract.py new file mode 100644 index 000000000..cc38c81ec --- /dev/null +++ b/pipeline/lib/node_source_extract.py @@ -0,0 +1,811 @@ +""" +Richer extraction of ComfyUI node sources for docs + hashing. + +Preset “depth” (optional): **NODE_SOURCE_EXTRACTION_DEPTH** + shallow — preamble off; no cross-file; tiny iteration budget (still class + optional resolve off) + standard — slim preamble + strict imports (recommended default when unset); no cross-file + deep — full file preamble above the class + cross-file resolution on + +Fine-grained env (always wins over preset when explicitly set): + + NODE_SOURCE_PREAMBLE_MODE=none|slim|full + none — skip same-file preamble; cross-file snippets still obey NODE_SOURCE_RESOLVE_IMPORTS + NODE_SOURCE_IMPORT_MATCH=strict|broad + strict — an ``import a.b.c`` is kept only if a call chain uses ``a.b.c`` or a longer prefix + (avoids pulling every ``import comfy.*`` when only ``comfy.sd`` is used) + broad — legacy behaviour: any ``comfy.`` callee pulled in all top-level ``comfy.*`` imports + NODE_SOURCE_SLIM_ITERATIONS=N (default 3, max 32) — passes expanding helper symbols in slim mode + + NODE_SOURCE_RESOLVE_IMPORTS=0|1 + NODE_SOURCE_RESOLVE_MAX_FUNCS (default 8) + NODE_SOURCE_RESOLVE_MAX_CHARS_EACH (default 6000) + NODE_SOURCE_MAX_CONTEXT_CHARS + +Overrides: NodeSourceExtractConfig / ``prepare_ai_input`` CLI (--source-*). +""" + +from __future__ import annotations + +import argparse +import ast +import os +import sys +import re +from dataclasses import dataclass, fields, replace +from pathlib import Path +from typing import Any, List, Optional, Set, Tuple + + +EXTRACTION_DEPTH_PRESETS: dict[str, dict[str, Any]] = { + "shallow": { + "preamble_mode": "none", + "resolve_imports": False, + "slim_iterations": 1, + "import_match": "strict", + }, + "standard": { + "preamble_mode": "slim", + "resolve_imports": False, + "slim_iterations": 3, + "import_match": "strict", + }, + "deep": { + "preamble_mode": "full", + "resolve_imports": True, + "slim_iterations": 3, + "import_match": "strict", + }, +} + + +@dataclass(frozen=True) +class NodeSourceExtractConfig: + """Configuration for preamble + optional cross-file resolution.""" + + preamble_mode: str = "slim" + resolve_imports: bool = False + max_context_chars: int = 200_000 + resolve_max_funcs: int = 8 + resolve_max_chars_each: int = 6000 + import_match: str = "strict" # strict | broad + slim_iterations: int = 3 + + @classmethod + def from_env(cls) -> "NodeSourceExtractConfig": + depth_key = os.getenv("NODE_SOURCE_EXTRACTION_DEPTH", "").strip().lower() + preset = EXTRACTION_DEPTH_PRESETS.get(depth_key, {}) + + def env_or(key: str) -> Optional[str]: + v = os.getenv(key) + if v is None: + return None + vs = v.strip() + return vs if vs != "" else None + + def env_int_fallback(env_key: str, fallback: int) -> int: + raw = env_or(env_key) + if raw is None: + return fallback + try: + return int(raw) + except ValueError: + return fallback + + # preamble + preamble_mode = env_or("NODE_SOURCE_PREAMBLE_MODE") + if preamble_mode is None: + preamble_mode = str(preset.get("preamble_mode", "slim")) + preamble_mode = preamble_mode.strip().lower() + if preamble_mode not in ("none", "slim", "full"): + preamble_mode = "slim" + + ri = env_or("NODE_SOURCE_RESOLVE_IMPORTS") + if ri is None: + resolve_imports = bool(preset.get("resolve_imports", False)) + else: + resolve_imports = ri == "1" + + im_raw = env_or("NODE_SOURCE_IMPORT_MATCH") + if im_raw is None: + import_match = str(preset.get("import_match", "strict")) + else: + import_match = im_raw.lower() + if import_match not in ("strict", "broad"): + import_match = "strict" + + si_raw = env_or("NODE_SOURCE_SLIM_ITERATIONS") + try: + slim_iterations = int(si_raw) if si_raw is not None else int(preset.get("slim_iterations", 3)) + except ValueError: + slim_iterations = int(preset.get("slim_iterations", 3)) + + slim_iterations = max(1, min(slim_iterations, 32)) + + max_context_chars = env_int_fallback("NODE_SOURCE_MAX_CONTEXT_CHARS", 200_000) + resolve_max_funcs = env_int_fallback("NODE_SOURCE_RESOLVE_MAX_FUNCS", 8) + resolve_max_chars_each = env_int_fallback("NODE_SOURCE_RESOLVE_MAX_CHARS_EACH", 6000) + + return cls( + preamble_mode=preamble_mode, + resolve_imports=resolve_imports, + max_context_chars=max_context_chars, + resolve_max_funcs=resolve_max_funcs, + resolve_max_chars_each=resolve_max_chars_each, + import_match=import_match, + slim_iterations=slim_iterations, + ) + + +def register_source_extract_cli_args(parser: argparse.ArgumentParser) -> None: + """Optional flags shared by prepare_ai_input (and tooling that uses the same parser).""" + parser.add_argument( + "--source-depth", + choices=list(EXTRACTION_DEPTH_PRESETS.keys()), + default=None, + help=( + "shallow|standard|deep bundle (CLI): maps to preamble / resolve / iterations / import_match. " + "Runs after env; narrower --source-* flags override individual fields.", + ), + ) + parser.add_argument( + "--source-preamble", + choices=["none", "slim", "full"], + default=None, + help="same-file preamble: none | slim | full. Default NODE_SOURCE_PREAMBLE_MODE / depth preset.", + ) + parser.add_argument( + "--source-resolve", + choices=["0", "1"], + default=None, + help="cross-file snippets (1=on). Default: NODE_SOURCE_RESOLVE_IMPORTS.", + ) + parser.add_argument( + "--source-max-context-chars", + type=int, + default=None, + help="max assembled extract length (default NODE_SOURCE_MAX_CONTEXT_CHARS / 200000).", + ) + parser.add_argument( + "--source-resolve-max-funcs", + type=int, + default=None, + help="max resolved callee snippets (NODE_SOURCE_RESOLVE_MAX_FUNCS).", + ) + parser.add_argument( + "--source-resolve-max-chars-each", + type=int, + default=None, + help="max chars per resolved snippet (NODE_SOURCE_RESOLVE_MAX_CHARS_EACH).", + ) + parser.add_argument( + "--source-import-match", + choices=["strict", "broad"], + default=None, + help=( + "slim preamble import filter: strict (prefix-realistic) vs broad (legacy). " + "NODE_SOURCE_IMPORT_MATCH." + ), + ) + parser.add_argument( + "--source-slim-iterations", + type=int, + default=None, + help="slim preamble expansion passes (1–32). NODE_SOURCE_SLIM_ITERATIONS.", + ) + + +def extract_config_from_parsed_args(namespace: argparse.Namespace) -> NodeSourceExtractConfig: + """Merge NODE_SOURCE_* env with optional CLI overrides (for testing without editing .env).""" + cfg = NodeSourceExtractConfig.from_env() + + sd = getattr(namespace, "source_depth", None) + if sd: + preset = EXTRACTION_DEPTH_PRESETS.get(sd.strip().lower()) + if preset: + cfg = replace( + cfg, + preamble_mode=str(preset.get("preamble_mode", cfg.preamble_mode)), + resolve_imports=bool(preset.get("resolve_imports", cfg.resolve_imports)), + slim_iterations=max( + 1, + min(int(preset.get("slim_iterations", cfg.slim_iterations)), 32), + ), + import_match=str(preset.get("import_match", cfg.import_match)), + ) + + replacements: dict[str, Any] = {} + + pm = getattr(namespace, "source_preamble", None) + if pm is not None: + replacements["preamble_mode"] = pm.strip().lower() + + sr = getattr(namespace, "source_resolve", None) + if sr is not None: + replacements["resolve_imports"] = sr == "1" + + smcc = getattr(namespace, "source_max_context_chars", None) + if smcc is not None: + replacements["max_context_chars"] = smcc + + srmf = getattr(namespace, "source_resolve_max_funcs", None) + if srmf is not None: + replacements["resolve_max_funcs"] = srmf + + srme = getattr(namespace, "source_resolve_max_chars_each", None) + if srme is not None: + replacements["resolve_max_chars_each"] = srme + + sim = getattr(namespace, "source_import_match", None) + if sim is not None: + replacements["import_match"] = sim.strip().lower() + + ssi = getattr(namespace, "source_slim_iterations", None) + if ssi is not None: + try: + replacements["slim_iterations"] = max(1, min(int(ssi), 32)) + except (TypeError, ValueError): + pass + + return replace(cfg, **replacements) if replacements else cfg + + +def merge_extract_config_overrides(base: Optional[NodeSourceExtractConfig] = None, **kw: Any) -> NodeSourceExtractConfig: + """Build config from optional base + explicit overrides (None = omit).""" + b = base or NodeSourceExtractConfig.from_env() + replacements: dict[str, Any] = {} + for f in fields(NodeSourceExtractConfig): + k = f.name + if k in kw and kw[k] is not None: + replacements[k] = kw[k] + return replace(b, **replacements) + + +def _is_registered_comfy_node_class(c: ast.ClassDef) -> bool: + for item in c.body: + if not isinstance(item, ast.FunctionDef): + continue + if item.name in ("INPUT_TYPES", "define_schema"): + return True + return False + + +def _stmt_source_lines(lines: List[str], stmt: ast.stmt) -> str: + return "\n".join(lines[stmt.lineno - 1 : stmt.end_lineno]) + + +def _attr_to_dotted(node: Optional[ast.AST]) -> Optional[str]: + if node is None: + return None + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + inner = _attr_to_dotted(node.value) + if inner is None: + return None + return f"{inner}.{node.attr}" + return None + + +def _collect_usage_from_class(class_node: ast.ClassDef) -> Tuple[Set[str], Set[str]]: + bare: Set[str] = set() + dotted: Set[str] = set() + skip = frozenset( + {"self", "cls", "Optional", "List", "Dict", "Set", "Tuple", "Union", "Any", "TypeVar"} + ) + + for node in ast.walk(class_node): + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + if node.id not in skip and not node.id.startswith("_"): + bare.add(node.id) + + if isinstance(node, ast.Call): + ch = _attr_to_dotted(node.func) + if ch: + dotted.add(ch) + elif isinstance(node.func, ast.Name): + bare.add(node.func.id) + return bare, dotted + + +def _import_top_bind(alias: ast.alias) -> str: + name = alias.asname or alias.name + return name.split(".")[0] + + +def _import_prefix_used_by_dotted(prefix: str, dotted: Set[str]) -> bool: + """True if some attribute access / call chain equals or extends ``prefix`` (dot-separated).""" + p = prefix.strip() + if not p: + return False + for d in dotted: + if d == p or d.startswith(p + "."): + return True + return False + + +def _preamble_stmt_used(stmt: ast.stmt, bare: Set[str], dotted: Set[str], import_match: str) -> bool: + if isinstance(stmt, ast.Import): + for al in stmt.names: + mod_full = al.name or "" + top = mod_full.split(".")[0] if mod_full else "" + + if al.asname: + if al.asname in bare: + return True + else: + # Only ``import pkg`` (no dotted path) attaches the root symbol itself. + # For ``import comfy.sd``, the bare Name ``comfy`` also appears inside attribute + # chains (comfy.sd...) and must NOT pull every ``import comfy.*`` line. + if top and top in bare and "." not in mod_full: + return True + + if import_match == "broad": + bind = _import_top_bind(al) + first = mod_full.split(".")[0] if mod_full else "" + if bind in bare: + return True + if any(x.startswith(bind + ".") or x.startswith(first + ".") for x in dotted): + return True + elif mod_full and _import_prefix_used_by_dotted(mod_full, dotted): + return True + return False + + if isinstance(stmt, ast.ImportFrom): + pkg = stmt.module or "" + pkg0 = pkg.split(".")[0] if pkg else "" + + if import_match == "broad" and pkg: + if any(d.startswith(pkg + ".") or (pkg0 and d.startswith(pkg0 + ".")) for d in dotted): + return True + elif pkg and _import_prefix_used_by_dotted(pkg, dotted): + return True + + for al in stmt.names: + if al.name == "*": + return True + bn = al.asname or al.name + if bn in bare or al.name in bare: + return True + if pkg: + fq = f"{pkg}.{al.name}" + if fq in dotted or _import_prefix_used_by_dotted(fq, dotted): + return True + + return False + + if isinstance(stmt, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + targets = [] + if isinstance(stmt, ast.Assign): + targets.extend(stmt.targets) + elif isinstance(stmt, ast.AnnAssign): + targets.append(stmt.target) + else: + targets.append(stmt.target) + ids = [] + for t in targets: + if isinstance(t, ast.Name): + ids.append(t.id) + return any(name in bare for name in ids) + + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)): + return stmt.name in bare + if isinstance(stmt, ast.ClassDef): + return stmt.name in bare + return False + + +def _helper_absorb_bare(funcs: List[ast.FunctionDef | ast.AsyncFunctionDef], bare: Set[str]) -> Set[str]: + out = set(bare) + for hf in funcs: + if hf.name not in out: + continue + for node in ast.walk(hf): + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + if node.id not in ("self", "cls"): + out.add(node.id) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + out.add(node.func.id) + return out + + +def _deepen_helpers(preamble_stmts: List[ast.stmt], bare: Set[str]) -> Set[str]: + hfs = [ + x + for x in preamble_stmts + if isinstance(x, (ast.FunctionDef, ast.AsyncFunctionDef)) and x.name in bare + ] + return _helper_absorb_bare(hfs, bare) + + +def _build_slim_preamble( + preamble_stmts: List[ast.stmt], + lines_list: List[str], + bare0: Set[str], + dotted0: Set[str], + *, + slim_iterations: int, + import_match: str, +) -> str: + bare = _deepen_helpers(preamble_stmts, set(bare0)) + dotted = set(dotted0) + n_pass = max(1, min(slim_iterations, 32)) + for _ in range(n_pass): + sel = [st for st in preamble_stmts if _preamble_stmt_used(st, bare, dotted, import_match)] + nb = set(bare) + for st in sel: + if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)): + for n in ast.walk(st): + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load): + nid = n.id + if nid not in ("self", "cls") and not nid.startswith("__"): + nb.add(nid) + if nb <= bare: + break + bare = nb | bare + bare = _deepen_helpers(preamble_stmts, bare) + chunks = [ + _stmt_source_lines(lines_list, st) + for st in preamble_stmts + if _preamble_stmt_used(st, bare, dotted, import_match) + ] + return "\n\n".join(c for c in chunks if c.strip()) + + +def _resolve_module_file(comfy_root: Path, mod_dotted: str) -> Optional[Path]: + comfy_root = comfy_root.resolve() + parts = mod_dotted.split(".") + if len(parts) == 1: + fp = comfy_root / f"{parts[0]}.py" + if fp.is_file(): + return fp + init_py = comfy_root / parts[0] / "__init__.py" + return init_py if init_py.is_file() else None + dpath = comfy_root.joinpath(*parts) + cand = dpath.with_suffix(".py") + if cand.is_file(): + return cand + init_py = dpath / "__init__.py" + return init_py if init_py.is_file() else None + + +def _slice_named_block(mod_file: Path, remainder: List[str]) -> Optional[str]: + """remainder: ['load_checkpoint_guess_config'] or nested ['Class','meth'].""" + try: + text = mod_file.read_text(encoding="utf-8") + tree = ast.parse(text) + except (SyntaxError, OSError): + return None + + if len(remainder) == 1: + tgt = remainder[0] + for stmt in ast.walk(tree): + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)): + if stmt.name == tgt and stmt.lineno is not None and stmt.end_lineno is not None: + chunk = "\n".join(text.split("\n")[stmt.lineno - 1 : stmt.end_lineno]) + return chunk + for stmt in tree.body: + if isinstance(stmt, ast.Assign): + for t in stmt.targets: + if isinstance(t, ast.Name) and t.id == tgt and stmt.lineno and stmt.end_lineno: + return "\n".join(text.split("\n")[stmt.lineno - 1 : stmt.end_lineno]) + return None + + body = tree.body + cur: Optional[ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef] = None + for i, part in enumerate(remainder[:-1]): + found_cls = None + for stmt in body: + if isinstance(stmt, ast.ClassDef) and stmt.name == part: + found_cls = stmt + break + if found_cls is None: + return None + body = found_cls.body + cur = found_cls + + last = remainder[-1] + for stmt in body: + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)) and stmt.name == last: + assert stmt.lineno and stmt.end_lineno + return "\n".join(text.split("\n")[stmt.lineno - 1 : stmt.end_lineno]) + return None + + +def _try_resolve_dotted(dotted_call: str, comfy_root: Path, resolve_max_chars_each: int) -> Optional[str]: + parts = dotted_call.split(".") + if len(parts) < 2: + return None + cap_n = resolve_max_chars_each + for depth in range(len(parts) - 1, 0, -1): + mod_dots = ".".join(parts[:depth]) + remainder = parts[depth:] + if not remainder: + continue + mf = _resolve_module_file(comfy_root, mod_dots) + if mf is None or not mf.is_file(): + continue + sn = _slice_named_block(mf, remainder) + if sn: + try: + rel = mf.relative_to(comfy_root) + except ValueError: + rel = mf.name + cap = sn[:cap_n] + if len(sn) > cap_n: + cap += "\n# ... truncated (resolve_max_chars_each) ..." + return f"# module: {mod_dots} file: {rel}\n{cap}" + return None + + +def _cross_file_section(comfy_root: Path, dotted: Set[str], config: NodeSourceExtractConfig) -> str: + if not (config.resolve_imports and comfy_root.is_dir()): + return "" + + comfy_root = comfy_root.resolve() + + chunks: List[str] = [] + n = 0 + for call in sorted(dotted): + if n >= config.resolve_max_funcs: + break + got = _try_resolve_dotted(call, comfy_root, config.resolve_max_chars_each) + if got: + chunks.extend([f"\n# --- resolved call chain: {call} ---", got]) + n += 1 + if not chunks: + return "" + hdr = "# --- cross-file context (NODE_SOURCE_RESOLVE_IMPORTS=1, under COMFYUI_PATH) ---" + return "\n".join([hdr] + chunks) + "\n" + + +def extract_node_source_with_context( + file_path: Path, + class_name: str, + *, + node_type_hint: Optional[str] = None, + comfy_root: Optional[Path] = None, + config: Optional[NodeSourceExtractConfig] = None, +) -> str: + del node_type_hint + cfg = config or NodeSourceExtractConfig.from_env() + max_chars = cfg.max_context_chars + if comfy_root is not None: + root = comfy_root + else: + _cu = Path(os.getenv("COMFYUI_PATH", "")) + root = _cu if _cu.is_dir() else Path() + + path = Path(file_path) + try: + content = path.read_text(encoding="utf-8") + except OSError: + return "" + + lines_list = content.split("\n") + + try: + tree = ast.parse(content) + except SyntaxError: + return _fallback_class_only_regex(content, lines_list, class_name) + + target: Optional[ast.ClassDef] = None + preamble_stmts: List[ast.stmt] = [] + + for stmt in tree.body: + if isinstance(stmt, ast.ClassDef) and stmt.name == class_name: + target = stmt + break + if isinstance(stmt, ast.ClassDef) and _is_registered_comfy_node_class(stmt): + continue + preamble_stmts.append(stmt) + + if target is None: + return _fallback_class_only_regex(content, lines_list, class_name) + + class_src_lines = lines_list[target.lineno - 1 : target.end_lineno] + class_src = "\n".join(class_src_lines) + + MARK_PRE = "# --- preceding context (same-file, AST-filtered unless PREAMBLE_MODE=full) ---" + MARK_CLS = "# --- node class ---" + + bare, dotted = _collect_usage_from_class(target) + + preamble_mode = (cfg.preamble_mode or "slim").strip().lower() + if preamble_mode == "full": + pre_segs = [_stmt_source_lines(lines_list, st) for st in preamble_stmts] + preamble_joined = "\n\n".join(s for s in pre_segs if s.strip()) + elif preamble_mode == "none": + preamble_joined = "" + else: + preamble_joined = _build_slim_preamble( + preamble_stmts, + lines_list, + bare, + dotted, + slim_iterations=cfg.slim_iterations, + import_match=cfg.import_match, + ) + + ext = _cross_file_section(root, dotted, cfg) if root.is_dir() else "" + + # Order: preamble (slim/full) → cross-file → node class + + blocks: List[str] = [] + if preamble_joined.strip(): + blocks.append(MARK_PRE + "\n" + preamble_joined) + if ext.strip(): + blocks.append(ext.strip()) + blocks.append(MARK_CLS + "\n" + class_src) + + out = "\n\n".join(blocks) + + cls_block_only = MARK_CLS + "\n" + class_src + note = "# --- preamble trimmed (max_context_chars cap) ---\n\n" + + if len(out) <= max_chars: + return out + + # Drop cross-file first, then preamble tail, keeping full classSrc + slack = max_chars - len(cls_block_only) - 48 + if slack <= 0: + return cls_block_only[:max_chars] + + if ext.strip(): + ext_short = ext + cand = ("\n\n".join([MARK_PRE + "\n" + preamble_joined, ext_short.strip(), cls_block_only]) if preamble_joined.strip() else ("\n\n".join([ext_short.strip(), cls_block_only]))) + if len(cand) <= max_chars: + return cand + cand2 = MARK_CLS + "\n" + class_src + if preamble_joined.strip(): + preamble_reduced = note + preamble_joined.strip()[- (slack // 2) :] + return (MARK_PRE + "\n" + preamble_reduced + "\n\n" + cls_block_only)[:max_chars] + return cls_block_only[:max_chars] + + if preamble_joined.strip(): + pre = preamble_joined.strip() + if len(pre) > slack: + pre = note + pre[-slack:] + return (MARK_PRE + "\n" + pre + "\n\n" + cls_block_only)[:max_chars] + return cls_block_only[:max_chars] + + +def _fallback_class_only_regex(content: str, lines_list: List[str], class_name: str) -> str: + pattern = rf"class\s+{re.escape(class_name)}\b.*?(?=\n(?:class\s|\Z|async def comfy_entrypoint))" + match = re.search(pattern, content, re.DOTALL) + if match: + return "# --- node class (fallback extract) ---\n" + match.group(0).rstrip() + try: + tree = ast.parse(content) + except SyntaxError: + return "" + cand: Optional[ast.ClassDef] = None + for stmt in tree.body: + if isinstance(stmt, ast.ClassDef) and stmt.name == class_name: + cand = stmt + break + if cand is None: + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == class_name: + cand = node + break + if cand is None: + return "" + return "# --- node class (fallback extract) ---\n" + "\n".join( + lines_list[cand.lineno - 1 : cand.end_lineno] + ) + + +_FALLBACK_MARK = "# --- node class (fallback extract) ---\n" + + +def extract_node_class_source(file_path: Path, class_name: str) -> str: + """Return only the node's `class` definition from the real source file (no preamble / cross-file bundle). + + Used for version fingerprints: contextual extracts can change when resolved callees or imports + shift, but the registered node class is the stable contract for "did this node change?" + """ + path = Path(file_path) + try: + content = path.read_text(encoding="utf-8") + except OSError: + return "" + + lines_list = content.split("\n") + try: + tree = ast.parse(content) + except SyntaxError: + return _strip_fallback_class_banner(_fallback_class_only_regex(content, lines_list, class_name)) + + target: Optional[ast.ClassDef] = None + for stmt in tree.body: + if isinstance(stmt, ast.ClassDef) and stmt.name == class_name: + target = stmt + break + + if target is None: + return _strip_fallback_class_banner(_fallback_class_only_regex(content, lines_list, class_name)) + + return "\n".join(lines_list[target.lineno - 1 : target.end_lineno]) + + +def class_source_from_contextual_bundle(text: str) -> str: + """If `text` is a contextual bundle (preamble + optional cross-file + node class), return class part only.""" + marker = "# --- node class ---" + if marker in text: + return text.split(marker, 1)[1].lstrip("\n") + if text.startswith(_FALLBACK_MARK): + return text[len(_FALLBACK_MARK) :].lstrip("\n") + return text + + +def _strip_fallback_class_banner(text: str) -> str: + if text.startswith(_FALLBACK_MARK): + return text[len(_FALLBACK_MARK) :] + return text + + +def extract_node_source_code( + file_path: Path, + class_name: str, + node_type: str, + *, + comfy_root: Optional[Path] = None, + config: Optional[NodeSourceExtractConfig] = None, +) -> str: + _ = node_type # compat + if comfy_root is not None: + cr_opt = comfy_root if comfy_root.is_dir() else None + else: + comfy = Path(os.getenv("COMFYUI_PATH", "")) + cr_opt = comfy if comfy.is_dir() else None + return extract_node_source_with_context( + file_path, class_name, comfy_root=cr_opt, config=config + ) + + +def extract_node_source_code_with_root( + file_path: Path, + class_name: str, + node_type: str, + *, + comfy_root: Path, + config: Optional[NodeSourceExtractConfig] = None, +) -> str: + return extract_node_source_code( + file_path, class_name, node_type, comfy_root=comfy_root, config=config + ) + + +def _cli_main(argv: Optional[List[str]] = None) -> int: + """ + Smoke-test extractor without modifying env: + python node_source_extract.py \\ + --py /abs/path/nodes.py --class CheckpointLoader \\ + [--comfy-root /path/to/ComfyUI] [--source-preamble slim|full] [--source-resolve 0|1] + """ + p = argparse.ArgumentParser(description="Print extracted node source to stdout.") + p.add_argument("--py", "--file", dest="py_file", type=Path, required=True) + p.add_argument("--class", dest="class_name", required=True, help="Python class name to extract.") + register_source_extract_cli_args(p) + p.add_argument("--comfy-root", type=Path, default=None, help="ComfyUI root (default COMFYUI_PATH).") + args = p.parse_args(argv) + cfg = extract_config_from_parsed_args(args) + root = args.comfy_root + if root is None: + c = Path(os.getenv("COMFYUI_PATH", "")) + root = c if c.is_dir() else None + txt = extract_node_source_code( + Path(args.py_file), + args.class_name, + "cli", + comfy_root=root, + config=cfg, + ) + if not txt: + return 1 + sys.stdout.write(txt) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_cli_main()) diff --git a/pipeline/lib/paths.py b/pipeline/lib/paths.py new file mode 100644 index 000000000..6b2a85aa5 --- /dev/null +++ b/pipeline/lib/paths.py @@ -0,0 +1,53 @@ +"""Central path configuration for doc_automation.""" + +from __future__ import annotations + +import os +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +LIB_DIR = REPO_ROOT / "lib" +SCRIPTS_DIR = REPO_ROOT / "scripts" +CONFIG_DIR = REPO_ROOT / "config" +DATA_DIR = REPO_ROOT / "data" +DOCS_DIR = REPO_ROOT / "docs" +AI_INPUT_DIR = REPO_ROOT / "ai_input" +LOGS_DIR = REPO_ROOT / "logs" +TRANSLATION_BATCHES_DIR = REPO_ROOT / "translation_batches" + +ENV_FILE = REPO_ROOT / ".env" +TRANSLATION_CONFIG = CONFIG_DIR / "translation_config.json" +DOC_RULES = CONFIG_DIR / "doc_rules.txt" +TRANSLATION_RULES = CONFIG_DIR / "translation_rules.txt" + +ALL_NODES_INFO = DATA_DIR / "all_nodes_info.json" +NODE_VERSIONS = DATA_DIR / "node_versions.json" +NODE_TRANSLATIONS = DATA_DIR / "node_translations.json" +MISSING_NODES_REPORT = DATA_DIR / "missing_nodes_report.json" + + +def ensure_data_dir() -> Path: + """Create data/ if missing (scanner and version DB write here).""" + DATA_DIR.mkdir(parents=True, exist_ok=True) + return DATA_DIR + + +def default_embedded_docs_path() -> Path: + env = os.getenv("EMBEDDED_DOCS_PATH") + if env: + return Path(env) + sibling = REPO_ROOT.parent / "embedded-docs" + if (sibling / "comfyui_embedded_docs" / "docs").is_dir(): + return sibling + return REPO_ROOT.parent + + +def embedded_docs_dir() -> Path: + return default_embedded_docs_path() / "comfyui_embedded_docs" / "docs" + + +def load_dotenv() -> None: + from dotenv import load_dotenv as _load + + _load(ENV_FILE) diff --git a/pipeline/main.py b/pipeline/main.py new file mode 100644 index 000000000..db43b6791 --- /dev/null +++ b/pipeline/main.py @@ -0,0 +1,1287 @@ +#!/usr/bin/env python3 +""" +Main control script for ComfyUI node documentation automation. +Orchestrates: scan -> prepare -> generate/translate -> update reports. + +Usage +----- + + # === Interactive (default when no args) === + python3 main.py + python3 main.py --interactive + # Then choose: Scan / Generate docs / Translate from the menu. + + # === Document generation (English, non-interactive) === + python3 main.py --mode test [--count N] [--force] + + # Generate default number of nodes (test mode, 20 nodes) + python3 main.py --mode test + + # Generate N nodes (test mode) + python3 main.py --count 50 + + # Generate all missing nodes + python3 main.py --mode all + + # Full refresh: English only unless you opt in to translation on CLI: + python3 main.py --mode regenerate-all + python3 main.py --mode regenerate-all --also-translate-all + # Interactive menu 7) defaults to running full translation after English; CLI needs the flag above. + + # Generate/force-regenerate a single node + python3 main.py --mode node --node [--force] + + # Update an existing node doc (re-reads source, sends current doc as reference so + # the AI updates params/outputs while preserving manual edits) + python3 main.py --mode node --node --force + + # Only scan, no generation + python3 main.py --scan-only + + # === Fix existing docs (no AI) === + python3 main.py --mode fix --fix-action doc-titles + python3 main.py --mode fix --fix-action doc-titles --hash-mode preserve --dry-run + python3 main.py --mode fix --fix-action doc-titles --hash-mode update + python3 main.py --mode fix --fix-action doc-titles --node KSampler --lang zh + + # === Translation (other languages) === + python3 main.py --translate --lang [--mode MODE] [--count N] [--force] + + # Translate to one language (test: 20 nodes) + python3 main.py --translate --lang zh --count 10 + python3 main.py --translate --lang pt-BR --mode all + + # Translate to all supported languages + python3 main.py --translate --all-languages --count 10 + python3 main.py --translate --all-languages --mode all + + # Force retranslate ALL locales for EVERY node that has en.md (ignores missing report) + python3 main.py --retranslate-all-languages + python3 main.py --translate --all-languages --mode all --force --force-all-translation-nodes + + # Force retranslate existing docs (batches from missing report unless --force-all-translation-nodes) + python3 main.py --translate --lang zh --mode all --force + + # Supported languages: zh, zh-TW, es, fr, ja, ko, ru, ar, tr, pt-BR, fa + +Options +------- + --mode test | all | resume | node | changed | regenerate-all | fix (default: test) + --fix-action doc-titles (with --mode fix; default doc-titles) + --fix-scope test | all (with --mode fix; default all) + --hash-mode preserve | update (with --mode fix; default preserve) + --dry-run With --mode fix: preview title fixes without writing files + --count N nodes in test mode (default: 20) + --node Node name (required with --mode node) + --force Overwrite existing docs/translations + --prepare-limit Only with --mode regenerate-all: prepare & regenerate first N nodes + --also-translate-all Only with --mode regenerate-all: after English, run all-languages translation (mode=all, force) + --translate Run translation workflow instead of generation + --lang Target language (required with --translate) + --all-languages Translate to all 11 languages + --force-all-translation-nodes Only with --translate: batch every node with en.md (prepare_translation --force-all-nodes) + --retranslate-all-languages Shorthand: all langs + mode all + force + force-all-translation-nodes + + Translated *.md files append the same English SHA footer line as en.md (trace English source version). + + # Interactive mode (no args, or --interactive): menu-driven + python3 main.py + python3 main.py --interactive +""" + +import os +import sys +import subprocess +from pathlib import Path + +from lib.paths import REPO_ROOT, load_dotenv + +load_dotenv() +from datetime import datetime + +# Supported languages for translation +LANGUAGES = ['zh', 'zh-TW', 'es', 'fr', 'ja', 'ko', 'ru', 'ar', 'tr', 'pt-BR', 'fa'] +LANG_NAMES = { + 'zh': '简体中文', 'zh-TW': '繁體中文', 'es': 'Español', 'fr': 'Français', + 'ja': '日本語', 'ko': '한국어', 'ru': 'Русский', 'ar': 'العربية', + 'tr': 'Türkçe', 'pt-BR': 'Português (BR)', 'fa': 'فارسی', +} + + +class DocumentationWorkflow: + """Main workflow controller for documentation generation""" + + def __init__(self): + self.repo_root = REPO_ROOT + self.script_dir = self.repo_root / "scripts" + self.scan_script = self.script_dir / "scan_missing_nodes.py" + self.prepare_script = self.script_dir / "prepare_ai_input.py" + self.generate_script = self.script_dir / "batch_generate_docs.py" + self.prepare_translation_script = self.script_dir / "prepare_translation.py" + self.translate_script = self.script_dir / "batch_translate_docs.py" + self.update_params_script = self.script_dir / "update_param_translations.py" + self.sync_frontend_script = self.script_dir / "sync_frontend_translations.py" + self.sync_to_comfy_docs_script = self.script_dir / "sync_to_comfy_docs.py" + self.fix_doc_titles_script = self.script_dir / "fix_doc_titles.py" + + fp = os.getenv("COMFYUI_FRONTEND_PATH", "").strip() + self.frontend_path = Path(fp) if fp else Path("") + + def run_command(self, script: Path, args: list, description: str) -> bool: + """Run a Python script with arguments""" + print(f"\n{'=' * 80}") + print(f"🚀 {description}") + print(f"{'=' * 80}\n") + + cmd = ["python3", str(script)] + args + env = os.environ.copy() + prefix = str(self.repo_root) + env["PYTHONPATH"] = prefix + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "") + result = subprocess.run(cmd, cwd=self.repo_root, env=env) + + if result.returncode != 0: + print(f"\n❌ Failed: {description}") + return False + + print(f"\n✅ Completed: {description}") + return True + + def scan_nodes(self) -> bool: + """Step 1: Scan for missing nodes""" + return self.run_command( + self.scan_script, + [], + "Step 1: Scanning for missing node documentation" + ) + + def prepare_nodes(self, mode: str, count: int = None, node_name: str = None) -> bool: + """Step 2: Prepare AI input for nodes""" + args = [mode] + + if mode == "test" and count: + args.append(str(count)) + elif mode == "node" and node_name: + args.append(node_name) + + return self.run_command( + self.prepare_script, + args, + f"Step 2: Preparing AI input ({mode} mode)" + ) + + def generate_docs(self, mode: str, count: int = None, node_name: str = None, force: bool = False) -> bool: + """Step 3: Generate documentation with AI""" + args = [mode] + + if mode == "test" and count: + args.extend(["--count", str(count)]) + elif mode == "node" and node_name: + args.extend(["--node", node_name]) + # "changed" mode: batch_generate_docs.py reads batch_nodes.json prepared by prepare_ai_input.py + + if force: + args.append("--force") + + return self.run_command( + self.generate_script, + args, + f"Step 3: Generating documentation ({mode} mode)" + ) + + def update_reports(self) -> bool: + """Step 4: Update all reports""" + print(f"\n{'=' * 80}") + print("🔄 Step 4: Updating reports") + print(f"{'=' * 80}\n") + + # Re-scan to update missing_nodes_report.json + if not self.run_command( + self.scan_script, + [], + "Updating missing_nodes_report.json" + ): + return False + + print(f"\n✅ All reports updated successfully") + return True + + def prepare_translation(self, lang: str, mode: str, count: int = None, force_all_nodes: bool = False) -> bool: + """Prepare translation batch for a specific language""" + args = ["--lang", lang, "--mode", mode] + + if mode == "test" and count: + args.extend(["--count", str(count)]) + + if force_all_nodes: + args.append("--force-all-nodes") + + return self.run_command( + self.prepare_translation_script, + args, + f"Preparing {lang} translation batch ({mode} mode{' + force-all-nodes' if force_all_nodes else ''})" + ) + + def translate_docs(self, lang: str, mode: str, count: int = None, force: bool = False) -> bool: + """Translate documentation to a specific language""" + args = ["--lang", lang, "--mode", mode] + + if mode == "test" and count: + args.extend(["--count", str(count)]) + + if force: + args.append("--force") + + return self.run_command( + self.translate_script, + args, + f"Translating to {lang} ({mode} mode)" + ) + + def sync_frontend_translations(self) -> bool: + """Sync and export frontend translations to node_translations.json""" + if not self.frontend_path.exists(): + print(f"⚠️ Warning: Frontend path not found: {self.frontend_path}") + print(" Skipping frontend translation sync") + return True # Don't fail if frontend not found + + return self.run_command( + self.sync_frontend_script, + [str(self.frontend_path), "--export"], + "Syncing frontend translations" + ) + + def update_param_translations(self, lang: str) -> bool: + """Update parameter translations from frontend for a specific language""" + return self.run_command( + self.update_params_script, + ["--lang", lang], + f"Updating parameter translations for {lang}" + ) + + def run_translation_workflow( + self, + lang: str, + mode: str = "test", + count: int = 10, + force: bool = False, + skip_initial_scan: bool = False, + skip_frontend_sync: bool = False, + force_all_nodes: bool = False, + ): + """Run translation workflow for a specific language""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + print("\n" + "=" * 80) + print("ComfyUI Documentation Translation - Main Workflow") + print("=" * 80) + print(f"Started at: {timestamp}") + print(f"Target language: {lang}") + print(f"Mode: {mode}") + if mode == "test": + print(f"Count: {count} nodes") + print(f"Force retranslate: {force}") + print(f"Prepare batch from all nodes with en.md: {force_all_nodes}") + print("=" * 80) + + # Step 0a: Sync frontend translations (unless skipped for multi-language) + if not skip_frontend_sync: + print(f"\n🔄 Step 0a: Syncing frontend translations...") + if not self.sync_frontend_translations(): + print("\n⚠️ Warning: Frontend translation sync failed, but continuing...") + + # Step 0b: Scan to update missing_nodes_report.json (unless skipped for multi-language) + if not skip_initial_scan: + print(f"\n📊 Step 0b: Scanning to update missing translations...") + if not self.scan_nodes(): + print("\n❌ Translation workflow failed at Step 0b: Scan") + return False + + # Step 1: Prepare translation batch (missing report or every node with en.md) + print(f"\n🔧 Step 1: Preparing {lang} translation batch...") + if not self.prepare_translation(lang, mode, count, force_all_nodes=force_all_nodes): + print("\n❌ Translation workflow failed at Step 1: Prepare") + return False + + # Step 2: Translate documents (trusts batch list, updates JSON incrementally) + print(f"\n🤖 Step 2: Translating to {lang}...") + if not self.translate_docs(lang, mode, count, force): + print("\n❌ Translation workflow failed at Step 2: Translate") + return False + + # Step 3: Update parameter translations from frontend + print(f"\n🔄 Step 3: Updating parameter names with frontend translations...") + if not self.update_param_translations(lang): + print("\n⚠️ Warning: Parameter translation update failed, but continuing...") + # Don't fail the workflow if parameter update fails + + # Success summary + end_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print("\n" + "=" * 80) + print("✅ Translation Workflow Completed!") + print("=" * 80) + print(f"Started: {timestamp}") + print(f"Finished: {end_timestamp}") + print(f"Language: {lang}") + print("\nNote: missing_nodes_report.json updated incrementally") + print(" Parameter names updated with frontend translations") + print("=" * 80 + "\n") + + return True + + def run_all_languages_translation(self, mode: str = "test", count: int = 10, force: bool = False, force_all_nodes: bool = False): + """Run translation workflow for all supported languages""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + languages = ['zh', 'zh-TW', 'es', 'fr', 'ja', 'ko', 'ru', 'ar', 'tr', 'pt-BR', 'fa'] + + print("\n" + "=" * 80) + print("ComfyUI Documentation Translation - ALL LANGUAGES") + print("=" * 80) + print(f"Started at: {timestamp}") + print(f"Languages: {', '.join(languages)}") + print(f"Mode: {mode}") + if mode == "test": + print(f"Count per language: {count} nodes") + print(f"Force retranslate: {force}") + print(f"Prepare batch from all nodes with en.md: {force_all_nodes}") + print("=" * 80) + + # Sync frontend translations first + print("\n🔄 Syncing frontend translations...") + if not self.sync_frontend_translations(): + print("\n⚠️ Warning: Frontend translation sync failed, but continuing...") + + # Initial scan to populate missing_nodes_report.json + print("\n📊 Initial scan to identify missing translations...") + if not self.scan_nodes(): + print("\n❌ Scan failed") + return False + + results = {} + + for lang in languages: + print(f"\n{'=' * 80}") + print(f"🌐 Processing language: {lang}") + print(f"{'=' * 80}") + + # Skip initial scan and frontend sync for each language (already done once) + success = self.run_translation_workflow( + lang, + mode, + count, + force, + skip_initial_scan=True, + skip_frontend_sync=True, + force_all_nodes=force_all_nodes, + ) + results[lang] = success + + if not success: + print(f"\n⚠️ Warning: Translation failed for {lang}, continuing with next language...") + + # Final scan to update complete status + print("\n🔄 Final scan to ensure all data is current...") + self.scan_nodes() + + # Final summary + end_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print("\n" + "=" * 80) + print("📊 ALL LANGUAGES TRANSLATION SUMMARY") + print("=" * 80) + print(f"Started: {timestamp}") + print(f"Finished: {end_timestamp}") + print("\nResults:") + for lang, success in results.items(): + status = "✅ Success" if success else "❌ Failed" + print(f" {lang}: {status}") + + successful = sum(1 for s in results.values() if s) + print(f"\nTotal: {successful}/{len(languages)} languages completed successfully") + print("=" * 80 + "\n") + + return all(results.values()) + + def run_fix_doc_titles_workflow( + self, + mode: str = "all", + count: int = 20, + node_name: str = None, + lang: str = None, + dry_run: bool = False, + sync_frontend: bool = True, + hash_mode: str = "preserve", + ) -> bool: + """Fix H1 titles in existing docs (missing / duplicate / frontend mismatch). No AI.""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + print("\n" + "=" * 80) + print("ComfyUI Documentation Fix — Document Titles") + print("=" * 80) + print(f"Started at: {timestamp}") + print(f"Scope: {mode}" + (f" (first {count} files)" if mode == "test" else "")) + print(f"Dry run: {dry_run}") + print(f"Hash mode: {hash_mode}") + print(f"Sync frontend first: {sync_frontend}") + if node_name: + print(f"Node: {node_name}") + if lang: + print(f"Language: {lang}") + print("=" * 80) + + if sync_frontend: + print("\n🔄 Step 1: Syncing frontend translations...") + if not self.sync_frontend_translations(): + print("\n⚠️ Frontend sync failed; continuing with existing node_translations.json") + + args = ["--mode", mode] + if mode == "test": + args.extend(["--count", str(count)]) + if node_name: + args.extend(["--node", node_name]) + if lang: + args.extend(["--lang", lang]) + if dry_run: + args.append("--dry-run") + args.extend(["--hash-mode", hash_mode]) + + ok = self.run_command( + self.fix_doc_titles_script, + args, + "Fix document titles (frontend display_name)", + ) + + end_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print("\n" + "=" * 80) + if ok: + print("✅ Fix Titles Workflow Completed!") + else: + print("❌ Fix Titles Workflow Failed") + print("=" * 80) + print(f"Started: {timestamp}") + print(f"Finished: {end_timestamp}") + print("=" * 80 + "\n") + return ok + + def _load_changed_nodes_from_scan(self) -> list[str]: + """Load the list of changed nodes from the latest scan report.""" + import json + scan_report = self.repo_root / "data" / "missing_nodes_report.json" + if not scan_report.exists(): + print(" ⚠️ No scan report found (missing_nodes_report.json)") + return [] + try: + with open(scan_report, "r", encoding="utf-8") as f: + report = json.load(f) + changed = report.get("changed_nodes", []) + print(f" 📋 Found {len(changed)} changed nodes in scan report") + return changed + except (json.JSONDecodeError, OSError) as e: + print(f" ⚠️ Could not read scan report: {e}") + return [] + + def run_changed_workflow(self, force: bool = False): + """Run workflow for nodes with changed source code""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + print("\n" + "=" * 80) + print("ComfyUI Documentation Automation - Changed Nodes Workflow") + print("=" * 80) + print(f"Started at: {timestamp}") + print(f"Force regenerate: {force}") + print("=" * 80) + + # Step 1: Scan to detect changed nodes + print("\n📊 Step 1: Scanning to identify changed nodes...") + if not self.scan_nodes(): + print("\n❌ Workflow failed at Step 1: Scan") + return False + + # Step 2: Prepare AI input for changed nodes + print("\n🔧 Step 2: Preparing AI input for changed nodes...") + if not self.run_command( + self.prepare_script, + ["changed"], + "Step 2: Preparing changed nodes" + ): + print("\n❌ Workflow failed at Step 2: Prepare") + return False + + # Step 3: Generate documentation (force=True so existing docs get overwritten) + # ⚠️ Save changed nodes BEFORE generation because batch_generate_docs.py + # re-runs scan_missing_nodes.py in _update_reports(), which overwrites + # the changed_nodes field in the scan report to empty. + changed_nodes = self._load_changed_nodes_from_scan() + print("\n🤖 Step 3: Generating documentation for changed nodes...") + if not self.generate_docs("changed", force=True): + print("\n❌ Workflow failed at Step 3: Generate") + return False + + # Step 4: Re-translate changed nodes for all languages + # Use the saved list from before generation (Step 3's batch_generate_docs + # re-runs scan and overwrites the changed_nodes field in the report) + if changed_nodes: + print(f"\n🌐 Step 4: Re-translating {len(changed_nodes)} changed nodes for all languages...") + node_list_str = ",".join(n["name"] if isinstance(n, dict) else n for n in changed_nodes) + languages = ['zh', 'zh-TW', 'es', 'fr', 'ja', 'ko', 'ru', 'ar', 'tr', 'pt-BR', 'fa'] + + # Sync frontend translations first + print("\n🔄 Syncing frontend translations...") + self.sync_frontend_translations() + + for lang in languages: + print(f"\n{'=' * 60}") + print(f"🌐 Translating changed nodes to {lang}...") + print(f"{'=' * 60}") + self.run_command( + self.translate_script, + ["--lang", lang, "--node-list", node_list_str, "--force"], + f"Re-translating changed nodes to {lang}" + ) + print(f"\n✅ Step 4 complete: {len(changed_nodes)} changed nodes re-translated across {len(languages)} languages.") + else: + print("\n⏭️ Step 4: No changed nodes to re-translate (skipping).") + + # Step 5: Final scan to update reports + print("\n🔄 Step 5: Final scan to update all reports...") + if not self.scan_nodes(): + print("\n❌ Workflow failed at Step 5: Final Update") + return False + + end_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print("\n" + "=" * 80) + print("✅ Changed Nodes Workflow Completed!") + print("=" * 80) + print(f"Started: {timestamp}") + print(f"Finished: {end_timestamp}") + print("=" * 80 + "\n") + + return True + + def run_regenerate_all_workflow(self, prepare_limit=None, translate_all_languages: bool = False) -> bool: + """Scan → prepare AI input for every node from all_nodes_info.json → regenerate all English docs with --force. + + Use ``prepare_limit`` for a capped dry run (first N nodes by name). + + If ``translate_all_languages`` is True, runs an all-languages translation pass after English regeneration + (``mode=all``, ``force=True``, ``force_all_nodes=True`` so every ``en.md`` is re-translated). + + Intended for extractor / pipeline changes or policy updates that require rewriting every ``en.md``, + not for day-to-day use (heavy on disk, API quota, and time). + """ + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + print("\n" + "=" * 80) + print("ComfyUI Documentation Automation — FULL REGENERATE (all nodes)") + print("=" * 80) + print(f"Started at: {timestamp}") + lim_msg = str(prepare_limit) if prepare_limit is not None else "none (full tree)" + print(f"Prepare limit: {lim_msg}") + print(f"Follow with all-language translation: {'yes' if translate_all_languages else 'no'}") + print("=" * 80) + + print("\n📊 Step 1: Scan (refresh all_nodes_info + reports)...") + if not self.scan_nodes(): + print("\n❌ Failed at Step 1: Scan") + return False + + prep_args = ["regenerate-all"] + if prepare_limit is not None: + prep_args.append(str(prepare_limit)) + + print("\n🔧 Step 2: Prepare AI input for ALL scanned nodes (may take long)...") + if not self.run_command( + self.prepare_script, + prep_args, + "Step 2: prepare_ai_input.py regenerate-all", + ): + print("\n❌ Failed at Step 2: Prepare") + return False + + print("\n🤖 Step 3: Regenerate ALL English docs (batch_generate_docs all --force)...") + if not self.generate_docs("all", force=True): + print("\n❌ Failed at Step 3: Generate") + return False + + print("\n🔄 Step 4: Final scan...") + if not self.scan_nodes(): + print("\n❌ Failed at Step 4: Final scan") + return False + + if translate_all_languages: + print("\n🌐 Step 5: Translate ALL languages (mode=all, force=True, force-all-nodes; long + many API calls)...") + if not self.run_all_languages_translation(mode="all", count=20, force=True, force_all_nodes=True): + print("\n❌ Failed at Step 5: All-languages translation") + return False + print("\n🔄 Step 6: Final scan after translations...") + if not self.scan_nodes(): + print("\n❌ Failed at Step 6: Final scan") + return False + + end = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print("\n" + "=" * 80) + print("✅ Full regenerate workflow finished.") + print("=" * 80) + print(f"Started: {timestamp}") + print(f"Finished: {end}") + if not translate_all_languages: + print("(Translations unchanged. Use --also-translate-all with regenerate-all, or run translate separately with --force.)") + print("=" * 80 + "\n") + return True + + def run_full_workflow(self, mode: str = "test", count: int = 20, force: bool = False): + """Run the complete workflow""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + print("\n" + "=" * 80) + print("ComfyUI Documentation Automation - Main Workflow") + print("=" * 80) + print(f"Started at: {timestamp}") + print(f"Mode: {mode}") + if mode == "test": + print(f"Count: {count} nodes") + print(f"Force regenerate: {force}") + print("=" * 80) + + # Step 1: Always scan first to get latest missing nodes + print("\n📊 Step 1: Scanning to identify missing documentation...") + if not self.scan_nodes(): + print("\n❌ Workflow failed at Step 1: Scan") + return False + + # Step 2: Prepare AI input (will read from fresh missing_nodes_report.json) + print("\n🔧 Step 2: Preparing AI input for missing nodes...") + if not self.prepare_nodes(mode, count): + print("\n❌ Workflow failed at Step 2: Prepare") + return False + + # Step 3: Generate documentation (only for newly prepared nodes) + print("\n🤖 Step 3: Generating documentation with AI...") + if not self.generate_docs(mode, count, force=force): + print("\n❌ Workflow failed at Step 3: Generate") + return False + + # Step 4: Final scan to update reports with new status + print("\n🔄 Step 4: Final scan to update all reports...") + if not self.scan_nodes(): + print("\n❌ Workflow failed at Step 4: Final Update") + return False + + # Success summary + end_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print("\n" + "=" * 80) + print("✅ Workflow Completed Successfully!") + print("=" * 80) + print(f"Started: {timestamp}") + print(f"Finished: {end_timestamp}") + print("\nAll reports are up to date:") + print(" - missing_nodes_report.json") + print(" - node_versions.json") + print("=" * 80 + "\n") + + return True + + +def _prompt(text: str, default: str = None) -> str: + """Prompt for input; return default if user presses Enter and default is set.""" + if default is not None: + prompt = f"{text} [{default}]: " + else: + prompt = f"{text}: " + value = input(prompt).strip() + return value if value else (default or "") + + +def _prompt_int(text: str, default: int = None) -> int: + """Prompt for integer; retry until valid.""" + while True: + raw = _prompt(text, str(default) if default is not None else None) + if not raw and default is not None: + return default + try: + return int(raw) + except ValueError: + print(" Please enter a number.") + + +def _prompt_yes_no(text: str, default: bool = False) -> bool: + """Prompt for y/n; default when Enter with no input.""" + d = "Y" if default else "n" + while True: + raw = _prompt(f"{text} (y/n)", d).strip().lower() or d.lower() + if raw in ("y", "yes"): + return True + if raw in ("n", "no"): + return False + print(" Enter y or n.") + + +def run_interactive(workflow: DocumentationWorkflow) -> bool: + """Run interactive menu-driven workflow.""" + print("\n" + "=" * 60) + print(" ComfyUI 文档自动化 - 交互式菜单") + print(" Documentation Automation - Interactive Menu") + print("=" * 60) + + while True: + print("\n请选择操作 / Choose action:") + print(" 1) 仅扫描 (Scan only)") + print(" 2) 生成英文文档 (Generate English docs)") + print(" 3) 翻译（子菜单：单语 / 全部语言；全部语言里可选「强制全量重译」）") + print(" (Translate: one lang / all langs; submenu includes force-retranslate-all)") + print(" 4) 生成缺失文档并全部翻译 (Generate missing + translate all)") + print(" 5) 同步到 Comfy 文档 (Sync to Comfy docs)") + print(" 6) 更新变更节点文档 (Regenerate docs for changed nodes)") + print(" 7) 全量重跑英文（可选随后全语言翻译）(FULL en.md; optional all-lang translate)") + print(" 8) 强制全语言重译全部节点（每个 en.md → 11 语覆盖；等同 CLI --retranslate-all-languages）") + print(" (Force-retranslate ALL langs for EVERY node with en.md; API-heavy)") + print(" 9) 修复已有文档 (Fix existing docs — no AI)") + print(" 0) 退出 (Exit)") + choice = _prompt("选项 / Choice", "0").strip() + + if choice == "0": + print("Bye.") + return True + + if choice == "1": + ok = workflow.scan_nodes() + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + if choice == "2": + print("\n--- 生成模式 ---") + print(" 1) test - 生成指定数量的缺失节点 (default 20)") + print(" 2) all - 生成所有缺失节点") + print(" 3) node - 仅生成单个节点") + sub = _prompt("模式 (1/2/3)", "1").strip() + if sub == "2": + mode = "all" + count = 20 + node_name = None + elif sub == "3": + mode = "node" + node_name = _prompt("节点名称 (Node name)").strip() + if not node_name: + print(" 未输入节点名，已取消。") + continue + count = None + else: + mode = "test" + count = _prompt_int("生成数量 (Count)", 20) + node_name = None + force = _prompt_yes_no("是否覆盖已有文档 (Force overwrite)?", False) + print() + if mode == "node": + ok = ( + workflow.scan_nodes() + and workflow.prepare_nodes("node", node_name=node_name) + and workflow.generate_docs("node", node_name=node_name, force=force) + and workflow.update_reports() + ) + else: + ok = workflow.run_full_workflow(mode=mode, count=count, force=force) + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + if choice == "3": + print("\n--- 翻译 ---") + print(" 1) 单语言 (One language)") + print(" 2) 全部语言 (All languages)") + tr_choice = _prompt("1 或 2", "1").strip() + if tr_choice == "2": + print(" 1) 全部缺失 (all) - 按报告仅翻译当前缺失（每种语言整批缺失）") + print(" 2) 指定数量 (test) - 每种语言只翻译缺失队列前 N 条") + print(" 3) 强制全量重译 - 每个有 en.md 的节点全部语种覆盖（忽略缺失报告；CLI: --retranslate-all-languages）") + all_or_count = _prompt("1 / 2 / 3", "1").strip() + if all_or_count == "3": + return workflow.run_all_languages_translation(mode="all", count=20, force=True, force_all_nodes=True) + if all_or_count == "2": + count = _prompt_int("每种语言处理数量 (Count per language)", 20) + force = _prompt_yes_no("是否覆盖已有翻译 (Force overwrite)?", False) + return workflow.run_all_languages_translation(mode="test", count=count, force=force) + force = _prompt_yes_no("是否覆盖已有翻译 (Force overwrite)?", False) + return workflow.run_all_languages_translation(mode="all", count=20, force=force) + print("\n可选语言:") + for i, lang in enumerate(LANGUAGES, 1): + print(f" {i:2}) {lang} {LANG_NAMES.get(lang, '')}") + lang_idx = _prompt_int("语言编号 (1-11)", 1) + if not (1 <= lang_idx <= len(LANGUAGES)): + print(" 无效编号。") + continue + lang = LANGUAGES[lang_idx - 1] + print("\n 1) test - 翻译指定数量 (默认 20)") + print(" 2) all - 翻译全部缺失") + tm = _prompt("模式 (1/2)", "1").strip() + mode = "all" if tm == "2" else "test" + count = _prompt_int("数量 (test 时)", 20) if mode == "test" else 20 + force = _prompt_yes_no("是否覆盖已有翻译 (Force overwrite)?", False) + print() + ok = workflow.run_translation_workflow(lang=lang, mode=mode, count=count, force=force) + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + if choice == "4": + print("\n--- 生成缺失文档并全部翻译（一次性跑完，中间不再确认）---") + print(" 1) test - 先生成指定数量的缺失英文文档，再对全部语言翻译同样数量") + print(" 2) all - 先生成所有缺失英文文档，再对全部语言翻译所有缺失（推荐，一次性完成）") + sub = _prompt("模式 (1/2)", "2").strip() + if sub == "2": + gen_mode, gen_count = "all", 20 + tr_mode, tr_count = "all", 10 + else: + gen_mode = "test" + gen_count = _prompt_int("生成数量 (Count)", 20) + tr_mode = "test" + tr_count = gen_count + force_gen = _prompt_yes_no("是否覆盖已有英文文档 (Force overwrite)?", False) + force_tr = _prompt_yes_no("是否覆盖已有翻译 (Force overwrite)?", False) + print("\n将一次性执行：先生成英文文档 → 再全部语言翻译，中间不再询问。") + print("[Step 1/2] 生成英文文档...") + if not workflow.run_full_workflow(mode=gen_mode, count=gen_count, force=force_gen): + print(" 生成失败，已取消。") + if _prompt_yes_no("继续操作? (Continue?)", False): + continue + return False + print("\n[Step 2/2] 全部语言翻译（自动连续执行）...") + ok = workflow.run_all_languages_translation(mode=tr_mode, count=tr_count, force=force_tr) + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + if choice == "5": + print("\n--- 同步到 Comfy 文档 ---") + print(" 将 embedded-docs 的 en.md/zh.md 与图片同步到 comfy/docs (built-in-nodes)。") + print(" 1) test - 同步前 N 个节点 (默认 10)") + print(" 2) all - 同步所有有 en.md 的节点") + sub = _prompt("模式 (1/2)", "1").strip() + mode = "all" if sub == "2" else "test" + count = _prompt_int("数量 (test 时)", 10) if mode == "test" else 10 + dry = _prompt_yes_no("仅预览不写入 (Dry run)?", False) + no_json = _prompt_yes_no("不更新 docs.json (No docs.json)?", False) + args = ["--mode", mode] + if mode == "test": + args.extend(["--count", str(count)]) + if dry: + args.append("--dry-run") + if no_json: + args.append("--no-docs-json") + print() + ok = workflow.run_command( + workflow.sync_to_comfy_docs_script, + args, + "Sync to Comfy docs (built-in-nodes + docs.json)" + ) + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + if choice == "6": + print("\n--- 更新变更节点文档 ---") + print(" 扫描源码变更 → 重新生成有变动节点的英文文档。") + force = _prompt_yes_no("是否强制覆盖已有文档 (Force overwrite)?", True) + print() + ok = workflow.run_changed_workflow(force=force) + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + if choice == "7": + print("\n--- 全量英文文档重生 ---") + print(" 会：扫描 → 对所有已扫描节点跑 prepare_ai_input → batch_generate_docs all --force。") + print(" ⚠️ 耗时长；会重写每个节点的 en.md 并占用大量 API。") + print(" 默认会在英文完成后继续全语言翻译；若只在交互里改了英文不想动翻译，选 n（或 CLI 仅用 --mode regenerate-all 不加翻译）。") + also_tr = _prompt_yes_no( + "英文完成后是否继续「全语言翻译」（mode=all + 强制覆盖翻译）? (Also translate all langs?)", + True, + ) + if not _prompt_yes_no("确认继续？", False): + print(" 已取消。") + continue + lim_raw = _prompt("仅先做前 N 个节点（调试，留空=全部）Prepare limit / Enter for all").strip() + prepare_limit = int(lim_raw) if lim_raw else None + if prepare_limit is not None and prepare_limit <= 0: + print(" 无效数量。") + continue + print() + ok = workflow.run_regenerate_all_workflow( + prepare_limit=prepare_limit, + translate_all_languages=also_tr, + ) + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + if choice == "8": + print("\n--- 强制全语言重译全部节点 ---") + print(" 会对「每个已有 en.md 的节点」在全部 11 种语言上覆盖写入翻译（忽略缺失报告）。") + print(" ⚠️ API 与时间消耗极大；等同于: python3 main.py --retranslate-all-languages") + if not _prompt_yes_no("确认执行？", False): + print(" 已取消。") + continue + print() + ok = workflow.run_all_languages_translation( + mode="all", count=20, force=True, force_all_nodes=True + ) + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + if choice == "9": + print("\n--- 修复已有文档 (Fix) ---") + print(" 1) 文档标题 — 缺失 / 重复 / 与前端 display_name 不一致") + print(" (Doc titles from frontend nodeDefs; no AI)") + sub = _prompt("选项 (1)", "1").strip() + if sub != "1": + print(" 暂仅支持 1) 文档标题。") + continue + print("\n Hash 处理 / SHA footer:") + print(" 1) preserve - 保留原 disclaimer + SHA（推荐，翻译已对齐时）") + print(" 2) update - 从 en.md / ai_input 重新写入 SHA（并重写 disclaimer）") + hash_choice = _prompt("Hash (1/2)", "1").strip() + hash_mode = "update" if hash_choice == "2" else "preserve" + print("\n 1) test - 扫描前 N 个文件 (默认 20)") + print(" 2) all - 扫描全部已有 .md") + scope = _prompt("范围 (1/2)", "2").strip() + fix_mode = "all" if scope == "2" else "test" + fix_count = _prompt_int("文件数量 (test 时)", 20) if fix_mode == "test" else 20 + dry_run = _prompt_yes_no("仅预览不写入 (Dry run)?", True) + sync_fe = _prompt_yes_no("先同步前端 nodeDefs 翻译? (Sync frontend)", True) + node_name = _prompt("仅单个节点 (留空=全部) Node name").strip() or None + lang_raw = _prompt("仅单语言代码 en/zh/... (留空=全部) Lang").strip() or None + if lang_raw and lang_raw not in (["en"] + LANGUAGES): + print(f" 无效语言: {lang_raw}") + continue + print() + ok = workflow.run_fix_doc_titles_workflow( + mode=fix_mode, + count=fix_count, + node_name=node_name, + lang=lang_raw, + dry_run=dry_run, + sync_frontend=sync_fe, + hash_mode=hash_mode, + ) + if ok and _prompt_yes_no("继续操作? (Continue?)", False): + continue + return ok + + print(" 请输入 0–9。") + + +def main(): + """Main entry point""" + import argparse + + parser = argparse.ArgumentParser( + description='ComfyUI Documentation Automation - Main Controller', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +Examples: + # Generate 20 nodes (default) + python3 main.py + + # Generate 50 nodes + python3 main.py --count 50 + + # Generate all missing nodes + python3 main.py --mode all + + # Full refresh: rebuild ai_input for every scanned node → regenerate ALL en.md (API-heavy) + python3 main.py --mode regenerate-all + python3 main.py --mode regenerate-all --prepare-limit 50 + python3 main.py --mode regenerate-all --also-translate-all + + # Generate single node (new doc) + python3 main.py --mode node --node AudioEncoderEncode + + # Update an existing node doc based on latest source code + # (sends current en.md as reference so the AI keeps manual edits) + python3 main.py --mode node --node TrainLoraNode --force + + # Force regenerate existing docs (batch) + python3 main.py --count 10 --force + + # Only scan (no generation) + python3 main.py --scan-only + + # Translation workflow + python3 main.py --translate --lang zh --count 10 + python3 main.py --translate --lang zh --mode all + python3 main.py --translate --lang es --count 20 --force + + # Translate all languages at once (zh, zh-TW, es, fr, ja, ko, ru, ar, tr, pt-BR, fa) + # This will automatically sync frontend translations and update parameter names + python3 main.py --translate --all-languages --count 10 + python3 main.py --translate --all-languages --mode all + python3 main.py --retranslate-all-languages + + # Fix existing doc titles (no AI) + python3 main.py --mode fix --fix-action doc-titles --dry-run + python3 main.py --mode fix --fix-action doc-titles + ''' + ) + + parser.add_argument( + '--mode', + choices=['test', 'all', 'resume', 'node', 'changed', 'regenerate-all', 'fix'], + default='test', + help=( + 'Generation mode (default: test). ' + '"node" generates or updates a single node (requires --node). ' + 'Pair with --force to update an existing doc: re-reads source code and ' + 'sends the current en.md as reference so the AI updates params/outputs ' + 'while preserving manual edits. ' + 'Use "changed" to regenerate docs for nodes with updated source code. ' + '"regenerate-all" scans then prepares EVERY node from all_nodes_info.json and runs ' + 'batch_generate_docs all --force (see --prepare-limit). Add --also-translate-all to ' + 'retranslate every locale for every node with en.md afterwards (mode=all, force, force-all-nodes). ' + 'Use "fix" with --fix-action doc-titles to repair H1 titles in existing docs (no AI).' + ) + ) + + parser.add_argument( + '--fix-action', + choices=['doc-titles'], + default=None, + help='With --mode fix: which repair to run (default: doc-titles)', + ) + + parser.add_argument( + '--fix-scope', + choices=['test', 'all'], + default='all', + help='Only with --mode fix: scan first N files (test) or all docs (all, default)', + ) + + parser.add_argument( + '--dry-run', + action='store_true', + help='With --mode fix: preview title fixes without writing files', + ) + + parser.add_argument( + '--hash-mode', + choices=['preserve', 'update'], + default='preserve', + help=( + 'With --mode fix: preserve original disclaimer+SHA (default) or ' + 'update SHA from en.md/ai_input and rewrite disclaimer' + ), + ) + + parser.add_argument( + '--prepare-limit', + type=int, + default=None, + metavar='N', + help=( + 'Only with --mode regenerate-all: prepare and regenerate English docs only for ' + 'the first N nodes (sorted by name). Omit for entire tree.' + ), + ) + + parser.add_argument( + '--also-translate-all', + action='store_true', + help=( + 'Only with --mode regenerate-all: after regenerating all English docs, run ' + 'all-languages translation (mode=all, force, force-all-nodes).' + ), + ) + + parser.add_argument( + '--count', + type=int, + default=20, + help='Number of nodes to generate/translate in test mode (default: 20)' + ) + + parser.add_argument( + '--node', + type=str, + help='Node name for single node mode' + ) + + parser.add_argument( + '--force', + action='store_true', + help='Force regenerate/retranslate existing documentation' + ) + + parser.add_argument( + '--scan-only', + action='store_true', + help='Only run scan, skip generation/translation' + ) + + parser.add_argument( + '--translate', + action='store_true', + help='Run translation workflow instead of generation workflow' + ) + + parser.add_argument( + '--lang', + type=str, + choices=['zh', 'zh-TW', 'es', 'fr', 'ja', 'ko', 'ru', 'ar', 'tr', 'pt-BR', 'fa'], + help='Target language for translation (required with --translate)' + ) + + parser.add_argument( + '--all-languages', + action='store_true', + help='Translate to all supported languages (zh, zh-TW, es, fr, ja, ko, ru, ar, tr, pt-BR, fa)' + ) + + parser.add_argument( + '--force-all-translation-nodes', + action='store_true', + help=( + 'Only with --translate: include every folder under docs that has en.md in prepare_translation ' + '(sorted by name), not only nodes listed as missing. Use with --force so translations are overwritten.' + ), + ) + + parser.add_argument( + '--retranslate-all-languages', + action='store_true', + help=( + 'Force-retranslate all supported languages for every node that has en.md. Equivalent to ' + '--translate --all-languages --mode all --force --force-all-translation-nodes.' + ), + ) + + parser.add_argument( + '--interactive', '-i', + action='store_true', + help='Show interactive menu (default when no other args)' + ) + + args = parser.parse_args() + + if args.retranslate_all_languages: + args.translate = True + args.all_languages = True + args.force = True + args.mode = 'all' + args.force_all_translation_nodes = True + + if getattr(args, 'force_all_translation_nodes', False) and not args.translate: + print("⚠️ Note: --force-all-translation-nodes only applies with --translate; ignoring.") + + if args.prepare_limit is not None and args.mode != "regenerate-all": + print("⚠️ Note: --prepare-limit only applies with --mode regenerate-all; ignoring this flag.") + + if args.also_translate_all and args.mode != "regenerate-all": + print("⚠️ Note: --also-translate-all only applies with --mode regenerate-all; ignoring this flag.") + + if args.dry_run and args.mode != "fix": + print("⚠️ Note: --dry-run only applies with --mode fix; ignoring this flag.") + + if args.hash_mode != "preserve" and args.mode != "fix": + print("⚠️ Note: --hash-mode only applies with --mode fix; ignoring this flag.") + + if args.fix_action and args.mode != "fix": + print("⚠️ Note: --fix-action only applies with --mode fix; ignoring this flag.") + + # No args or --interactive: run interactive menu + if args.interactive or len(sys.argv) == 1: + workflow = DocumentationWorkflow() + success = run_interactive(workflow) + sys.exit(0 if success else 1) + + # Validate arguments + if args.mode == 'node' and not args.node: + print("❌ Error: --node is required when using --mode node") + parser.print_help() + sys.exit(1) + + if args.translate and not args.lang and not args.all_languages: + print("❌ Error: --lang or --all-languages is required when using --translate") + print("Available languages: zh, zh-TW, es, fr, ja, ko, ru, ar, tr, pt-BR, fa") + parser.print_help() + sys.exit(1) + + # Create workflow controller + workflow = DocumentationWorkflow() + + # Run scan-only mode + if args.scan_only: + success = workflow.scan_nodes() + sys.exit(0 if success else 1) + + # Run fix workflow (no AI) + if args.mode == 'fix': + fix_action = args.fix_action or 'doc-titles' + if fix_action != 'doc-titles': + print(f"❌ Error: unknown --fix-action {fix_action!r}") + sys.exit(1) + success = workflow.run_fix_doc_titles_workflow( + mode=args.fix_scope, + count=args.count, + node_name=args.node, + lang=args.lang, + dry_run=args.dry_run, + hash_mode=args.hash_mode, + ) + sys.exit(0 if success else 1) + + # Run translation workflow + if args.translate: + if args.mode == "regenerate-all": + print("❌ Error: do not combine --translate with --mode regenerate-all. Use --also-translate-all instead.") + sys.exit(1) + if args.also_translate_all: + print("❌ Error: do not combine --translate with --also-translate-all.") + sys.exit(1) + if args.all_languages: + # Translate all languages + success = workflow.run_all_languages_translation( + mode=args.mode if args.mode != 'node' else 'test', + count=args.count, + force=args.force, + force_all_nodes=args.force_all_translation_nodes, + ) + else: + # Translate single language + success = workflow.run_translation_workflow( + lang=args.lang, + mode=args.mode if args.mode != 'node' else 'test', + count=args.count, + force=args.force, + force_all_nodes=args.force_all_translation_nodes, + ) + sys.exit(0 if success else 1) + + # Run generation workflow + if args.mode == 'node': + # Single node workflow + success = ( + workflow.scan_nodes() and + workflow.prepare_nodes('node', node_name=args.node) and + workflow.generate_docs('node', node_name=args.node, force=args.force) and + workflow.update_reports() + ) + elif args.mode == 'changed': + # Changed nodes workflow + success = workflow.run_changed_workflow(force=args.force) + elif args.mode == 'regenerate-all': + success = workflow.run_regenerate_all_workflow( + prepare_limit=args.prepare_limit, + translate_all_languages=args.also_translate_all, + ) + else: + # Batch workflow + success = workflow.run_full_workflow( + mode=args.mode, + count=args.count, + force=args.force + ) + + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() + diff --git a/pipeline/requirements.txt b/pipeline/requirements.txt new file mode 100644 index 000000000..fb9998920 --- /dev/null +++ b/pipeline/requirements.txt @@ -0,0 +1,3 @@ +openai>=1.0.0 +python-dotenv>=1.0.0 + diff --git a/pipeline/scripts/batch_generate_docs.py b/pipeline/scripts/batch_generate_docs.py new file mode 100644 index 000000000..9a5f7f5ac --- /dev/null +++ b/pipeline/scripts/batch_generate_docs.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +""" +Batch generate node documentation using AI API +""" + +import os +import sys +import json +import time +from pathlib import Path +from typing import Dict, List, Optional +from datetime import datetime +from openai import OpenAI + +import runtime # noqa: F401 +from lib.doc_disclaimer import compose_document, create_en_disclaimer, strip_ai_disclaimer +from lib.doc_title import ensure_doc_title, strip_leading_h1 +from lib.hash_footer import ( + format_source_hash_footer, + load_node_source_sha256, + strip_source_hash_footer, +) +from lib.paths import ( + AI_INPUT_DIR, + LOGS_DIR, + REPO_ROOT, + SCRIPTS_DIR, + default_embedded_docs_path, + load_dotenv, +) + +load_dotenv() + +# Configuration +API_KEY = os.getenv('DEEPSEEK_API_KEY') +API_BASE_URL = os.getenv('API_BASE_URL', 'https://api.deepseek.com') +API_MODEL = os.getenv('API_MODEL', 'deepseek-chat') +BATCH_SIZE = int(os.getenv('BATCH_SIZE', '5')) +MAX_RETRIES = int(os.getenv('MAX_RETRIES', '3')) +DELAY_BETWEEN_REQUESTS = int(os.getenv('DELAY_BETWEEN_REQUESTS', '2')) + +# Path configuration +AI_INPUT_PATH = AI_INPUT_DIR +DOCS_OUTPUT_PATH = default_embedded_docs_path() / "comfyui_embedded_docs" / "docs" +LOG_PATH = LOGS_DIR + +# Create necessary directories +LOG_PATH.mkdir(parents=True, exist_ok=True) + + +class DocGenerationLogger: + """Documentation generation logger""" + + def __init__(self): + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + self.log_file = LOG_PATH / f"generation_{timestamp}.log" + self.success_count = 0 + self.failed_count = 0 + self.skipped_count = 0 + self.failed_nodes = [] + + def log(self, message: str, level: str = "INFO"): + """Log a message""" + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + log_entry = f"[{timestamp}] [{level}] {message}" + print(log_entry) + + with open(self.log_file, 'a', encoding='utf-8') as f: + f.write(log_entry + '\n') + + def log_success(self, node_name: str): + """Log a successful generation""" + self.success_count += 1 + self.log(f"✅ Successfully generated: {node_name}", "SUCCESS") + + def log_failure(self, node_name: str, error: str): + """Log a failed generation""" + self.failed_count += 1 + self.failed_nodes.append({'node': node_name, 'error': error}) + self.log(f"❌ Generation failed: {node_name} - {error}", "ERROR") + + def log_skip(self, node_name: str, reason: str): + """Log a skipped node""" + self.skipped_count += 1 + self.log(f"⏭️ Skipped: {node_name} - {reason}", "SKIP") + + def summary(self): + """Output generation summary""" + self.log("=" * 80) + self.log("📊 Generation Summary") + self.log("=" * 80) + self.log(f"✅ Success: {self.success_count}") + self.log(f"❌ Failed: {self.failed_count}") + self.log(f"⏭️ Skipped: {self.skipped_count}") + self.log(f"📁 Log file: {self.log_file}") + + if self.failed_nodes: + self.log("\nFailed nodes:") + for item in self.failed_nodes: + self.log(f" - {item['node']}: {item['error']}") + + +class AIDocGenerator: + """AI documentation generator""" + + def __init__(self): + if not API_KEY: + raise ValueError("❌ DEEPSEEK_API_KEY not found, please configure it in .env file") + + self.client = OpenAI( + api_key=API_KEY, + base_url=API_BASE_URL + ) + self.logger = DocGenerationLogger() + + def generate_doc(self, node_name: str, prompt: str, retry_count: int = 0) -> Optional[str]: + """Generate a single document using AI""" + try: + self.logger.log(f"🤖 Generating documentation: {node_name}") + + response = self.client.chat.completions.create( + model=API_MODEL, + messages=[ + {"role": "system", "content": "You are a technical documentation expert specializing in ComfyUI nodes. Generate clear, accurate, and factual documentation based on source code."}, + {"role": "user", "content": prompt} + ], + stream=False, + temperature=0.3 # Lower temperature for more consistent output + ) + + content = response.choices[0].message.content + + # Validate generated content + if not content or len(content) < 100: + raise ValueError("Generated documentation too short") + + # Check for required sections (updated to match new structure) + if "## Inputs" not in content or "## Outputs" not in content: + raise ValueError("Missing required documentation sections (Inputs/Outputs)") + + return content + + except Exception as e: + if retry_count < MAX_RETRIES: + self.logger.log(f"⚠️ Retrying ({retry_count + 1}/{MAX_RETRIES}): {node_name}", "RETRY") + time.sleep(DELAY_BETWEEN_REQUESTS * 2) + return self.generate_doc(node_name, prompt, retry_count + 1) + else: + raise e + + def save_doc(self, node_name: str, content: str) -> bool: + """Save generated documentation with disclaimer at the bottom of the file.""" + try: + # Create node documentation directory + doc_dir = DOCS_OUTPUT_PATH / node_name + doc_dir.mkdir(parents=True, exist_ok=True) + + disclaimer = create_en_disclaimer(node_name) + + src_hash = load_node_source_sha256(node_name) + if src_hash: + footer = format_source_hash_footer(src_hash) + else: + footer = "" + self.logger.log( + f"⚠️ No fingerprint for {node_name}: missing ai_input basic_info/source — en.md saved without hash footer", + "SKIP", + ) + + body = ensure_doc_title(strip_leading_h1(content), node_name, "en") + final_content = compose_document(body, disclaimer, footer) + + # Save English documentation + doc_file = doc_dir / "en.md" + with open(doc_file, 'w', encoding='utf-8') as f: + f.write(final_content) + + self.logger.log(f"💾 Saved: {doc_file}") + return True + + except Exception as e: + self.logger.log(f"❌ Save failed: {node_name} - {e}", "ERROR") + return False + + def build_update_prompt(self, base_prompt: str, existing_doc: str) -> str: + """Wrap the base prompt with existing doc context for incremental update.""" + return ( + base_prompt + + "\n\n" + + "## Existing Documentation (for reference)\n\n" + + "The following is the **current en.md** for this node. " + + "It may contain manual edits, extra context, or sections that go beyond what the source code alone implies. " + + "**Preserve all such content unless the updated source code directly contradicts it.**\n\n" + + "Your job is to produce an updated en.md that:\n" + + "1. Reflects any parameter additions, removals, or type changes visible in the new source code above.\n" + + "2. Keeps all human-written descriptions, notes, and extra sections that are still accurate.\n" + + "3. Does NOT discard or rewrite content just because it was not derived from the source code — " + + "only remove content that is factually incorrect given the new source.\n\n" + + "```markdown\n" + + existing_doc + + "\n```\n" + ) + + def process_node(self, node_name: str, force: bool = False) -> bool: + """Process a single node""" + doc_file = DOCS_OUTPUT_PATH / node_name / "en.md" + if doc_file.exists() and not force: + self.logger.log_skip(node_name, "Documentation already exists") + return True + + # Read AI prompt + prompt_file = AI_INPUT_PATH / node_name / "ai_prompt.txt" + if not prompt_file.exists(): + self.logger.log_skip(node_name, "AI prompt file not found") + return False + + try: + with open(prompt_file, 'r', encoding='utf-8') as f: + prompt = f.read() + + # If an existing doc is present and we are force-regenerating, send it + # to the AI as context so human edits are preserved. + if force and doc_file.exists(): + with open(doc_file, 'r', encoding='utf-8') as f: + existing_doc = strip_ai_disclaimer(strip_source_hash_footer(f.read())) + self.logger.log(f"📄 Existing doc found — using update prompt to preserve edits: {node_name}") + prompt = self.build_update_prompt(prompt, existing_doc) + + content = self.generate_doc(node_name, prompt) + + if content: + if self.save_doc(node_name, content): + self.logger.log_success(node_name) + return True + + return False + + except Exception as e: + self.logger.log_failure(node_name, str(e)) + return False + + def batch_process(self, node_names: List[str], force: bool = False): + """Batch process nodes""" + total = len(node_names) + self.logger.log("=" * 80) + self.logger.log(f"🚀 Starting batch documentation generation") + self.logger.log(f"📊 Total: {total} nodes") + self.logger.log(f"🔧 API: {API_MODEL}") + self.logger.log(f"⚙️ Batch size: {BATCH_SIZE}") + self.logger.log("=" * 80) + self.logger.log("") + + consecutive_failures = 0 + MAX_CONSECUTIVE_FAILURES = 5 + + for idx, node_name in enumerate(node_names, 1): + self.logger.log(f"\n[{idx}/{total}] Processing node: {node_name}") + self.logger.log("-" * 60) + + success = self.process_node(node_name, force) + + # Track consecutive failures + if success: + consecutive_failures = 0 # Reset on success or skip + else: + # Only increment if it was a real failure (not a skip) + doc_file = DOCS_OUTPUT_PATH / node_name / "en.md" + if not doc_file.exists(): # Real failure, not a skip + consecutive_failures += 1 + + if consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + self.logger.log("") + self.logger.log("=" * 80, "ERROR") + self.logger.log(f"❌ Consecutive failures reached {MAX_CONSECUTIVE_FAILURES}, terminating", "ERROR") + self.logger.log("=" * 80, "ERROR") + self.logger.log(f"Success: {self.logger.success_count}, Failed: {self.logger.failed_count}, Skipped: {self.logger.skipped_count}", "ERROR") + self.logger.log("Please check:", "ERROR") + self.logger.log(" 1. API key is correctly configured", "ERROR") + self.logger.log(" 2. Network connection is stable", "ERROR") + self.logger.log(" 3. API has sufficient balance", "ERROR") + self.logger.log("=" * 80, "ERROR") + sys.exit(1) + else: + consecutive_failures = 0 # Reset if it was a skip + + # Delay requests to avoid API rate limits + if idx < total and idx % BATCH_SIZE == 0: + self.logger.log(f"⏸️ Batch completed, waiting {DELAY_BETWEEN_REQUESTS} seconds...") + time.sleep(DELAY_BETWEEN_REQUESTS) + + # Output summary + self.logger.log("") + self.logger.summary() + + # Update reports if any documents were generated + if self.logger.success_count > 0: + self.logger.log("") + self.logger.log("=" * 80) + self.logger.log("🔄 Updating reports...") + self.logger.log("=" * 80) + self._update_reports() + + def _update_reports(self): + """Update missing_nodes_report.json after generation""" + try: + import subprocess + + # Re-run scan_missing_nodes.py to update the report + scan_script = SCRIPTS_DIR / "scan_missing_nodes.py" + + self.logger.log(f"📊 Running scan to update missing_nodes_report.json...") + result = subprocess.run( + ["python3", str(scan_script)], + capture_output=True, + text=True, + cwd=REPO_ROOT + ) + + if result.returncode == 0: + self.logger.log("✅ missing_nodes_report.json updated successfully") + # Extract summary from output + for line in result.stdout.split('\n'): + if 'Missing' in line or 'nodes' in line.lower(): + self.logger.log(f" {line.strip()}") + else: + self.logger.log(f"⚠️ Failed to update report: {result.stderr}", "WARN") + + except Exception as e: + self.logger.log(f"⚠️ Error updating reports: {e}", "WARN") + + +def main(): + """Main function""" + import argparse + + parser = argparse.ArgumentParser(description='Batch generate node documentation using AI API') + parser.add_argument('mode', choices=['test', 'all', 'node', 'resume', 'changed'], + help='Run mode: test(test), all(all), node(single node), resume(continue unfinished), changed(regenerate changed nodes)') + parser.add_argument('--count', type=int, default=5, + help='Number of nodes to generate in test mode (default: 5)') + parser.add_argument('--node', type=str, + help='Node name for single node mode') + parser.add_argument('--force', action='store_true', + help='Force regenerate existing documentation') + + args = parser.parse_args() + + # Check AI input directory + if not AI_INPUT_PATH.exists(): + print("❌ ai_input directory not found, please run prepare_ai_input.py first") + return + + # Get all prepared nodes from ai_input directory + prepared_nodes = [d.name for d in AI_INPUT_PATH.iterdir() + if d.is_dir() and (d / "ai_prompt.txt").exists()] + + if not prepared_nodes: + print("❌ No prepared nodes found, please run prepare_ai_input.py first") + return + + # Filter to only nodes without documentation (unless force mode) + if not args.force: + nodes_without_docs = [ + node for node in prepared_nodes + if not (DOCS_OUTPUT_PATH / node / "en.md").exists() + ] + else: + nodes_without_docs = prepared_nodes + + nodes_to_process = [] + + print(f"📊 AI input bundles on disk (folders with ai_prompt.txt): {len(prepared_nodes)}") + + # Select nodes to process based on mode + if args.mode == 'changed': + # Read the batch_nodes.json written by prepare_ai_input.py changed mode + batch_file = AI_INPUT_PATH / "batch_nodes.json" + if not batch_file.exists(): + print("❌ batch_nodes.json not found, please run prepare_ai_input.py changed first") + return + with open(batch_file, 'r', encoding='utf-8') as f: + batch_nodes = json.load(f) + nodes_to_process = [ + n['node_name'] + for n in batch_nodes + if (AI_INPUT_PATH / n['node_name'] / "ai_prompt.txt").exists() + ] + print( + f"🔄 Changed mode: {len(nodes_to_process)} node(s) in this batch (from batch_nodes.json).\n" + f" (Scan compares live ComfyUI class-body hash vs node_versions.json 'current_hash'.)\n" + f" The {len(prepared_nodes)} bundles above include older runs — only the batch list is regenerated." + ) + else: + if not args.force: + print(f"📝 Missing en.md (among bundles): {len(nodes_without_docs)}") + else: + print( + f"📝 --force: regenerate pool = all {len(nodes_without_docs)} bundles " + "(not \"missing docs\" — every bundle is eligible for the chosen mode)." + ) + + if args.mode == 'test': + nodes_to_process = nodes_without_docs[:args.count] + print(f"💡 Test mode: Generate first {len(nodes_to_process)} nodes with missing documentation") + + elif args.mode == 'all': + nodes_to_process = nodes_without_docs + print(f"🚀 Full mode: Generate all {len(nodes_to_process)} nodes with missing documentation") + + elif args.mode == 'node': + if not args.node: + print("❌ Please specify node name using --node") + return + if args.node in prepared_nodes: + nodes_to_process = [args.node] + else: + print(f"❌ Node not found: {args.node}") + return + + elif args.mode == 'resume': + nodes_to_process = nodes_without_docs + print(f"🔄 Resume mode: Generate remaining {len(nodes_to_process)} nodes") + + if not nodes_to_process: + print("✅ All nodes already have documentation!") + return + + # Show summary + print(f"\nPreparing to generate documentation for {len(nodes_to_process)} nodes") + print(f"API: {API_MODEL}") + print(f"Output directory: {DOCS_OUTPUT_PATH}") + print(f"Mode: {'Force regenerate' if args.force else 'Skip existing documentation'}") + print() + + # Start generation + generator = AIDocGenerator() + generator.batch_process(nodes_to_process, args.force) + + +if __name__ == "__main__": + main() + diff --git a/pipeline/scripts/batch_translate_docs.py b/pipeline/scripts/batch_translate_docs.py new file mode 100644 index 000000000..fb9c8ede5 --- /dev/null +++ b/pipeline/scripts/batch_translate_docs.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +""" +Batch translate documentation using AI +Trusts the batch list from prepare_translation.py (files already verified as missing) +""" + +import argparse +import json +import os +import re +import sys +import time +import logging +from pathlib import Path +from datetime import datetime +from openai import OpenAI + +import runtime # noqa: F401 +from lib.doc_disclaimer import ( + compose_document, + create_translated_disclaimer, + strip_ai_disclaimer, +) +from lib.doc_title import ensure_doc_title, strip_leading_h1 +from lib.hash_footer import ( + extract_english_source_fingerprint_hex, + format_source_hash_footer, + load_node_source_sha256, + strip_source_hash_footer, + strip_trailing_fingerprint_section, +) +from lib.paths import ( + LOGS_DIR, + TRANSLATION_BATCHES_DIR, + TRANSLATION_CONFIG, + default_embedded_docs_path, + load_dotenv, +) +from update_translation_status import batch_update_translations + +load_dotenv() + +DOCS_PATH = default_embedded_docs_path() / "comfyui_embedded_docs" / "docs" +TRANSLATION_CONFIG_FILE = TRANSLATION_CONFIG + +# GitHub repository info +GITHUB_REPO = "Comfy-Org/embedded-docs" +GITHUB_BRANCH = "main" + +# Ensure logs directory exists +LOGS_DIR.mkdir(exist_ok=True) + +# Setup logging +log_file = LOGS_DIR / f"translation_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" +logging.basicConfig( + level=logging.INFO, + format='[%(asctime)s] [%(levelname)s] %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[ + logging.FileHandler(log_file, encoding='utf-8'), + logging.StreamHandler() + ] +) + +logger = logging.getLogger(__name__) + +# AI Configuration +DEFAULT_API_KEY = os.getenv('DEEPSEEK_API_KEY', '') +DEFAULT_BASE_URL = "https://api.deepseek.com" +DEFAULT_MODEL = os.getenv('API_MODEL', 'deepseek-chat') +DEFAULT_BATCH_SIZE = 5 + +# Custom log level for success + +def replace_heading_placeholders(content, lang_config): + """Replace placeholder headings with actual headings. + Handles both bare placeholders and AI-generated '## {heading_xxx}' patterns + to avoid doubled heading markers (e.g. '## ## 输入').""" + # First, strip any leading '## ' or '# ' from lines containing placeholders + for placeholder in ('{heading_overview}', '{heading_inputs}', '{heading_outputs}'): + pattern = re.compile(r'^#{1,2}\s+' + re.escape(placeholder) + r'\s*$', re.MULTILINE) + content = pattern.sub(placeholder, content) + + content = content.replace('{heading_overview}', f"## {lang_config.get('heading_overview', 'Overview')}") + content = content.replace('{heading_inputs}', f"## {lang_config.get('heading_inputs', 'Inputs')}") + content = content.replace('{heading_outputs}', f"## {lang_config.get('heading_outputs', 'Outputs')}") + return content + + +# Multi-language preamble patterns — AI sometimes prepends "Here is the translation..." +# before the actual node description. These patterns detect and strip that paragraph. +# Each entry is a regex that matches the first sentence of a preamble paragraph. +_PREAMBLE_PATTERNS: dict[str, list[str]] = { + 'zh': [r'^以下是为您翻译', r'^以下是翻译结果', r'^这是.*?翻译'], + 'zh-TW': [r'^以下是为您翻譯', r'^以下是翻譯結果', r'^這是.*?翻譯'], + 'ja': [r'^以下が翻訳', r'^翻訳結果'], + 'ko': [r'^다음은.*?번역', r'^번역 결과'], + 'ru': [r'^Вот перевод', r'^Перевод документаци'], + 'es': [r'^Aquí está la traducción', r'^Esta es la traducción', r'^Traducción de'], + 'fr': [r'^Voici la traduction', r'^Voici le document', r'^Traduction de'], + 'ar': [r'^هذه هي الترجمة', r'^إليك الترجمة', r'^أنت خبير في الترجمة', r'^هذا هو المستند'], + 'tr': [r'^İşte çeviri', r'^Çeviri sonucu'], + 'pt-BR': [r'^Aqui está a tradução', r'^Esta é a tradução'], + 'fa': [r'^این ترجمه', r'^در زیر ترجمه'], +} + + +def strip_ai_preamble(content: str, lang: str) -> str: + """Remove translation preamble that the AI sometimes prepends. + + The AI sometimes adds a sentence like 'Here is the translation into Russian:' + before the actual node description. This function detects and removes it. + + The preamble is expected to be the first paragraph after the H1 title + (which is already stripped by strip_leading_h1 before this is called). + """ + patterns = _PREAMBLE_PATTERNS.get(lang, []) + if not patterns: + return content + + lines = content.split('\n') + # Find the first non-empty line + first_content_idx = 0 + while first_content_idx < len(lines) and not lines[first_content_idx].strip(): + first_content_idx += 1 + + if first_content_idx >= len(lines): + return content + + first_line = lines[first_content_idx].strip() + for pat in patterns: + if re.search(pat, first_line): + # Remove the preamble line and any following blank lines up to actual content + del lines[first_content_idx] + # Remove trailing blank lines after preamble + while first_content_idx < len(lines) and not lines[first_content_idx].strip(): + del lines[first_content_idx] + break + + return '\n'.join(lines) + +def _fix_output_names_in_translation(translated_content: str, full_en: str) -> str: + """Parse the English en.md Outputs table and force output names in translated content + to match the English original by row index. This prevents the AI from translating + distinct output names (e.g. 'positive'/'negative') into the same translated word. + + Works by: + 1. Extracting output names from en.md Outputs table in order (row by row). + 2. Extracting output table lines from translated content. + 3. Replacing the first column (output name) of each row with its English counterpart. + """ + # Extract English output names + en_outputs_match = re.search( + r'##\s+(?:Outputs|输出|輸出|出力|출력|Выходы|Salidas|Sorties|المخرجات|Çıktılar|خروجی‌ها)\s*\n\n(.*?)(?=\n##|\Z)', + full_en, re.DOTALL + ) + if not en_outputs_match: + return translated_content # No outputs table in English doc, nothing to fix + + en_table = en_outputs_match.group(1).strip() + en_lines = en_table.split('\n') + if len(en_lines) < 3: + return translated_content + + # Skip header row (| Name | Type | ...) and separator row (|---|---|...) + # Output names are backtick-wrapped in first column of data rows + en_output_names = [] + for line in en_lines[2:]: # Skip header + separator + line = line.strip() + if not line.startswith('|'): + continue + parts = [p.strip() for p in line.split('|')] + if len(parts) >= 2: + # Extract backtick-wrapped name from first column + name = parts[1].strip('`').strip() + if name: + en_output_names.append(name) + + if not en_output_names: + return translated_content + + # Now find and fix the Outputs table in translated content + # Match any known heading for "Outputs" in any language + tr_outputs_match = re.search( + r'##\s+(?:输出|輸出|出力|출력|Выходы|Salidas|Sorties|Outputs|المخرجات|Çıktılar|خروجی‌ها)\s*\n\n(.*?)(?=\n##|\Z)', + translated_content, re.DOTALL + ) + if not tr_outputs_match: + return translated_content + + tr_table = tr_outputs_match.group(1) + tr_lines = tr_table.split('\n') + + # Build new table lines with English names enforced + new_lines = list(tr_lines) + data_row_idx = 0 + for i, line in enumerate(tr_lines): + if i < 2: # Keep header and separator rows as-is + continue + line = line.strip() + if not line.startswith('|'): + continue + parts = line.split('|') + if len(parts) < 2: + continue + # Check if we have an English name for this row + if data_row_idx < len(en_output_names): + en_name = en_output_names[data_row_idx] + # Replace the output name column (first column after initial pipe) + # The backtick-wrapped name in the first data column + orig_name_match = re.match(r'^\|(\s*`[^`]*`\s*)', line) + if orig_name_match: + old = orig_name_match.group(1) + new = f' `{en_name}` ' + new_lines[i] = line.replace(old, new, 1) + data_row_idx += 1 + + new_table = '\n'.join(new_lines) + translated_content = translated_content[:tr_outputs_match.start(1)] + new_table + translated_content[tr_outputs_match.end(1):] + return translated_content + + +def translate_document(node_name, target_lang, lang_config, api_key, base_url, model): + """Translate a single document""" + + # Read English source document + source_file = DOCS_PATH / node_name / "en.md" + if not source_file.exists(): + raise FileNotFoundError(f"English source not found: {source_file}") + + with open(source_file, 'r', encoding='utf-8') as f: + full_en = f.read() + + # Same hex as en.md footer when present; fallback matches pipeline / English doc generation. + src_fp_hex = extract_english_source_fingerprint_hex(full_en) or load_node_source_sha256(node_name) + + # Omit footer, disclaimer, and H1 title from LLM input; title is injected from frontend after translate. + source_content = strip_leading_h1(strip_ai_disclaimer(strip_source_hash_footer(full_en))) + + # Build prompt using language-specific template + prompt_template = lang_config.get('prompt_template', '') + full_prompt = prompt_template + "\n\n" + source_content + + # Call AI API + client = OpenAI(api_key=api_key, base_url=base_url) + + max_retries = 3 + for attempt in range(max_retries): + try: + response = client.chat.completions.create( + model=model, + messages=[ + {"role": "user", "content": full_prompt} + ], + temperature=0.3, + stream=False + ) + + content = response.choices[0].message.content + + content = strip_ai_disclaimer(content) + + # Strip AI preamble ("Here is the translation...") in the target language + content = strip_ai_preamble(content, target_lang) + + # Replace placeholder headings with actual headings + content = replace_heading_placeholders(content, lang_config) + + # Localized node title from frontend display_name (not AI-translated) + content = ensure_doc_title(strip_leading_h1(content), node_name, target_lang) + + # Drop any model-added localized fingerprint block; keep one English footer for traceability. + content = strip_trailing_fingerprint_section(content) + + # Post-process: enforce English output names. The AI sometimes translates output names + # (e.g. 'positive' → '正向', 'négatif') causing duplicates in the Outputs table. + # We parse the en.md Outputs table and force the first column to match. + content = _fix_output_names_in_translation(content, full_en) + + disclaimer = create_translated_disclaimer(target_lang, node_name, lang_config) + footer = format_source_hash_footer(src_fp_hex) if src_fp_hex else "" + final_content = compose_document(content, disclaimer, footer) + if not src_fp_hex: + logger.warning( + "No source fingerprint for %s; translated file has no SHA footer " + "(add footer to en.md or ensure ai_input/%s/basic_info.json has source_hash).", + node_name, + node_name, + ) + + return final_content + + except Exception as e: + if attempt < max_retries - 1: + wait_time = (attempt + 1) * 2 + logger.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {wait_time}s...") + time.sleep(wait_time) + else: + raise + +def process_node(node_name, target_lang, lang_config, api_key, base_url, model, force=False): + """ + Process a single node translation. + When force=False, skips if target file already exists (do not overwrite). + """ + output_file = DOCS_PATH / node_name / f"{target_lang}.md" + if output_file.exists() and not force: + logger.info(f"⏭️ Skip (already exists): {node_name}") + return "skipped" + + try: + logger.info(f"🤖 Translating: {node_name}") + + # Translate document + translated_content = translate_document(node_name, target_lang, lang_config, api_key, base_url, model) + + # Save translated document + output_dir = DOCS_PATH / node_name + output_dir.mkdir(exist_ok=True) + + with open(output_file, 'w', encoding='utf-8') as f: + f.write(translated_content) + + logger.info(f"💾 Saved: {output_file}") + logger.log(25, f"✅ Successfully translated: {node_name}") + + return "success" + + except Exception as e: + logger.error(f"❌ Translation failed {node_name}: {e}") + return "failed" + +def main(): + """Main function""" + parser = argparse.ArgumentParser(description="Batch translate docs using prepared batch file") + parser.add_argument("--lang", required=True, help="Target language code") + parser.add_argument("--mode", choices=("test", "all"), default="all", + help="test = limit to --count, all = whole batch (default: all)") + parser.add_argument("--count", type=int, default=20, + help="Max nodes in test mode (default: 20)") + parser.add_argument("--force", action="store_true", help="Overwrite existing translations") + parser.add_argument("--node-list", type=str, default=None, + help="Comma-separated list of node names to translate (overrides batch file)") + parser.add_argument("--node-list-file", type=str, default=None, + help="Path to JSON file with {'nodes': ['NodeA', 'NodeB']} (overrides batch file)") + args = parser.parse_args() + target_lang = args.lang + mode = args.mode + force = args.force + + # Load translation config + with open(TRANSLATION_CONFIG_FILE, 'r', encoding='utf-8') as f: + translation_config = json.load(f) + + if target_lang not in translation_config: + print(f"❌ Error: Unknown language '{target_lang}'") + print(f"Available: {', '.join(translation_config.keys())}") + sys.exit(1) + + lang_config = translation_config[target_lang] + + # Determine which nodes to translate + nodes_to_translate: list[str] = [] + + if args.node_list: + # Direct node list from CLI argument + nodes_to_translate = [n.strip() for n in args.node_list.split(",") if n.strip()] + print(f"📋 Using CLI node list: {len(nodes_to_translate)} nodes") + elif args.node_list_file: + # Node list from JSON file + nl_path = Path(args.node_list_file) + if not nl_path.exists(): + print(f"❌ Error: Node list file not found: {nl_path}") + sys.exit(1) + with open(nl_path, 'r', encoding='utf-8') as f: + nl_data = json.load(f) + nodes_to_translate = nl_data.get('nodes', []) + print(f"📋 Using node list file: {len(nodes_to_translate)} nodes") + else: + # Load batch file (default behavior) + batch_file = TRANSLATION_BATCHES_DIR / f"batch_{target_lang}.json" + if not batch_file.exists(): + print(f"❌ Error: Batch file not found: {batch_file}") + print(f" Please run: python3 prepare_translation.py --lang {target_lang}") + sys.exit(1) + + with open(batch_file, 'r', encoding='utf-8') as f: + batch_data = json.load(f) + + nodes_to_translate = batch_data.get('nodes', []) + if mode == "test": + nodes_to_translate = nodes_to_translate[:args.count] + + print(f"📊 Batch prepared: {batch_data.get('total', 0)} nodes") + print(f"💡 {'Test' if mode == 'test' else 'Full'} mode: Translating {len(nodes_to_translate)} nodes") + print() + print(f"Target language: {lang_config['name']} ({target_lang})") + print(f"API: {DEFAULT_MODEL}") + print(f"Output: {DOCS_PATH}") + print(f"Mode: {'Force retranslate (overwrite)' if force else 'Normal (skip existing, do not overwrite)'}") + print() + + logger.info("=" * 80) + logger.info("🚀 Batch Translation Started") + logger.info(f"📊 Total: {len(nodes_to_translate)} nodes") + logger.info(f"🌐 Language: {lang_config['name']} ({target_lang})") + logger.info(f"🔧 API: {DEFAULT_MODEL}") + logger.info(f"⚙️ Batch size: {DEFAULT_BATCH_SIZE}") + logger.info("=" * 80) + logger.info("") + + # Translate documents + success_count = 0 + failed_count = 0 + skipped_count = 0 + consecutive_failures = 0 + MAX_CONSECUTIVE_FAILURES = 5 + completed_nodes = [] # Track completed nodes for batch update + + for idx, node_name in enumerate(nodes_to_translate, 1): + logger.info("") + logger.info(f"[{idx}/{len(nodes_to_translate)}] Processing node: {node_name}") + logger.info("-" * 60) + + result = process_node( + node_name, + target_lang, + lang_config, + DEFAULT_API_KEY, + DEFAULT_BASE_URL, + DEFAULT_MODEL, + force=force + ) + + if result == "success": + success_count += 1 + consecutive_failures = 0 # Reset counter on success + completed_nodes.append(node_name) # Track for batch update + elif result == "failed": + failed_count += 1 + consecutive_failures += 1 + + # Check if we've hit the consecutive failure limit + if consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + logger.error("") + logger.error("=" * 80) + logger.error(f"❌ Consecutive failures reached {MAX_CONSECUTIVE_FAILURES}, terminating") + logger.error("=" * 80) + logger.error(f"Success: {success_count}, Failed: {failed_count}, Skipped: {skipped_count}") + logger.error("Please check:") + logger.error(" 1. API key is correctly configured") + logger.error(" 2. Network connection is stable") + logger.error(" 3. API has sufficient balance") + logger.error("=" * 80) + print() + print("=" * 80) + print(f"❌ Consecutive failures: {MAX_CONSECUTIVE_FAILURES}, terminated automatically") + print(f"Success: {success_count}, Failed: {failed_count}, Skipped: {skipped_count}") + print(f"📁 Log: {log_file}") + print("=" * 80) + sys.exit(1) + elif result == "skipped": + skipped_count += 1 + consecutive_failures = 0 # Reset counter on skip + + # Rate limiting + if idx % DEFAULT_BATCH_SIZE == 0 and idx < len(nodes_to_translate): + logger.info("⏸️ Batch rest for 2 seconds...") + time.sleep(2) + + logger.info("") + logger.info("=" * 80) + logger.info("📊 Translation Summary") + logger.info("=" * 80) + logger.info(f"✅ Success: {success_count}") + logger.info(f"❌ Failed: {failed_count}") + logger.info(f"⏭️ Skipped: {skipped_count}") + logger.info(f"📁 Log file: {log_file}") + logger.info("") + + # Update translation status in JSON + if completed_nodes: + logger.info("=" * 80) + logger.info("🔄 Updating translation status in JSON...") + logger.info("=" * 80) + batch_update_translations({target_lang: completed_nodes}) + logger.info(f"✅ Removed {target_lang} from {len(completed_nodes)} nodes' missing languages") + logger.info("") + + print() + print("=" * 80) + print(f"✅ Translation completed! Success: {success_count}, Failed: {failed_count}, Skipped: {skipped_count}") + if completed_nodes: + print(f"📝 Updated missing_nodes_report.json ({len(completed_nodes)} nodes)") + print(f"📁 Log: {log_file}") + print("=" * 80) + +if __name__ == "__main__": + main() diff --git a/pipeline/scripts/check_config.py b/pipeline/scripts/check_config.py new file mode 100644 index 000000000..d5775f283 --- /dev/null +++ b/pipeline/scripts/check_config.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +Check configuration and verify all paths are correctly set +""" + +import os +import sys +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import ENV_FILE, REPO_ROOT, load_dotenv + +def check_config(): + """Verify all configuration settings""" + + env_path = ENV_FILE + + print("=" * 80) + print("Configuration Check") + print("=" * 80) + print() + + # Check .env file existence + if not env_path.exists(): + print(f"❌ .env file not found at: {env_path}") + print(f" Please copy env.example to .env and configure it:") + print(f" cp env.example .env") + return False + + print(f"✅ .env file found: {env_path}") + load_dotenv() + print() + + all_ok = True + + # Check repository paths + print("Repository Paths:") + print("-" * 80) + + comfyui_path = os.getenv('COMFYUI_PATH') + if comfyui_path: + path = Path(comfyui_path) + if path.exists(): + print(f"✅ COMFYUI_PATH: {path}") + # Check for key files + if (path / "nodes.py").exists(): + print(f" ✓ Found nodes.py") + if (path / "comfy_extras").exists(): + print(f" ✓ Found comfy_extras/") + else: + print(f"❌ COMFYUI_PATH not found: {path}") + all_ok = False + else: + print(f"⚠️ COMFYUI_PATH not set in .env") + all_ok = False + + print() + + frontend_path = os.getenv('COMFYUI_FRONTEND_PATH') + if frontend_path: + path = Path(frontend_path) + if path.exists(): + print(f"✅ COMFYUI_FRONTEND_PATH: {path}") + else: + print(f"❌ COMFYUI_FRONTEND_PATH not found: {path}") + all_ok = False + else: + print(f"⚠️ COMFYUI_FRONTEND_PATH not set (optional)") + + print() + + embedded_docs_path = os.getenv('EMBEDDED_DOCS_PATH') + if embedded_docs_path: + path = Path(embedded_docs_path) + if path.exists(): + print(f"✅ EMBEDDED_DOCS_PATH: {path}") + docs_path = path / "comfyui_embedded_docs" / "docs" + if docs_path.exists(): + print(f" ✓ Found docs directory: {docs_path}") + # Count existing docs + doc_count = len([d for d in docs_path.iterdir() if d.is_dir()]) + print(f" ✓ Existing node docs: {doc_count}") + else: + print(f"❌ EMBEDDED_DOCS_PATH not found: {path}") + all_ok = False + else: + print(f"⚠️ EMBEDDED_DOCS_PATH not set in .env") + all_ok = False + + print() + print("API Configuration:") + print("-" * 80) + + # Check API key + api_key = os.getenv('DEEPSEEK_API_KEY') + if api_key and api_key != 'your_deepseek_api_key_here': + print(f"✅ DEEPSEEK_API_KEY: {'*' * 20}{api_key[-4:]}") + else: + print(f"❌ DEEPSEEK_API_KEY not configured") + all_ok = False + + # Check API settings + api_base = os.getenv('API_BASE_URL', 'https://api.deepseek.com') + api_model = os.getenv('API_MODEL', 'deepseek-chat') + print(f"✅ API_BASE_URL: {api_base}") + print(f"✅ API_MODEL: {api_model}") + + print() + print("Batch Settings:") + print("-" * 80) + batch_size = os.getenv('BATCH_SIZE', '5') + max_retries = os.getenv('MAX_RETRIES', '3') + delay = os.getenv('DELAY_BETWEEN_REQUESTS', '2') + print(f"✅ BATCH_SIZE: {batch_size}") + print(f"✅ MAX_RETRIES: {max_retries}") + print(f"✅ DELAY_BETWEEN_REQUESTS: {delay}s") + + print() + print("=" * 80) + if all_ok: + print("✅ All critical configurations are correct!") + print(" You can now run:") + print(" python3 scripts/scan_missing_nodes.py") + else: + print("❌ Some configurations need attention.") + print(" Please update your .env file.") + print("=" * 80) + + return all_ok + +if __name__ == "__main__": + sys.exit(0 if check_config() else 1) + diff --git a/pipeline/scripts/check_md_links.py b/pipeline/scripts/check_md_links.py new file mode 100644 index 000000000..f729b63e2 --- /dev/null +++ b/pipeline/scripts/check_md_links.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +检查 Markdown 文档中的链接有效性和占位符 +用法: python check_md_links.py [--fix-placeholders] +""" + +import os +import re +import sys +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import embedded_docs_dir + +DOCS_ROOT = embedded_docs_dir() + +# Supported file extensions +doc_exts = {'.md', '.mdx'} + +# Match Markdown images/links and HTML img/video/audio/source tag src attributes +MD_LINK_RE = re.compile(r'!\[[^\]]*\]\(([^)]+)\)|\[[^\]]*\]\(([^)]+)\)') +HTML_SRC_RE = re.compile(r'<(?:img|video|audio|source)[^>]+src=["\']([^"\'>]+)["\']', re.IGNORECASE) +PLACEHOLDER_RE = re.compile(r'\{(heading_\w+)\}') + +# 不同语言的标题映射 +HEADING_TRANSLATIONS = { + 'en': { + 'heading_overview': '## Overview', + 'heading_inputs': '## Inputs', + 'heading_outputs': '## Outputs', + 'heading_usage': '## Usage', + 'heading_examples': '## Examples', + }, + 'zh': { + 'heading_overview': '## 概述', + 'heading_inputs': '## 输入', + 'heading_outputs': '## 输出', + 'heading_usage': '## 用法', + 'heading_examples': '## 示例', + }, + 'es': { + 'heading_overview': '## Descripción general', + 'heading_inputs': '## Entradas', + 'heading_outputs': '## Salidas', + 'heading_usage': '## Uso', + 'heading_examples': '## Ejemplos', + }, + 'fr': { + 'heading_overview': '## Aperçu', + 'heading_inputs': '## Entrées', + 'heading_outputs': '## Sorties', + 'heading_usage': '## Utilisation', + 'heading_examples': '## Exemples', + }, + 'ja': { + 'heading_overview': '## 概要', + 'heading_inputs': '## 入力', + 'heading_outputs': '## 出力', + 'heading_usage': '## 使用方法', + 'heading_examples': '## 例', + }, + 'ko': { + 'heading_overview': '## 개요', + 'heading_inputs': '## 입력', + 'heading_outputs': '## 출력', + 'heading_usage': '## 사용법', + 'heading_examples': '## 예시', + }, + 'ru': { + 'heading_overview': '## Обзор', + 'heading_inputs': '## Входы', + 'heading_outputs': '## Выходы', + 'heading_usage': '## Использование', + 'heading_examples': '## Примеры', + }, +} + +def get_language_from_filename(filename): + """从文件名获取语言代码""" + stem = Path(filename).stem + return stem if stem in HEADING_TRANSLATIONS else None + +def is_local_link(link): + """只检查本地相对路径（非 http/https/data: 开头）""" + link = link.strip() + return not (link.startswith('http://') or link.startswith('https://') or link.startswith('data:')) + +def find_links_in_line(line): + """提取行中的所有本地链接""" + links = [] + for m in MD_LINK_RE.finditer(line): + for g in m.groups(): + if g and is_local_link(g): + links.append(g) + for m in HTML_SRC_RE.finditer(line): + g = m.group(1) + if g and is_local_link(g): + links.append(g) + return links + +def find_placeholders_in_content(content): + """查找内容中的占位符""" + return PLACEHOLDER_RE.findall(content) + +def check_file(fpath, fix_placeholders=False): + """检查单个文件的链接和占位符""" + errors = [] + placeholder_issues = [] + rel_fpath = fpath.relative_to(DOCS_ROOT.parent.parent) + lang = get_language_from_filename(fpath.name) + + with open(fpath, 'r', encoding='utf-8') as f: + content = f.read() + lines = content.split('\n') + + # 检查占位符 + placeholders = find_placeholders_in_content(content) + if placeholders: + placeholder_issues.append(f"{rel_fpath}: 发现占位符 {placeholders}") + + # 如果需要修复且能识别语言 + if fix_placeholders and lang: + translations = HEADING_TRANSLATIONS[lang] + modified = False + for placeholder in placeholders: + pattern = '{' + placeholder + '}' + if placeholder in translations: + content = content.replace(pattern, translations[placeholder]) + modified = True + + if modified: + with open(fpath, 'w', encoding='utf-8') as f: + f.write(content) + placeholder_issues[-1] += " [已修复]" + + # 检查链接 + for idx, line in enumerate(lines, 1): + for link in find_links_in_line(line): + link_path = link.split('#')[0].split('?')[0] + if not link_path: + continue + + if link_path.startswith('/'): + abs_path = DOCS_ROOT / link_path.lstrip('/') + else: + try: + abs_path = (fpath.parent / link_path).resolve() + if not abs_path.exists(): + abs_path_alt = (fpath.parent / link_path).absolute() + if abs_path_alt.exists(): + abs_path = abs_path_alt + except (OSError, ValueError): + abs_path = (fpath.parent / link_path).absolute() + + if not abs_path.exists(): + errors.append(f"[链接失效] {rel_fpath}:{idx}: {link}") + + return errors, placeholder_issues + +def check_links(): + if not DOCS_ROOT.exists(): + print(f"错误: 文档目录不存在: {DOCS_ROOT}") + sys.exit(1) + + fix_placeholders = '--fix-placeholders' in sys.argv + link_errors = [] + placeholder_issues = [] + + print(f"正在检查 {DOCS_ROOT} 下的所有文档...") + + for root, _, files in os.walk(DOCS_ROOT): + for fname in files: + if Path(fname).suffix.lower() in doc_exts: + fpath = Path(root) / fname + errors, placeholders = check_file(fpath, fix_placeholders) + link_errors.extend(errors) + placeholder_issues.extend(placeholders) + + has_issues = False + + if placeholder_issues: + has_issues = True + print("\n" + "=" * 80) + print(f"发现 {len(placeholder_issues)} 个文件包含占位符：") + print("=" * 80) + for issue in placeholder_issues: + print(f" {issue}") + if fix_placeholders: + print("\n✓ 占位符已自动修复") + else: + print("\n提示: 运行 --fix-placeholders 参数来自动替换占位符") + + if link_errors: + has_issues = True + print("\n" + "=" * 80) + print(f"发现 {len(link_errors)} 个无效链接：") + print("=" * 80) + for i, err in enumerate(link_errors): + if i < 10: + print(f" {err}") + elif i == 10: + print(f"\n ... 还有 {len(link_errors) - 10} 个错误（仅显示前10个）") + break + print("\n请修正上述链接问题。") + + if not has_issues: + print("\n✓ 所有检查通过！") + else: + sys.exit(1) + +if __name__ == '__main__': + check_links() diff --git a/pipeline/scripts/check_outputs.py b/pipeline/scripts/check_outputs.py new file mode 100644 index 000000000..a7897d42e --- /dev/null +++ b/pipeline/scripts/check_outputs.py @@ -0,0 +1,20 @@ +import json +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import NODE_TRANSLATIONS + +data = json.load(open(NODE_TRANSLATIONS, encoding="utf-8")) +zh = data.get('zh', {}) + +nodes_with_outputs = [] +for n, v in zh.items(): + if v.get('outputs'): + nodes_with_outputs.append((n, v.get('outputs', {}))) + +print(f'有输出翻译的节点数: {len(nodes_with_outputs)}') +print('\n示例:') +for n, o in nodes_with_outputs[:5]: + print(f'\n{n}:') + print(json.dumps(o, ensure_ascii=False, indent=2)) + diff --git a/pipeline/scripts/cleanup_duplicate_hashes.py b/pipeline/scripts/cleanup_duplicate_hashes.py new file mode 100644 index 000000000..162ad743b --- /dev/null +++ b/pipeline/scripts/cleanup_duplicate_hashes.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Clean up duplicate hash records in node_versions.json + +This script removes duplicate version records that have the same hash, +keeping only the first occurrence of each unique hash. +""" + +import json +from collections import OrderedDict + +import runtime # noqa: F401 +from lib.paths import NODE_VERSIONS + + +def cleanup_duplicate_hashes(): + """Remove duplicate hash records from node_versions.json""" + version_db_path = NODE_VERSIONS + + if not version_db_path.exists(): + print("❌ node_versions.json not found") + return + + print("📖 Loading node_versions.json...") + with open(version_db_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + nodes = data.get("nodes", {}) + total_removed = 0 + nodes_cleaned = [] + + print(f"🔍 Processing {len(nodes)} nodes...") + print("=" * 80) + + for node_name, node_data in nodes.items(): + versions = node_data.get("versions", []) + if not versions: + continue + + # Track seen hashes, keep only first occurrence + seen_hashes = {} + unique_versions = [] + removed_count = 0 + + for version in versions: + hash_val = version.get("source_hash", "") + if hash_val in seen_hashes: + # Duplicate hash, skip it + removed_count += 1 + total_removed += 1 + else: + # First occurrence of this hash, keep it + seen_hashes[hash_val] = True + unique_versions.append(version) + + if removed_count > 0: + nodes_cleaned.append(node_name) + print(f" ✅ {node_name}: 移除了 {removed_count} 个重复记录 (保留 {len(unique_versions)} 个)") + node_data["versions"] = unique_versions + + # Update current_hash to match the last version (most recent) + if unique_versions: + node_data["current_hash"] = unique_versions[-1]["source_hash"] + node_data["last_updated"] = unique_versions[-1]["extracted_at"] + + if total_removed > 0: + print("=" * 80) + print(f"\n📊 清理完成:") + print(f" - 清理了 {len(nodes_cleaned)} 个节点") + print(f" - 移除了 {total_removed} 个重复的 hash 记录") + + # Backup original file + backup_path = version_db_path.with_suffix('.json.backup') + print(f"\n💾 备份原文件到: {backup_path}") + with open(backup_path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + # Save cleaned data + print(f"💾 保存清理后的数据...") + with open(version_db_path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + print(f"\n✅ 清理完成！") + else: + print("\n✅ 没有发现重复记录，无需清理") + + +if __name__ == "__main__": + cleanup_duplicate_hashes() diff --git a/pipeline/scripts/fix_doc_titles.py b/pipeline/scripts/fix_doc_titles.py new file mode 100644 index 000000000..72a5b3ed3 --- /dev/null +++ b/pipeline/scripts/fix_doc_titles.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +""" +Fix missing, duplicate, or incorrect H1 titles in existing node documentation. + +Titles are set from frontend nodeDefs display_name (locale → English → class name), +not from AI. Preserves disclaimer and SHA footer on each file. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import runtime # noqa: F401 +from lib.doc_disclaimer import strip_ai_disclaimer +from lib.doc_title import ( + HashMode, + analyze_title_issues, + fix_document_title, + load_node_translations, +) +from lib.hash_footer import ( + extract_english_source_fingerprint_hex, + load_node_source_sha256, + strip_source_hash_footer, +) +from lib.paths import TRANSLATION_CONFIG, embedded_docs_dir, load_dotenv + +load_dotenv() + +DOCS_PATH = embedded_docs_dir() +DOC_LANGS = ["en", "zh", "zh-TW", "es", "fr", "ja", "ko", "ru", "ar", "tr", "pt-BR", "fa"] + + +def resolve_en_source_hex(node_name: str, hash_mode: HashMode) -> Optional[str]: + """SHA hex for update mode: ai_input → en.md on disk.""" + if hash_mode != "update": + return None + from_ai = load_node_source_sha256(node_name) + if from_ai: + return from_ai + en_path = DOCS_PATH / node_name / "en.md" + if en_path.is_file(): + return extract_english_source_fingerprint_hex(en_path.read_text(encoding="utf-8")) + return None + + +def load_translation_config() -> Dict[str, Any]: + if not TRANSLATION_CONFIG.is_file(): + return {} + with open(TRANSLATION_CONFIG, encoding="utf-8") as f: + return json.load(f) + + +def iter_doc_files( + node_filter: Optional[str] = None, + lang_filter: Optional[str] = None, +) -> List[Tuple[Path, str, str]]: + if not DOCS_PATH.is_dir(): + return [] + + out: List[Tuple[Path, str, str]] = [] + for node_dir in sorted(DOCS_PATH.iterdir(), key=lambda p: p.name.lower()): + if not node_dir.is_dir(): + continue + if node_filter and node_dir.name != node_filter: + continue + langs = [lang_filter] if lang_filter else DOC_LANGS + for lang in langs: + md = node_dir / f"{lang}.md" + if md.is_file(): + out.append((md, node_dir.name, lang)) + return out + + +def fix_file( + path: Path, + node_name: str, + lang: str, + translations: Dict[str, Any], + lang_config: Dict[str, Any], + dry_run: bool, + hash_mode: HashMode, + en_source_hex: Optional[str], +) -> Tuple[bool, List[str]]: + original = path.read_text(encoding="utf-8") + body = strip_ai_disclaimer(strip_source_hash_footer(original)) + issues = analyze_title_issues(body, node_name, lang, translations) + if not issues: + return False, [] + + if not dry_run: + path.write_text( + fix_document_title( + original, + node_name, + lang, + translations, + lang_config, + hash_mode=hash_mode, + en_source_hex=en_source_hex, + ), + encoding="utf-8", + ) + return True, issues + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Fix H1 titles in existing node docs (frontend display_name)" + ) + parser.add_argument("--mode", choices=("test", "all"), default="all") + parser.add_argument("--count", type=int, default=20, help="Max files in test mode") + parser.add_argument("--node", type=str, help="Only this node folder") + parser.add_argument("--lang", type=str, choices=DOC_LANGS, help="Only this language") + parser.add_argument("--dry-run", action="store_true", help="Report only, do not write") + parser.add_argument( + "--hash-mode", + choices=("preserve", "update"), + default="preserve", + help=( + "preserve: keep original disclaimer + SHA footer unchanged (default); " + "update: rewrite disclaimer and sync SHA from en.md / ai_input" + ), + ) + args = parser.parse_args() + hash_mode: HashMode = args.hash_mode + + if not DOCS_PATH.is_dir(): + print(f"❌ Docs directory not found: {DOCS_PATH}") + return 1 + + translations = load_node_translations() + if not translations: + print( + "⚠️ Warning: data/node_translations.json not found or empty.\n" + " Titles will fall back to class names. Run:\n" + " python3 scripts/sync_frontend_translations.py --export" + ) + + lang_config = load_translation_config() + files = iter_doc_files(args.node, args.lang) + if args.mode == "test": + files = files[: args.count] + + print("=" * 80) + print("Fix document titles (frontend display_name)") + print("=" * 80) + print(f"Docs: {DOCS_PATH}") + print(f"Files to scan: {len(files)}") + print(f"Mode: {'dry-run' if args.dry_run else 'write'}") + print(f"Hash: {hash_mode}") + print("=" * 80) + print() + + fixed_count = 0 + skipped_count = 0 + issue_totals: Dict[str, int] = {} + en_hash_cache: Dict[str, Optional[str]] = {} + + for path, node_name, lang in files: + en_hex: Optional[str] = None + if hash_mode == "update": + if node_name not in en_hash_cache: + en_hash_cache[node_name] = resolve_en_source_hex(node_name, hash_mode) + en_hex = en_hash_cache[node_name] + + changed, issues = fix_file( + path, + node_name, + lang, + translations, + lang_config, + args.dry_run, + hash_mode, + en_hex, + ) + if changed: + fixed_count += 1 + labels = ", ".join(issues) + action = "would fix" if args.dry_run else "fixed" + print(f"✅ {action}: {node_name}/{lang}.md ({labels})") + for code in issues: + issue_totals[code] = issue_totals.get(code, 0) + 1 + else: + skipped_count += 1 + + print() + print("=" * 80) + print("Summary") + print("=" * 80) + print(f"{'Would fix' if args.dry_run else 'Fixed'}: {fixed_count}") + print(f"OK (no change): {skipped_count}") + if issue_totals: + print("Issues addressed:") + for code, n in sorted(issue_totals.items()): + print(f" - {code}: {n}") + print("=" * 80) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline/scripts/fix_translations.py b/pipeline/scripts/fix_translations.py new file mode 100644 index 000000000..0937e65bf --- /dev/null +++ b/pipeline/scripts/fix_translations.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +修复翻译文件中的占位符和参数名称问题 +""" + +import json +import re +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import NODE_TRANSLATIONS, TRANSLATION_CONFIG, embedded_docs_dir + +DOCS_PATH = embedded_docs_dir() +TRANSLATION_CONFIG_FILE = TRANSLATION_CONFIG +TRANSLATIONS_FILE = NODE_TRANSLATIONS + +def fix_heading_placeholders(content, lang_config): + """替换标题占位符""" + content = content.replace('{heading_overview}', f"## {lang_config.get('heading_overview', 'Overview')}") + content = content.replace('{heading_inputs}', f"## {lang_config.get('heading_inputs', 'Inputs')}") + content = content.replace('{heading_outputs}', f"## {lang_config.get('heading_outputs', 'Outputs')}") + return content + +def update_param_names(content, node_name, lang, frontend_translations): + """更新参数名称""" + if lang not in frontend_translations: + return content, False + + if node_name not in frontend_translations[lang]: + return content, False + + node_trans = frontend_translations[lang][node_name] + original_content = content + changes_made = [] + + # 更新输入参数名 + if 'inputs' in node_trans: + for param_name, param_data in node_trans['inputs'].items(): + frontend_name = param_data.get('name', '') + if frontend_name and frontend_name != param_name: + # 替换表格中的参数名 + pattern = rf'\|\s*`{re.escape(param_name)}`\s*\|' + replacement = f'| `{frontend_name}` |' + if re.search(pattern, content): + content = re.sub(pattern, replacement, content) + changes_made.append(f"{param_name} → {frontend_name}") + + # 更新输出参数名 + if 'outputs' in node_trans: + lines = content.split('\n') + in_output_section = False + output_row_index = 0 + + for i, line in enumerate(lines): + # 检测输出部分 + if re.match(r'##\s+(?:输出|輸出|出力|출력|Выходы|Salidas|Sorties|Outputs|المخرجات|Çıktılar)', line): + in_output_section = True + output_row_index = 0 + continue + elif line.startswith('##'): + in_output_section = False + continue + + # 更新输出名称 + if in_output_section and line.strip().startswith('|'): + if 'Output Name' in line or 'Data Type' in line or '---' in line: + continue + + output_idx_str = str(output_row_index) + if output_idx_str in node_trans['outputs']: + output_data = node_trans['outputs'][output_idx_str] + frontend_name = output_data.get('name', '') + + if frontend_name: + old_match = re.match(r'(\|\s*)`([^`]+)`(\s*\|)', line) + if old_match: + old_name = old_match.group aufgrund eines Errors unterbrochen wurde, ich muss fortfahren diff --git a/pipeline/scripts/generate_docs.py b/pipeline/scripts/generate_docs.py new file mode 100644 index 000000000..6f26ddc26 --- /dev/null +++ b/pipeline/scripts/generate_docs.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +""" +Auto-extract information from ComfyUI node source code and generate basic documentation +""" + +import os +import re +import ast +import json +from pathlib import Path +from typing import Dict, List, Any, Optional +import inspect +import runtime # noqa: F401 +from lib.paths import MISSING_NODES_REPORT, embedded_docs_dir, load_dotenv + +load_dotenv() + +COMFYUI_PATH = Path(os.getenv('COMFYUI_PATH', '')) +DOCS_PATH = embedded_docs_dir() + + +class NodeInfoExtractor: + """Node information extractor""" + + def __init__(self, file_path: Path): + self.file_path = file_path + self.content = "" + self.node_info = {} + + def read_file(self): + """Read file content""" + try: + with open(self.file_path, 'r', encoding='utf-8') as f: + self.content = f.read() + return True + except Exception as e: + print(f"❌ Unable to read file {self.file_path}: {e}") + return False + + def extract_new_api_node(self, node_name: str) -> Optional[Dict[str, Any]]: + """Extract new API (io.Schema) node information""" + try: + # Find Schema definition corresponding to node_id + pattern = rf'node_id\s*=\s*["\']({re.escape(node_name)})["\']' + match = re.search(pattern, self.content) + if not match: + return None + + # Find corresponding define_schema method + schema_start = match.start() + lines_before = self.content[:schema_start].split('\n') + + # Find class definition + class_name = None + for line in reversed(lines_before): + class_match = re.search(r'class\s+(\w+)', line) + if class_match: + class_name = class_match.group(1) + break + + if not class_name: + return None + + # Extract Schema content + info = { + 'node_name': node_name, + 'class_name': class_name, + 'type': 'new_api', + 'inputs': [], + 'outputs': [], + 'category': None, + 'description': None + } + + # Extract category + category_match = re.search(r'category\s*=\s*["\']([^"\']+)["\']', self.content[schema_start:schema_start+2000]) + if category_match: + info['category'] = category_match.group(1) + + # Extract inputs + inputs_section = re.search(r'inputs\s*=\s*\[(.*?)\]', self.content[schema_start:schema_start+3000], re.DOTALL) + if inputs_section: + inputs_text = inputs_section.group(1) + info['inputs'] = self._parse_io_schema_inputs(inputs_text) + + # Extract outputs + outputs_section = re.search(r'outputs\s*=\s*\[(.*?)\]', self.content[schema_start:schema_start+2000], re.DOTALL) + if outputs_section: + outputs_text = outputs_section.group(1) + info['outputs'] = self._parse_io_schema_outputs(outputs_text) + + return info + + except Exception as e: + print(f"⚠️ Failed to extract new API node {node_name} information: {e}") + return None + + def _parse_io_schema_inputs(self, inputs_text: str) -> List[Dict[str, Any]]: + """Parse io.Schema inputs""" + inputs = [] + + # Match various Input types + # e.g.: io.Clip.Input("clip") + # io.String.Input("tags", multiline=True, default="xxx") + # io.Float.Input("strength", default=1.0, min=0.0, max=10.0) + + input_patterns = [ + r'io\.(\w+)\.Input\(["\']([^"\']+)["\']([^)]*)\)', + ] + + for pattern in input_patterns: + for match in re.finditer(pattern, inputs_text): + data_type = match.group(1).upper() + param_name = match.group(2) + params_str = match.group(3) + + input_info = { + 'name': param_name, + 'type': data_type, + 'required': True, + 'default': None, + 'min': None, + 'max': None, + 'step': None, + 'multiline': False, + 'tooltip': None + } + + # Extract parameters + if 'default' in params_str: + default_match = re.search(r'default\s*=\s*([^,)]+)', params_str) + if default_match: + input_info['default'] = default_match.group(1).strip() + + if 'min' in params_str: + min_match = re.search(r'min\s*=\s*([^,)]+)', params_str) + if min_match: + input_info['min'] = min_match.group(1).strip() + + if 'max' in params_str: + max_match = re.search(r'max\s*=\s*([^,)]+)', params_str) + if max_match: + input_info['max'] = max_match.group(1).strip() + + if 'step' in params_str: + step_match = re.search(r'step\s*=\s*([^,)]+)', params_str) + if step_match: + input_info['step'] = step_match.group(1).strip() + + if 'multiline=True' in params_str: + input_info['multiline'] = True + + if 'tooltip' in params_str: + tooltip_match = re.search(r'tooltip\s*=\s*["\']([^"\']+)["\']', params_str) + if tooltip_match: + input_info['tooltip'] = tooltip_match.group(1) + + inputs.append(input_info) + + return inputs + + def _parse_io_schema_outputs(self, outputs_text: str) -> List[Dict[str, Any]]: + """Parse io.Schema outputs""" + outputs = [] + + # Match io.XXX.Output() + output_pattern = r'io\.(\w+)\.Output\(\)' + + for match in re.finditer(output_pattern, outputs_text): + data_type = match.group(1).upper() + outputs.append({ + 'type': data_type, + 'name': data_type # Default to using type as name + }) + + return outputs + + def extract_classic_node(self, class_name: str) -> Optional[Dict[str, Any]]: + """Extract classic node (INPUT_TYPES) information""" + try: + # Parse using AST + tree = ast.parse(self.content) + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == class_name: + info = { + 'node_name': class_name, + 'class_name': class_name, + 'type': 'classic', + 'inputs': [], + 'outputs': [], + 'category': None, + 'description': None, + 'class_docstring': ast.get_docstring(node), # Extract class docstring + 'return_names': [] + } + + # Find various class attributes + for item in node.body: + if isinstance(item, ast.FunctionDef) and item.name == 'INPUT_TYPES': + info['inputs'] = self._parse_classic_input_types(item) + + elif isinstance(item, ast.Assign): + # Find RETURN_TYPES + for target in item.targets: + if isinstance(target, ast.Name) and target.id == 'RETURN_TYPES': + info['outputs'] = self._parse_classic_return_types(item.value) + + # Find RETURN_NAMES + elif isinstance(target, ast.Name) and target.id == 'RETURN_NAMES': + info['return_names'] = self._parse_classic_return_names(item.value) + + # Find CATEGORY + elif isinstance(target, ast.Name) and target.id == 'CATEGORY': + if isinstance(item.value, ast.Constant): + info['category'] = item.value.value + + # Find DESCRIPTION + elif isinstance(target, ast.Name) and target.id == 'DESCRIPTION': + if isinstance(item.value, ast.Constant): + info['description'] = item.value.value + + return info + + return None + + except Exception as e: + print(f"⚠️ Failed to extract classic node {class_name} information: {e}") + return None + + def _parse_classic_input_types(self, func_node: ast.FunctionDef) -> List[Dict[str, Any]]: + """Parse INPUT_TYPES method - extract using regular expressions""" + inputs = [] + + # Find INPUT_TYPES method source code + func_start = func_node.lineno + func_end = func_node.end_lineno + source_lines = self.content.split('\n') + func_source = '\n'.join(source_lines[func_start-1:func_end]) + + # Extract required and optional parameters + for section in ['required', 'optional']: + section_pattern = rf'"{section}"\s*:\s*\{{([^}}]*?)\}}' + section_match = re.search(section_pattern, func_source, re.DOTALL) + + if section_match: + section_content = section_match.group(1) + # Extract each parameter + param_pattern = r'"([^"]+)"\s*:\s*\((.*?)\)' + + for param_match in re.finditer(param_pattern, section_content, re.DOTALL): + param_name = param_match.group(1) + param_config = param_match.group(2) + + # Extract data type (first element) + type_match = re.search(r'IO\.(\w+)', param_config) + data_type = type_match.group(1).upper() if type_match else 'STRING' + + input_info = { + 'name': param_name, + 'type': data_type, + 'required': (section == 'required'), + 'default': None, + 'min': None, + 'max': None, + 'step': None, + 'options': None, + 'multiline': False, + 'tooltip': None + } + + # Extract values from config dictionary + if 'default' in param_config: + default_match = re.search(r'"default"\s*:\s*([^,}\n]+)', param_config) + if default_match: + input_info['default'] = default_match.group(1).strip().strip('"') + + if 'min' in param_config: + min_match = re.search(r'"min"\s*:\s*([^,}\n]+)', param_config) + if min_match: + input_info['min'] = min_match.group(1).strip() + + if 'max' in param_config: + max_match = re.search(r'"max"\s*:\s*([^,}\n]+)', param_config) + if max_match: + input_info['max'] = max_match.group(1).strip() + + if 'step' in param_config: + step_match = re.search(r'"step"\s*:\s*([^,}\n]+)', param_config) + if step_match: + input_info['step'] = step_match.group(1).strip() + + if 'options' in param_config: + options_match = re.search(r'"options"\s*:\s*\[([^\]]+)\]', param_config) + if options_match: + input_info['options'] = options_match.group(1).strip() + + if 'multiline' in param_config: + input_info['multiline'] = 'True' in param_config + + if 'tooltip' in param_config: + tooltip_match = re.search(r'"tooltip"\s*:\s*"([^"]*)"', param_config) + if tooltip_match: + input_info['tooltip'] = tooltip_match.group(1) + + inputs.append(input_info) + + return inputs + + def _parse_classic_return_types(self, value_node: ast.expr) -> List[Dict[str, Any]]: + """Parse RETURN_TYPES""" + outputs = [] + + if isinstance(value_node, ast.Tuple): + for elt in value_node.elts: + if isinstance(elt, ast.Constant): + outputs.append({ + 'type': elt.value, + 'name': elt.value + }) + elif isinstance(elt, ast.Attribute): + # Handle IO.IMAGE format + if isinstance(elt.value, ast.Name) and elt.value.id == 'IO': + outputs.append({ + 'type': elt.attr.upper(), + 'name': elt.attr.upper() + }) + + return outputs + + def _parse_classic_return_names(self, value_node: ast.expr) -> List[str]: + """Parse RETURN_NAMES""" + names = [] + + if isinstance(value_node, ast.Tuple): + for elt in value_node.elts: + if isinstance(elt, ast.Constant): + names.append(elt.value) + + return names + + +class DocumentGenerator: + """Documentation generator""" + + def __init__(self, node_info: Dict[str, Any]): + self.info = node_info + + def generate_markdown(self) -> str: + """Generate Markdown documentation""" + doc_parts = [] + + # 1. Function description (placeholder) + doc_parts.append(self._generate_description()) + doc_parts.append("") + + # 2. How it works (placeholder) + doc_parts.append("## How It Works") + doc_parts.append("") + doc_parts.append(self._generate_how_it_works()) + doc_parts.append("") + + # 3. Input parameters + if self.info.get('inputs'): + doc_parts.append("## Inputs") + doc_parts.append("") + doc_parts.append(self._generate_inputs_table()) + doc_parts.append("") + + # 4. Output results + if self.info.get('outputs'): + doc_parts.append("## Outputs") + doc_parts.append("") + doc_parts.append(self._generate_outputs_table()) + doc_parts.append("") + + return "\n".join(doc_parts) + + def _generate_description(self) -> str: + """Generate function description""" + node_name = self.info['node_name'] + category = self.info.get('category', 'utility') + + # Basic description template + return f"This node performs operations in the {category} category. [Description to be added]" + + def _generate_how_it_works(self) -> str: + """Generate how it works explanation""" + return "[Detailed explanation of how this node works to be added]" + + def _generate_inputs_table(self) -> str: + """Generate input parameters table""" + if not self.info.get('inputs'): + return "" + + lines = [ + "| Parameter | Description | Data Type | Input Type | Default | Range |", + "|-----------|-------------|-----------|------------|---------|-------|" + ] + + for inp in self.info['inputs']: + param_name = f"`{inp['name']}`" + data_type = inp['type'] + input_type = "Multiline text" if inp.get('multiline') else "Widget" + default = inp.get('default', '-') + + # Build range + range_str = '-' + if inp.get('min') is not None and inp.get('max') is not None: + range_str = f"{inp['min']} - {inp['max']}" + if inp.get('step'): + range_str += f" (step: {inp['step']})" + + description = inp.get('tooltip', '[Description to be added]') + + lines.append(f"| {param_name} | {description} | {data_type} | {input_type} | {default} | {range_str} |") + + return "\n".join(lines) + + def _generate_outputs_table(self) -> str: + """Generate output results table""" + if not self.info.get('outputs'): + return "" + + lines = [ + "| Output Name | Description | Data Type |", + "|-------------|-------------|-----------|" + ] + + for idx, out in enumerate(self.info['outputs'], 1): + output_name = f"`{out['name']}`" + data_type = out['type'] + description = "[Description to be added]" + + lines.append(f"| {output_name} | {description} | {data_type} |") + + return "\n".join(lines) + + + +def generate_doc_for_node(node_name: str, file_path: Path, node_type: str) -> bool: + """Generate documentation for a single node""" + print(f"📝 Processing node: {node_name}") + + # 1. Extract node information + extractor = NodeInfoExtractor(file_path) + if not extractor.read_file(): + return False + + if node_type == 'new_api': + node_info = extractor.extract_new_api_node(node_name) + else: + node_info = extractor.extract_classic_node(node_name) + + if not node_info: + print(f" ⚠️ Unable to extract node information") + return False + + # 2. Generate documentation + generator = DocumentGenerator(node_info) + markdown_content = generator.generate_markdown() + + # 3. Save documentation + doc_dir = DOCS_PATH / node_name + doc_dir.mkdir(parents=True, exist_ok=True) + + doc_file = doc_dir / "en.md" + with open(doc_file, 'w', encoding='utf-8') as f: + f.write(markdown_content) + + print(f" ✅ Documentation generated: {doc_file}") + + # 4. Save extracted raw information (for debugging) + info_file = doc_dir / "_node_info.json" + with open(info_file, 'w', encoding='utf-8') as f: + json.dump(node_info, f, indent=2, ensure_ascii=False) + + return True + + +def main(): + """Main function""" + import sys + + print("=" * 80) + print("ComfyUI Node Documentation Auto-Generation Tool") + print("=" * 80) + print() + + # Read missing nodes report + report_file = MISSING_NODES_REPORT + if not report_file.exists(): + print("❌ missing_nodes_report.json not found. Please run scan_missing_nodes.py first.") + return + + with open(report_file, 'r', encoding='utf-8') as f: + report = json.load(f) + + missing_nodes = report['missing_nodes'] + print(f"📊 Total missing node documentations: {len(missing_nodes)}\n") + + # Parse options from command line arguments + nodes_to_process = [] + + if len(sys.argv) > 1: + mode = sys.argv[1] + if mode == 'all': + nodes_to_process = missing_nodes + elif mode == 'test': + count = int(sys.argv[2]) if len(sys.argv) > 2 else 10 + nodes_to_process = missing_nodes[:count] + elif mode == 'node': + if len(sys.argv) < 3: + print("❌ Please specify node name: python3 generate_docs.py node ") + return + node_name = sys.argv[2] + nodes_to_process = [n for n in missing_nodes if n['name'] == node_name] + if not nodes_to_process: + print(f"❌ Node not found: {node_name}") + return + else: + print("❌ Invalid argument") + print("Usage:") + print(" python3 generate_docs.py test [count] # Generate docs for first N nodes (default 10)") + print(" python3 generate_docs.py all # Generate docs for all missing nodes") + print(" python3 generate_docs.py node # Generate docs for the specified node") + return + else: + # Default: generate docs for the first 5 missing nodes for testing + print("💡 Default test mode: generating docs for the first 5 missing nodes") + print(" Use argument: test [count] | all | node ") + print() + nodes_to_process = missing_nodes[:5] + + print() + print(f"🚀 Starting to generate documentation for {len(nodes_to_process)} nodes...") + print() + + # Generate documentation + success_count = 0 + failed_count = 0 + + for node in nodes_to_process: + node_name = node['name'] + file_path = COMFYUI_PATH / node['file'] + node_type = node['type'] + + try: + if generate_doc_for_node(node_name, file_path, node_type): + success_count += 1 + else: + failed_count += 1 + except Exception as e: + print(f" ❌ Generation failed: {e}") + failed_count += 1 + + print() + + # Summary + print("=" * 80) + print("📊 Generation completed") + print("=" * 80) + print(f"✅ Success: {success_count}") + print(f"❌ Failed: {failed_count}") + print() + + +if __name__ == "__main__": + main() + diff --git a/pipeline/scripts/migrate_docs_format.py b/pipeline/scripts/migrate_docs_format.py new file mode 100644 index 000000000..b807b5609 --- /dev/null +++ b/pipeline/scripts/migrate_docs_format.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +""" +One-time migration for embedded-docs markdown files: + +1. Move AI disclaimer blockquote from top to bottom (before SHA footer if present). +2. Reorder parameter / output tables so Description is the second column. + +Usage: + python3 migrate_docs_format.py --dry-run # preview changes + python3 migrate_docs_format.py # apply to all docs + python3 migrate_docs_format.py --node KSampler # single node +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import runtime # noqa: F401 +from lib.doc_disclaimer import ( + compose_document, + create_en_disclaimer, + create_translated_disclaimer, + strip_ai_disclaimer, +) +from lib.hash_footer import ( + SOURCE_HASH_FOOTER_RE, + extract_english_source_fingerprint_hex, + format_source_hash_footer, + strip_source_hash_footer, +) +from lib.paths import TRANSLATION_CONFIG, default_embedded_docs_path, embedded_docs_dir, load_dotenv + +load_dotenv() + +DOCS_PATH = embedded_docs_dir() + +# Header cell (normalized) -> role for column reordering +_DESCRIPTION_HINTS = ( + "description", + "descripción", + "descripcion", + "descrição", + "descricao", + "beschreibung", + "описание", + "açıklama", + "aciklama", + "descriere", + "描述", + "说明", + "說明", + "説明", + "설명", + "الوصف", + "توضیح", + "توضیحات", + "descrição da função", + "descripción de la función", + "function description", + "descrição da função", +) + +_TABLE_ROW_RE = re.compile(r"^\s*\|") +_SEPARATOR_RE = re.compile(r"^\s*\|?[\s:\-|]+\|?\s*$") + + +def _normalize_header(cell: str) -> str: + return re.sub(r"\s+", " ", cell.strip().lower()) + + +def _parse_table_cells(line: str) -> Optional[List[str]]: + if not _TABLE_ROW_RE.match(line): + return None + inner = line.strip() + if inner.startswith("|"): + inner = inner[1:] + if inner.endswith("|"): + inner = inner[:-1] + return [c.strip() for c in inner.split("|")] + + +def _is_separator_row(line: str) -> bool: + cells = _parse_table_cells(line) + if not cells: + return False + return all(re.fullmatch(r":?-{2,}:?", c.strip()) or not c.strip() for c in cells) + + +def _description_col_index(headers: List[str]) -> Optional[int]: + for i, cell in enumerate(headers): + norm = _normalize_header(cell) + if any(h in norm for h in _DESCRIPTION_HINTS): + return i + return None + + +def _reorder_table_row(cells: List[str], order: List[int]) -> str: + padded = cells + [""] * (max(order) + 1 - len(cells)) + reordered = [padded[i] if i < len(padded) else "" for i in order] + return "| " + " | ".join(reordered) + " |" + + +def _migrate_table_block(lines: List[str]) -> Tuple[List[str], bool]: + """Reorder one markdown table so column 1 is Description (column 0 unchanged).""" + if len(lines) < 2: + return lines, False + headers = _parse_table_cells(lines[0]) + if not headers or len(headers) < 3: + return lines, False + + desc_idx = _description_col_index(headers) + if desc_idx is None or desc_idx == 1: + return lines, False + + order = [0, desc_idx] + [i for i in range(len(headers)) if i not in (0, desc_idx)] + out: List[str] = [_reorder_table_row(headers, order)] + + i = 1 + if i < len(lines) and _is_separator_row(lines[i]): + out.append("| " + " | ".join(["---"] * len(order)) + " |") + i += 1 + + changed = True + while i < len(lines): + line = lines[i] + if not _TABLE_ROW_RE.match(line): + break + if _is_separator_row(line): + i += 1 + continue + cells = _parse_table_cells(line) + if cells: + out.append(_reorder_table_row(cells, order)) + else: + out.append(line) + i += 1 + + return out, changed + + +def migrate_tables(content: str) -> Tuple[str, int]: + """Reorder all tables where Description is not already the second column.""" + lines = content.split("\n") + out: List[str] = [] + i = 0 + tables_changed = 0 + + while i < len(lines): + line = lines[i] + headers = _parse_table_cells(line) + if headers and len(headers) >= 3 and _description_col_index(headers) is not None: + block = [line] + j = i + 1 + while j < len(lines) and (_TABLE_ROW_RE.match(lines[j]) or not lines[j].strip()): + if _TABLE_ROW_RE.match(lines[j]): + block.append(lines[j]) + elif not lines[j].strip() and block: + break + j += 1 + migrated, changed = _migrate_table_block(block) + out.extend(migrated) + if changed: + tables_changed += 1 + i = i + len(block) + continue + out.append(line) + i += 1 + + return "\n".join(out), tables_changed + + +def _load_translation_config() -> Dict[str, dict]: + if not TRANSLATION_CONFIG.exists(): + return {} + with open(TRANSLATION_CONFIG, encoding="utf-8") as f: + return json.load(f) + + +def _disclaimer_for_lang(node_name: str, lang: str, lang_config: Dict[str, dict]) -> str: + if lang == "en": + return create_en_disclaimer(node_name) + if lang in lang_config: + return create_translated_disclaimer(lang, node_name, lang_config[lang]) + return create_en_disclaimer(node_name).replace("/en.md", f"/{lang}.md") + + +def _extract_footer(content: str) -> str: + m = SOURCE_HASH_FOOTER_RE.search(content) + return m.group(0) if m else "" + + +def migrate_file( + path: Path, + node_name: str, + lang: str, + lang_config: Dict[str, dict], +) -> Tuple[bool, str]: + """ + Returns (changed, summary). + """ + original = path.read_text(encoding="utf-8") + footer_hex = extract_english_source_fingerprint_hex(original) + footer = format_source_hash_footer(footer_hex) if footer_hex else _extract_footer(original) + + body = strip_ai_disclaimer(strip_source_hash_footer(original)) + body, table_count = migrate_tables(body) + + disclaimer = _disclaimer_for_lang(node_name, lang, lang_config) + new_content = compose_document(body, disclaimer, footer) + + if new_content == original: + return False, "unchanged" + + path.write_text(new_content, encoding="utf-8") + parts = ["disclaimer→bottom"] + if table_count: + parts.append(f"tables={table_count}") + return True, ", ".join(parts) + + +def iter_doc_files(docs_path: Path, node_filter: Optional[str] = None) -> List[Tuple[Path, str, str]]: + """Yield (path, node_name, lang_code).""" + if not docs_path.exists(): + return [] + out: List[Tuple[Path, str, str]] = [] + for node_dir in sorted(docs_path.iterdir()): + if not node_dir.is_dir(): + continue + if node_filter and node_dir.name != node_filter: + continue + for md in sorted(node_dir.glob("*.md")): + out.append((md, node_dir.name, md.stem)) + return out + + +def main() -> int: + parser = argparse.ArgumentParser(description="Migrate embedded-docs: disclaimer to bottom + table columns") + parser.add_argument("--dry-run", action="store_true", help="Preview only; do not write files") + parser.add_argument("--node", type=str, help="Migrate a single node directory") + parser.add_argument( + "--docs-path", + type=Path, + default=DOCS_PATH, + help=f"Docs root (default: {DOCS_PATH})", + ) + args = parser.parse_args() + docs_path = args.docs_path + + if not docs_path.exists(): + print(f"ERROR: docs path not found: {docs_path}") + return 1 + + lang_config = _load_translation_config() + files = iter_doc_files(docs_path, args.node) + if not files: + print("No markdown files found.") + return 1 + + changed_files = 0 + unchanged_files = 0 + tables_total = 0 + errors: List[str] = [] + + print(f"Migrating {len(files)} file(s) under {docs_path} (dry_run={args.dry_run})") + + for path, node_name, lang in files: + try: + original = path.read_text(encoding="utf-8") + footer_hex = extract_english_source_fingerprint_hex(original) + footer = format_source_hash_footer(footer_hex) if footer_hex else _extract_footer(original) + body = strip_ai_disclaimer(strip_source_hash_footer(original)) + body, table_count = migrate_tables(body) + disclaimer = _disclaimer_for_lang(node_name, lang, lang_config) + new_content = compose_document(body, disclaimer, footer) + + if new_content == original: + unchanged_files += 1 + continue + + tables_total += table_count + changed_files += 1 + rel = path.relative_to(docs_path) + had_top = original.lstrip().startswith(">") + print(f" {'[dry-run] ' if args.dry_run else ''}✓ {rel} (tables={table_count}, was_top_disclaimer={had_top})") + + if not args.dry_run: + path.write_text(new_content, encoding="utf-8") + except Exception as e: + errors.append(f"{path}: {e}") + + print() + print("=" * 60) + print(f"Changed: {changed_files}") + print(f"Unchanged: {unchanged_files}") + print(f"Tables reordered (total): {tables_total}") + if errors: + print(f"Errors: {len(errors)}") + for err in errors[:20]: + print(f" - {err}") + return 1 + if args.dry_run and changed_files: + print("\nRe-run without --dry-run to apply.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline/scripts/prepare_ai_input.py b/pipeline/scripts/prepare_ai_input.py new file mode 100644 index 000000000..80e67035b --- /dev/null +++ b/pipeline/scripts/prepare_ai_input.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +""" +Prepare AI input: Extract node source code and metadata +""" + +import argparse +import os +import re +import json +from pathlib import Path +from typing import Dict, List, Any, Optional +import runtime # noqa: F401 +from lib.node_source_extract import ( + extract_node_source_code as _extract_node_source_impl, + extract_node_class_source, + register_source_extract_cli_args, + extract_config_from_parsed_args, +) +from lib.paths import ( + AI_INPUT_DIR, + ALL_NODES_INFO, + DOC_RULES, + MISSING_NODES_REPORT, + default_embedded_docs_path, + embedded_docs_dir, + load_dotenv, +) +from version_tracker import NodeVersionTracker + +load_dotenv() + +COMFYUI_PATH = Path(os.getenv('COMFYUI_PATH', '')) +DOCS_PATH = embedded_docs_dir() +OUTPUT_PATH = AI_INPUT_DIR + +# Initialize version tracker +version_tracker = NodeVersionTracker(OUTPUT_PATH) + + +def extract_node_source_code( + file_path: Path, + class_name: str, + node_type: str, + *, + extract_config=None, +) -> Optional[str]: + """Same-file preamble (imports/constants/helpers) + node class — see node_source_extract.py.""" + text = _extract_node_source_impl( + file_path, + class_name, + node_type, + comfy_root=COMFYUI_PATH, + config=extract_config, + ) + if not (text or "").strip(): + return None + return text + + +def extract_basic_info(source_code: str, node_name: str) -> Dict[str, Any]: + """Extract basic information from the source code""" + info = { + 'node_name': node_name, + 'category': None, + 'description': None, + 'docstring': None, + } + + # Extract category + category_match = re.search(r'CATEGORY\s*=\s*["\']([^"\']+)["\']', source_code) + if category_match: + info['category'] = category_match.group(1) + else: + category_match = re.search(r'category\s*=\s*["\']([^"\']+)["\']', source_code) + if category_match: + info['category'] = category_match.group(1) + + # Extract docstring (class level documentation) + docstring_match = re.search(r'class\s+\w+[^:]*:\s*(?:"""([^"]*?)"""|\'\'\'([^\']*?)\'\'\')', source_code, re.DOTALL) + if docstring_match: + info['docstring'] = (docstring_match.group(1) or docstring_match.group(2)).strip() + + # Extract DESCRIPTION + desc_match = re.search(r'DESCRIPTION\s*=\s*["\']([^"\']+)["\']', source_code) + if desc_match: + info['description'] = desc_match.group(1) + + return info + + +def create_ai_prompt(node_name: str, source_code: str, basic_info: Dict, doc_rules: str) -> str: + """Generate complete AI prompt for documentation""" + + github_link = f"https://github.com/Comfy-Org/embedded-docs/blob/main/comfyui_embedded_docs/docs/{node_name}/en.md" + + prompt = f"""# Task: Generate ComfyUI Node Documentation + +## Node Information + +**Node Name:** {node_name} +**Category:** {basic_info.get('category', 'Unknown')} + +## Node Source Code + +```python +{source_code} +``` + +## Extracted Information + +- **Docstring:** {basic_info.get('docstring', 'None')} +- **Description:** {basic_info.get('description', 'None')} + +## Documentation Requirements + +{doc_rules} + +## Your Task + +Please generate a complete English documentation (en.md) for this node following the structure: + +Generate the documentation with: + +1. **Overview** - A concise 1-3 sentence explanation of what this node does and how it works (extract from docstring if available) +2. **Inputs** - Complete parameter table with all input parameters +3. **Outputs** - Output table with return values + +### Table Format: + +**Inputs table:** +| Parameter | Description | Data Type | Required | Range | +|-----------|-------------|-----------|----------|-------| + +- Required column: Use "Yes" for required parameters, "No" for optional parameters +- Check the source code: parameters in "required" section are "Yes", in "optional" section are "No" +- For new API: check if `optional=True` in parameter definition + +**Outputs table:** +| Output Name | Description | Data Type | +|-------------|-------------|-----------| + +### Important Guidelines: + +- Keep data types in ENGLISH (IMAGE, STRING, INT, FLOAT, MODEL, CONDITIONING, etc.) +- Use backticks (`) for parameter names in tables +- Write in clear, non-technical language for general users +- Extract all information from the source code, including tooltips +- Use tooltip text EXACTLY as provided for parameter descriptions +- If a parameter has a default value, mention it in the Description column (e.g., "default: 1.0") +- If the node has a docstring or description, use it as foundation for your explanation +- Keep descriptions FACTUAL - avoid speculation or assumptions +- Do NOT add usage tips or best practices sections +- Do NOT include the AI disclaimer blockquote (it is appended automatically at the bottom of the file) +- Do NOT include a level-one heading (# title) — the node display name is prepended automatically from frontend translations +- Start directly with the overview paragraph (1-3 sentences), then ## Inputs and ## Outputs +- Focus on objective functionality based on the source code + +### CRITICAL: Parameter Constraints and Limitations + +**Carefully analyze the source code for parameter constraints:** + +1. **Numeric Limits**: Check for max/min values, batch limits (e.g., `max=8` → "maximum 8 items") +2. **Parameter Dependencies**: Look for if/else logic showing: + - Required combinations (e.g., "both image AND mask required") + - Conditional requirements (e.g., "when mode=X, param Y is needed") + - Mutual exclusions (e.g., "cannot use A and B together") +3. **Validation Logic**: Look for exceptions/errors in the code showing constraints +4. **Size Matching**: Note any dimension/shape matching requirements + +**Document constraints in:** +- Parameter descriptions (for individual constraints) +- A note after the Inputs table (for complex multi-parameter constraints) + +**Special Formatting for COMBO Parameters:** +- For parameters with multiple options, use `
` tags in the Range column to separate options +- Example Range column: `"option1"
"option2"
"option3"` +- In Description column, explain what each option does if they have different meanings +- Use `
` in Description too if needed for clarity + +Example: If code shows `if image is not None and mask is not None: path = "/edit"` +→ Document: "When both `image` and `mask` are provided, the node switches to editing mode" + +Please provide the complete markdown content for en.md. +""" + + return prompt + + +def prepare_node_for_ai( + node_name: str, + file_path: Path, + node_type: str, + class_name: str, + *, + extract_config=None, +) -> bool: + """Prepare AI input for a single node.""" + print(f"📝 Preparing node: {node_name}") + + # 1. Extract source code + source_code = extract_node_source_code( + file_path, + class_name, + node_type, + extract_config=extract_config, + ) + if not source_code: + print(f" ⚠️ Failed to extract source code") + return False + + # Fingerprint only the node class body so preamble / cross-file context does not perturb hashes. + class_fingerprint_src = extract_node_class_source(file_path, class_name) + if not (class_fingerprint_src or "").strip(): + print(f" ⚠️ Failed to extract node class body for versioning hash") + return False + + # 2. Check for version changes (class body only — same as ai_input/source_code bundle context differs) + is_changed, old_hash = version_tracker.check_node_changed(node_name, class_fingerprint_src) + if is_changed and old_hash: + print(f" 🔄 Source code changed (old hash: {old_hash[:16]}...)") + + # 3. Record version information + version_info = version_tracker.record_node_version( + node_name, + class_fingerprint_src, + metadata={ + "file_path": str(file_path.relative_to(COMFYUI_PATH)), + "node_type": node_type, + "class_name": class_name + } + ) + + # 4. Extract basic info + basic_info = extract_basic_info(source_code, node_name) + + # 5. Add version info to basic_info + basic_info["version_info"] = { + "extracted_at": version_info["extracted_at"], + "source_hash": version_info["source_hash"], + "source_length": version_info["source_length"], + "hash_scope": "node_class_body", + } + + # 6. Read documentation rules + rules_file = DOC_RULES + if rules_file.exists(): + with open(rules_file, 'r', encoding='utf-8') as f: + doc_rules = f.read() + else: + doc_rules = "Follow standard documentation practices." + + # 7. Create AI prompt + ai_prompt = create_ai_prompt(node_name, source_code, basic_info, doc_rules) + + # 8. Save to output directory + output_dir = OUTPUT_PATH / node_name + output_dir.mkdir(parents=True, exist_ok=True) + + # Save source code + with open(output_dir / "source_code.py", 'w', encoding='utf-8') as f: + f.write(source_code) + + # Save basic info + with open(output_dir / "basic_info.json", 'w', encoding='utf-8') as f: + json.dump(basic_info, f, indent=2, ensure_ascii=False) + + # Save AI prompt + with open(output_dir / "ai_prompt.txt", 'w', encoding='utf-8') as f: + f.write(ai_prompt) + + print(f" ✅ Prepared: {output_dir}") + return True + + +def create_batch_file(nodes_data: List[Dict]) -> None: + """Create batch file containing information of all processed nodes""" + batch_file = OUTPUT_PATH / "batch_nodes.json" + + with open(batch_file, 'w', encoding='utf-8') as f: + json.dump(nodes_data, f, indent=2, ensure_ascii=False) + + print(f"\n✅ Batch file created: {batch_file}") + + +def main(): + """Main function""" + import argparse + + pre = argparse.ArgumentParser( + description=( + "Prepare ai_input from ComfyUI sources. " + "Positional arguments: test [count] | all | node | changed " + "(omit for default: first 5 missing)." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="Tip: put all options before positional words, e.g.\n" + " python3 prepare_ai_input.py --source-preamble slim --source-resolve 0 node CheckpointLoaderSimple", + ) + register_source_extract_cli_args(pre) + known, rest = pre.parse_known_args() + extract_cfg = extract_config_from_parsed_args(known) + + print("=" * 80) + print("ComfyUI Node AI Input Preparation Tool") + print("=" * 80) + print() + + # Create output directory + OUTPUT_PATH.mkdir(parents=True, exist_ok=True) + + # Read missing nodes report + report_file = MISSING_NODES_REPORT + if not report_file.exists(): + print("❌ missing_nodes_report.json not found. Please run scan_missing_nodes.py first") + return + + with open(report_file, "r", encoding="utf-8") as f: + report = json.load(f) + + missing_nodes = report["missing_nodes"] + changed_nodes = report.get("changed_nodes", []) + + # Also load all_nodes_info.json for forced single-node updates on nodes that + # already have documentation (not present in missing_nodes). + all_nodes_info_file = ALL_NODES_INFO + all_nodes_info = {} + if all_nodes_info_file.exists(): + with open(all_nodes_info_file, "r", encoding="utf-8") as f: + raw = json.load(f) + # Normalise: the file is {"total": N, "nodes": {...}} or a flat dict + nodes_dict = raw.get("nodes", raw) + for nid, meta in nodes_dict.items(): + all_nodes_info[nid] = { + "name": nid, + "file": meta["file"], + "type": meta["type"], + "class_name": meta.get("class_name", nid), + } + + # Filter nodes that don't have documentation yet + nodes_without_docs = [] + for node in missing_nodes: + node_name = node["name"] + doc_file = DOCS_PATH / node_name / "en.md" + if not doc_file.exists(): + nodes_without_docs.append(node) + + print(f"📊 Total missing nodes: {len(missing_nodes)}") + print(f"📝 Nodes definitely missing documentation: {len(nodes_without_docs)}") + print(f"🔄 Nodes with changed source code: {len(changed_nodes)}\n") + + # Select nodes to process based on command arguments + nodes_to_process = [] + + if len(rest) > 0: + mode = rest[0] + if mode == "all": + nodes_to_process = nodes_without_docs + elif mode == "test": + count = int(rest[1]) if len(rest) > 1 else 10 + nodes_to_process = nodes_without_docs[:count] + elif mode == "node": + if len(rest) < 2: + print("❌ Please specify node name: python3 prepare_ai_input.py node ") + return + node_name = rest[1] + # First look in missing_nodes; if not found, fall back to all_nodes_info so + # that nodes with existing docs can also be force-refreshed from source. + nodes_to_process = [n for n in missing_nodes if n["name"] == node_name] + if not nodes_to_process: + if node_name in all_nodes_info: + nodes_to_process = [all_nodes_info[node_name]] + print(f" ℹ️ Node already has docs — re-preparing from latest source code: {node_name}") + else: + print(f"❌ Node not found in missing_nodes_report.json or all_nodes_info.json: {node_name}") + return + elif mode == "changed": + if not changed_nodes: + print("✅ No changed nodes to prepare.") + return + nodes_to_process = changed_nodes + print(f"🔄 Changed mode: will re-prepare {len(nodes_to_process)} node(s) with updated source code\n") + elif mode in ("regenerate-all", "everything"): + if not all_nodes_info: + print("❌ all_nodes_info.json missing or empty. Run scan_missing_nodes.py first.") + return + items = sorted(all_nodes_info.values(), key=lambda x: x["name"]) + if len(rest) > 1: + try: + lim = int(rest[1]) + if lim <= 0: + print("❌ Limit must be a positive integer.") + return + items = items[:lim] + print(f" ℹ️ Limit active: first {lim} nodes only\n") + except ValueError: + print(f"❌ Invalid limit (expected integer): {rest[1]!r}") + return + nodes_to_process = items + print( + "⚙️ regenerate-all: preparing AI input for every node in all_nodes_info.json " + f"({len(nodes_to_process)} node(s))\n" + ) + else: + print("❌ Invalid argument") + print("Usage:") + print(" python3 prepare_ai_input.py [OPTIONS] test [count] # Prepare first N nodes (default 10)") + print(" python3 prepare_ai_input.py [OPTIONS] all # Prepare all missing nodes") + print(" python3 prepare_ai_input.py [OPTIONS] node # Prepare specified node") + print(" python3 prepare_ai_input.py [OPTIONS] changed # Re-prepare nodes with changed source") + print(" python3 prepare_ai_input.py [OPTIONS] regenerate-all [limit] # Prepare ALL scanned nodes") + print(" Options: --help (see --source-* flags)") + return + else: + # Default: prepare first 5 nodes without documentation + print("💡 Default test mode: preparing first 5 nodes without documentation") + print(" Use: [OPTIONS] test [count] | all | node | changed | regenerate-all [limit]") + print() + nodes_to_process = nodes_without_docs[:5] + + print(f"🚀 Starting to prepare AI input for {len(nodes_to_process)} nodes...\n") + + # Prepare nodes + success_count = 0 + failed_count = 0 + prepared_nodes = [] + + for node in nodes_to_process: + node_name = node["name"] + file_path = COMFYUI_PATH / node["file"] + node_type = node["type"] + class_name = node.get("class_name", node_name) + + try: + if prepare_node_for_ai( + node_name, + file_path, + node_type, + class_name, + extract_config=extract_cfg, + ): + success_count += 1 + prepared_nodes.append( + { + "node_name": node_name, + "file": node["file"], + "type": node_type, + "ai_input_dir": str(OUTPUT_PATH / node_name), + } + ) + else: + failed_count += 1 + except Exception as e: + print(f" ❌ Preparation failed: {e}") + failed_count += 1 + + # Create batch file + if prepared_nodes: + create_batch_file(prepared_nodes) + + # Summary + print("\n" + "=" * 80) + print("📊 Preparation complete") + print("=" * 80) + print(f"✅ Success: {success_count}") + print(f"❌ Failed: {failed_count}") + print(f"\n📁 Output directory: {OUTPUT_PATH}") + print("\n💡 Next step: use the files in the ai_input directory to generate documentation") + print() + + +if __name__ == "__main__": + main() + diff --git a/pipeline/scripts/prepare_translation.py b/pipeline/scripts/prepare_translation.py new file mode 100644 index 000000000..58e32a954 --- /dev/null +++ b/pipeline/scripts/prepare_translation.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" +Prepare translation batch list from missing_nodes_report.json +Verifies that files truly don't exist before adding to batch +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import ( + MISSING_NODES_REPORT, + TRANSLATION_BATCHES_DIR, + TRANSLATION_CONFIG, + embedded_docs_dir, + load_dotenv, +) + +load_dotenv() + +DOCS_PATH = embedded_docs_dir() +REPORT_FILE = MISSING_NODES_REPORT +TRANSLATION_OUTPUT_DIR = TRANSLATION_BATCHES_DIR +TRANSLATION_CONFIG_FILE = TRANSLATION_CONFIG + +def main(): + """Main function""" + parser = argparse.ArgumentParser(description="Prepare translation batch from missing_nodes_report.json") + parser.add_argument("--lang", required=True, help="Target language code (e.g. zh, es)") + parser.add_argument("--mode", choices=("test", "all"), default="all", + help="test = first N nodes, all = all missing (default: all)") + parser.add_argument("--count", type=int, default=20, + help="Max nodes in test mode (default: 20)") + parser.add_argument( + "--force-all-nodes", + action="store_true", + help=( + "Build batch from every node that has en.md under docs (sorted by name), " + "ignoring missing_nodes_report.json. Use with batch_translate_docs.py --force to overwrite all translations." + ), + ) + args = parser.parse_args() + target_lang = args.lang + mode = args.mode + count = args.count + force_all_nodes = args.force_all_nodes + + # Load translation config to validate language + with open(TRANSLATION_CONFIG_FILE, 'r', encoding='utf-8') as f: + translation_config = json.load(f) + + if not target_lang or target_lang not in translation_config: + print("Error: Please specify a valid target language with --lang") + print(f"Available languages: {', '.join(translation_config.keys())}") + sys.exit(1) + + lang_info = translation_config[target_lang] + + print("=" * 80) + print("ComfyUI Translation Batch Preparation") + print("=" * 80) + print(f"\nTarget language: {lang_info['name']} ({target_lang})") + print(f"Mode: {mode}") + if mode == "test": + print(f"Count: {count}") + print(f"Force-all nodes (ignore missing report): {force_all_nodes}") + print() + + # Whole-tree batches: every folder with en.md + if force_all_nodes: + if not DOCS_PATH.exists(): + print(f"❌ Error: Docs path not found: {DOCS_PATH}") + sys.exit(1) + all_with_en = sorted( + p.name for p in DOCS_PATH.iterdir() + if p.is_dir() and (p / "en.md").exists() + ) + if mode == "test": + nodes_to_process = all_with_en[:count] + print(f"🚀 Test mode + force-all-nodes: batch size {len(nodes_to_process)} (have en.md)") + else: + nodes_to_process = all_with_en + print(f"🚀 Full mode + force-all-nodes: batch size {len(nodes_to_process)} (have en.md)") + TRANSLATION_OUTPUT_DIR.mkdir(exist_ok=True) + batch_data = { + "target_language": target_lang, + "language_name": lang_info["name"], + "nodes": nodes_to_process, + "total": len(nodes_to_process), + "truly_missing": len(nodes_to_process), + "already_exist": 0, + "force_all_nodes": True, + } + batch_file = TRANSLATION_OUTPUT_DIR / f"batch_{target_lang}.json" + with open(batch_file, "w", encoding="utf-8") as f: + json.dump(batch_data, f, indent=2, ensure_ascii=False) + print("\n" + "=" * 80) + print("📊 Batch Preparation Summary (force-all-nodes)") + print("=" * 80) + print(f"✅ Nodes in batch: {len(nodes_to_process)}") + print(f"📋 Batch file: {batch_file}") + print(f"\n💡 Next: python3 batch_translate_docs.py --lang {target_lang} --mode {mode} --count {count} --force") + print("=" * 80) + return + + # Load report + if not REPORT_FILE.exists(): + print(f"❌ Error: Report file not found: {REPORT_FILE}") + print(" Please run: python3 scan_missing_nodes.py first") + sys.exit(1) + + with open(REPORT_FILE, 'r', encoding='utf-8') as f: + report = json.load(f) + + # Get nodes missing target language from JSON + print(f"📊 Reading missing translations from {REPORT_FILE.name}...") + nodes_from_json = [] + for doc in report.get('incomplete_docs', []): + if target_lang in doc.get('missing_languages', []): + nodes_from_json.append(doc['node']) + + print(f" Found {len(nodes_from_json)} nodes marked as missing {target_lang} in JSON") + + # Verify which ones truly don't have the translation file + print(f"\n🔍 Verifying actual file status...") + nodes_truly_missing = [] + nodes_already_exist = [] + + for node_name in nodes_from_json: + # Check if English source exists + source_file = DOCS_PATH / node_name / "en.md" + if not source_file.exists(): + print(f" ⚠️ Skipping {node_name}: No English source") + continue + + # Check if target translation exists + target_file = DOCS_PATH / node_name / f"{target_lang}.md" + if target_file.exists(): + nodes_already_exist.append(node_name) + else: + nodes_truly_missing.append(node_name) + + print(f" ✅ Truly missing: {len(nodes_truly_missing)}") + print(f" ⏭️ Already exist: {len(nodes_already_exist)}") + + # Update JSON to remove nodes that already have translations + if nodes_already_exist: + print(f"\n🔄 Updating {REPORT_FILE.name} to remove {len(nodes_already_exist)} nodes that already exist...") + updated_incomplete_docs = [] + for doc in report.get('incomplete_docs', []): + if doc['node'] in nodes_already_exist and target_lang in doc.get('missing_languages', []): + # Remove this language from missing_languages + doc['missing_languages'] = [lang for lang in doc['missing_languages'] if lang != target_lang] + # Only keep if still has missing languages + if doc.get('missing_languages'): + updated_incomplete_docs.append(doc) + + report['incomplete_docs'] = updated_incomplete_docs + + # Save updated report + with open(REPORT_FILE, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2, ensure_ascii=False) + print(f" ✅ Updated {REPORT_FILE.name}") + + if not nodes_truly_missing: + print(f"\n✅ All nodes already have {target_lang} translation!") + # Write empty batch so batch_translate_docs can run and do nothing (avoids reusing old batch) + TRANSLATION_OUTPUT_DIR.mkdir(exist_ok=True) + batch_data = { + 'target_language': target_lang, + 'language_name': lang_info['name'], + 'nodes': [], + 'total': 0, + 'truly_missing': 0, + 'already_exist': len(nodes_already_exist) + } + batch_file = TRANSLATION_OUTPUT_DIR / f"batch_{target_lang}.json" + with open(batch_file, 'w', encoding='utf-8') as f: + json.dump(batch_data, f, indent=2, ensure_ascii=False) + print(f"📋 Empty batch written: {batch_file}") + return + + # Determine which nodes to process + if mode == "test": + nodes_to_process = nodes_truly_missing[:count] + print(f"\n🚀 Test mode: Preparing batch for {len(nodes_to_process)} nodes") + else: + nodes_to_process = nodes_truly_missing + print(f"\n🚀 Full mode: Preparing batch for all {len(nodes_to_process)} nodes") + + # Create output directory + TRANSLATION_OUTPUT_DIR.mkdir(exist_ok=True) + + # Create batch file + batch_data = { + 'target_language': target_lang, + 'language_name': lang_info['name'], + 'nodes': nodes_to_process, + 'total': len(nodes_to_process), + 'truly_missing': len(nodes_truly_missing), + 'already_exist': len(nodes_already_exist) + } + + batch_file = TRANSLATION_OUTPUT_DIR / f"batch_{target_lang}.json" + with open(batch_file, 'w', encoding='utf-8') as f: + json.dump(batch_data, f, indent=2, ensure_ascii=False) + + print("\n" + "=" * 80) + print("📊 Batch Preparation Summary") + print("=" * 80) + print(f"✅ Nodes to translate: {len(nodes_to_process)}") + print(f"📋 Batch file: {batch_file}") + print(f"\n💡 Next: python3 batch_translate_docs.py --lang {target_lang} --mode {mode} --count {count}") + print("=" * 80) + +if __name__ == "__main__": + main() diff --git a/pipeline/scripts/replace_placeholders.py b/pipeline/scripts/replace_placeholders.py new file mode 100644 index 000000000..86644b975 --- /dev/null +++ b/pipeline/scripts/replace_placeholders.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +""" +替换文档中的占位符为对应语言的标题 +用法: python replace_placeholders.py [--check-only] +""" + +import os +import re +import sys +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import embedded_docs_dir + +DOCS_ROOT = embedded_docs_dir() + +# 不同语言的标题映射 +HEADING_TRANSLATIONS = { + 'en': { + 'heading_overview': '## Overview', + 'heading_inputs': '## Inputs', + 'heading_outputs': '## Outputs', + 'heading_usage': '## Usage', + 'heading_examples': '## Examples', + }, + 'zh': { + 'heading_overview': '## 概述', + 'heading_inputs': '## 输入', + 'heading_outputs': '## 输出', + 'heading_usage': '## 用法', + 'heading_examples': '## 示例', + }, + 'es': { + 'heading_overview': '## Descripción general', + 'heading_inputs': '## Entradas', + 'heading_outputs': '## Salidas', + 'heading_usage': '## Uso', + 'heading_examples': '## Ejemplos', + }, + 'fr': { + 'heading_overview': '## Aperçu', + 'heading_inputs': '## Entrées', + 'heading_outputs': '## Sorties', + 'heading_usage': '## Utilisation', + 'heading_examples': '## Exemples', + }, + 'ja': { + 'heading_overview': '## 概要', + 'heading_inputs': '## 入力', + 'heading_outputs': '## 出力', + 'heading_usage': '## 使用方法', + 'heading_examples': '## 例', + }, + 'ko': { + 'heading_overview': '## 개요', + 'heading_inputs': '## 입력', + 'heading_outputs': '## 출력', + 'heading_usage': '## 사용법', + 'heading_examples': '## 예시', + }, + 'ru': { + 'heading_overview': '## Обзор', + 'heading_inputs': '## Входы', + 'heading_outputs': '## Выходы', + 'heading_usage': '## Использование', + 'heading_examples': '## Примеры', + }, +} + +def get_language_from_filename(filename): + """从文件名获取语言代码""" + stem = Path(filename).stem + return stem if stem in HEADING_TRANSLATIONS else None + +def replace_placeholders_in_file(filepath, check_only=False): + """替换文件中的占位符""" + lang = get_language_from_filename(filepath.name) + if not lang: + return False, [] + + translations = HEADING_TRANSLATIONS[lang] + + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + + modified = False + replacements = [] + + for placeholder, heading in translations.items(): + pattern = '{' + placeholder + '}' + if pattern in content: + replacements.append(f" {filepath.relative_to(DOCS_ROOT.parent.parent)}: {placeholder} -> {heading}") + content = content.replace(pattern, heading) + modified = True + + if modified and not check_only: + with open(filepath, 'w', encoding='utf-8') as f: + f.write(content) + + return modified, replacements + +def main(): + check_only = '--check-only' in sys.argv + + files_with_placeholders = [] + all_replacements = [] + + for root, _, files in os.walk(DOCS_ROOT): + for fname in files: + if fname.endswith('.md') or fname.endswith('.mdx'): + fpath = Path(root) / fname + modified, replacements = replace_placeholders_in_file(fpath, check_only) + + if modified: + files_with_placeholders.append(fpath) + all_replacements.extend(replacements) + + if files_with_placeholders: + if check_only: + print(f"\n发现 {len(files_with_placeholders)} 个文件包含占位符：") + for repl in all_replacements: + print(repl) + print(f"\n运行不带 --check-only 参数来替换这些占位符。") + sys.exit(1) + else: + print(f"\n已替换 {len(files_with_placeholders)} 个文件中的占位符：") + for repl in all_replacements: + print(repl) + print(f"\n✓ 完成！") + else: + print("✓ 未发现需要替换的占位符。") + +if __name__ == '__main__': + main() + diff --git a/pipeline/scripts/runtime.py b/pipeline/scripts/runtime.py new file mode 100644 index 000000000..8093d0f52 --- /dev/null +++ b/pipeline/scripts/runtime.py @@ -0,0 +1,8 @@ +"""Bootstrap: ensure repo root is on sys.path when running scripts directly.""" + +import sys +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) diff --git a/pipeline/scripts/scan_missing_nodes.py b/pipeline/scripts/scan_missing_nodes.py new file mode 100644 index 000000000..a1110b1cb --- /dev/null +++ b/pipeline/scripts/scan_missing_nodes.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +""" +Scan ComfyUI codebase to find all nodes and identify missing documentation +""" + +import os +import re +import ast +import hashlib +from pathlib import Path +from typing import Set, Dict, List +import json +import runtime # noqa: F401 +from lib.node_source_extract import extract_node_class_source +from lib.paths import ( + ALL_NODES_INFO, + MISSING_NODES_REPORT, + NODE_VERSIONS, + embedded_docs_dir, + ensure_data_dir, + load_dotenv, +) + +load_dotenv() + +COMFYUI_PATH = Path(os.getenv('COMFYUI_PATH', '')) +DOCS_PATH = embedded_docs_dir() + +# Python file paths to scan +SCAN_PATHS = [ + COMFYUI_PATH / "nodes.py", + COMFYUI_PATH / "comfy_extras", + COMFYUI_PATH / "comfy_api_nodes", +] + + +class NodeScanner: + """Node scanner to identify all ComfyUI nodes""" + + def __init__(self): + self.all_nodes: Set[str] = set() + self.node_info: Dict[str, Dict] = {} + self.class_mappings: Dict[str, str] = {} # mapping_name -> class_name + + def scan_file_for_nodes(self, file_path: Path): + """Scan a Python file to extract node definitions""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Extract NODE_CLASS_MAPPINGS first (this is the authoritative source) + mappings = self._extract_node_class_mappings(content) + self.class_mappings.update(mappings) + + # Method 1: Find new API nodes with io.Schema node_id + schema_pattern = r'node_id\s*=\s*["\']([^"\']+)["\']' + for match in re.finditer(schema_pattern, content): + node_id = match.group(1) + class_name = self._find_class_name(content, match.start()) + category = self._extract_category_near_position(content, match.start(), window=2000) + if not category and class_name != "Unknown": + category = self._extract_category_classic(content, class_name) + # Use NODE_CLASS_MAPPINGS name if available, otherwise use node_id + node_name = node_id + for mapping_name, mapped_class in mappings.items(): + if mapped_class == class_name: + node_name = mapping_name + break + self.all_nodes.add(node_name) + info = { + 'file': str(file_path.relative_to(COMFYUI_PATH)), + 'type': 'new_api', + 'class_name': class_name, + 'node_id': node_id + } + if category: + info['category'] = category + self.node_info[node_name] = info + + # Method 2: Find classic nodes with INPUT_TYPES method + try: + tree = ast.parse(content) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + # Check if class has INPUT_TYPES method + has_input_types = any( + isinstance(item, ast.FunctionDef) and item.name == "INPUT_TYPES" + for item in node.body + ) + if has_input_types: + class_name = node.name + # Skip base classes and abstract classes + if not class_name.startswith('_') and class_name not in ['ComfyNode', 'Base', 'ComfyNodeABC']: + # Use NODE_CLASS_MAPPINGS name if available + node_name = class_name + for mapping_name, mapped_class in mappings.items(): + if mapped_class == class_name: + node_name = mapping_name + break + + self.all_nodes.add(node_name) + category = self._extract_category_classic(content, class_name) + info = { + 'file': str(file_path.relative_to(COMFYUI_PATH)), + 'type': 'classic', + 'class_name': class_name + } + if category: + info['category'] = category + self.node_info[node_name] = info + except SyntaxError: + print(f"⚠️ Syntax error, skipping AST parsing: {file_path}") + + except Exception as e: + print(f"❌ Failed to read file {file_path}: {e}") + + def _find_class_name(self, content: str, pos: int) -> str: + """Find class name by searching backwards from position""" + lines_before = content[:pos].split('\n') + for line in reversed(lines_before): + match = re.search(r'class\s+(\w+)', line) + if match: + return match.group(1) + return "Unknown" + + def _extract_category_near_position(self, content: str, pos: int, window: int = 2000) -> str: + """Extract category= or CATEGORY = from content near position (Schema or class block). + + For Schema-style nodes the layout is always: + IO.Schema( + node_id="NodeName", + category="...", ← comes AFTER node_id in the same call + ) + So we search in a small window AFTER pos first to avoid picking up + category= values from neighbouring nodes that appear before pos. + Only fall back to the full bidirectional window if nothing is found after. + """ + # Priority: search after pos (catches Schema-style nodes reliably) + after_block = content[pos:min(len(content), pos + 600)] + m = re.search(r'category\s*=\s*["\']([^"\']+)["\']', after_block) + if m: + return m.group(1).strip() + # Fallback: full bidirectional window (classic CATEGORY = "..." may come before) + start = max(0, pos - window) + end = min(len(content), pos + window) + block = content[start:end] + m = re.search(r'CATEGORY\s*=\s*["\']([^"\']+)["\']', block) + if m: + return m.group(1).strip() + return "" + + def _extract_category_classic(self, content: str, class_name: str) -> str: + """Extract CATEGORY from classic node class block (and optionally from base class in same file).""" + class_pattern = re.compile( + r"^class\s+" + re.escape(class_name) + r"\s*[:(].*?(?=^class\s|\Z)", + re.MULTILINE | re.DOTALL, + ) + m = class_pattern.search(content) + if not m: + return "" + block = m.group(0) + # CATEGORY = "..." + c = re.search(r"CATEGORY\s*=\s*[\"']([^\"']+)[\"']", block) + if c: + return c.group(1).strip() + # Schema-style in same block + c = re.search(r"category\s*=\s*[\"']([^\"']+)[\"']", block) + if c: + return c.group(1).strip() + # Base class: get parent and look for category in base's block (e.g. ImageProcessingNode) + base_m = re.search(r"^class\s+" + re.escape(class_name) + r"\s*\(\s*(\w+)", content, re.MULTILINE) + if base_m: + base_name = base_m.group(1) + if base_name not in ("ComfyNode", "io.ComfyNode", "IO.ComfyNode"): + base_pattern = re.compile( + r"^class\s+" + re.escape(base_name) + r"\s*[:(].*?(?=^class\s|\Z)", + re.MULTILINE | re.DOTALL, + ) + b = base_pattern.search(content) + if b: + base_block = b.group(0) + bc = re.search(r"category\s*=\s*[\"']([^\"']+)[\"']", base_block) + if bc: + return bc.group(1).strip() + bc = re.search(r"CATEGORY\s*=\s*[\"']([^\"']+)[\"']", base_block) + if bc: + return bc.group(1).strip() + return "" + + def _extract_node_class_mappings(self, content: str) -> Dict[str, str]: + """Extract NODE_CLASS_MAPPINGS dictionary from file content""" + mappings = {} + + # Find NODE_CLASS_MAPPINGS = { ... } + pattern = r'NODE_CLASS_MAPPINGS\s*=\s*\{([^}]+)\}' + match = re.search(pattern, content, re.DOTALL) + + if match: + mappings_content = match.group(1) + # Extract "key": ClassName pairs + pair_pattern = r'["\']([^"\']+)["\']\s*:\s*(\w+)' + for pair_match in re.finditer(pair_pattern, mappings_content): + mapping_name = pair_match.group(1) + class_name = pair_match.group(2) + mappings[mapping_name] = class_name + + return mappings + + def scan_directory(self, dir_path: Path): + """Recursively scan all Python files in directory""" + if not dir_path.exists(): + print(f"⚠️ Path does not exist: {dir_path}") + return + + for py_file in dir_path.rglob("*.py"): + # Skip some special files + if '__pycache__' in str(py_file) or py_file.name.startswith('test_'): + continue + self.scan_file_for_nodes(py_file) + + def scan_all(self): + """Scan all configured paths""" + print("🔍 Starting to scan ComfyUI nodes...") + + for path in SCAN_PATHS: + if path.is_file(): + print(f"📄 Scanning file: {path.name}") + self.scan_file_for_nodes(path) + elif path.is_dir(): + print(f"📁 Scanning directory: {path.name}") + self.scan_directory(path) + + print(f"✅ Scan completed, found {len(self.all_nodes)} nodes\n") + + def save_all_nodes_info(self, output_path: Path) -> None: + """Save node_name -> { file, type, class_name, category?, ... } for all scanned nodes (for sync script).""" + data = { + "total": len(self.all_nodes), + "nodes": {name: self.node_info.get(name, {}) for name in sorted(self.all_nodes)}, + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +class DocumentationChecker: + """Documentation checker""" + + def __init__(self, docs_path: Path): + self.docs_path = docs_path + self.documented_nodes: Set[str] = set() + + def scan_existing_docs(self): + """Scan existing documentation folders""" + print("📚 Scanning existing documentation...") + + if not self.docs_path.exists(): + print(f"❌ Documentation path does not exist: {self.docs_path}") + return + + for item in self.docs_path.iterdir(): + if item.is_dir() and not item.name.startswith('.'): + self.documented_nodes.add(item.name) + + print(f"✅ Found {len(self.documented_nodes)} existing documentation\n") + + def check_doc_completeness(self, node_name: str) -> Dict[str, bool]: + """Check documentation completeness for a single node""" + node_dir = self.docs_path / node_name + languages = ['en', 'zh', 'zh-TW', 'es', 'fr', 'ja', 'ko', 'ru', 'ar', 'tr', 'pt-BR', 'fa'] + + completeness = {} + for lang in languages: + doc_file = node_dir / f"{lang}.md" + completeness[lang] = doc_file.exists() + + return completeness + + +def main(): + """Main function""" + print("=" * 80) + print("ComfyUI Node Documentation Missing Scanner") + print("=" * 80) + print() + + ensure_data_dir() + + # 1. Scan all nodes + scanner = NodeScanner() + scanner.scan_all() + + # Save full node info (including category) for sync script + all_nodes_path = ALL_NODES_INFO + scanner.save_all_nodes_info(all_nodes_path) + print(f"💾 All nodes info (with category) saved to: {all_nodes_path}\n") + + # 2. Scan existing documentation + doc_checker = DocumentationChecker(DOCS_PATH) + doc_checker.scan_existing_docs() + + # 3. Comparison analysis + print("=" * 80) + print("📊 Analysis Results") + print("=" * 80) + print() + + missing_nodes = scanner.all_nodes - doc_checker.documented_nodes + extra_docs = doc_checker.documented_nodes - scanner.all_nodes + + print(f"🔢 Statistics:") + print(f" - Total nodes in code: {len(scanner.all_nodes)}") + print(f" - Nodes with documentation: {len(doc_checker.documented_nodes)}") + print(f" - Nodes missing documentation: {len(missing_nodes)}") + print(f" - Extra documentation: {len(extra_docs)} (possibly deprecated nodes)") + print() + + # 4. Output missing nodes list + if missing_nodes: + print("❌ Nodes missing documentation:") + print("-" * 80) + + # Group by file + nodes_by_file = {} + for node in sorted(missing_nodes): + if node in scanner.node_info: + file_path = scanner.node_info[node]['file'] + if file_path not in nodes_by_file: + nodes_by_file[file_path] = [] + nodes_by_file[file_path].append(node) + + for file_path, nodes in sorted(nodes_by_file.items()): + print(f"\n📄 {file_path}") + for node in sorted(nodes): + info = scanner.node_info.get(node, {}) + node_type = info.get('type', 'unknown') + print(f" - {node} ({node_type})") + + print() + else: + print("✅ Great! All nodes have documentation!") + print() + + # 5. Output extra documentation list + if extra_docs: + print("⚠️ Possibly deprecated documentation (no corresponding node found in code):") + print("-" * 80) + for doc in sorted(extra_docs): + print(f" - {doc}") + print() + + # 6. Check completeness of existing documentation (only for nodes that already have docs) + print("=" * 80) + print("📋 Checking language completeness of existing documentation") + print("=" * 80) + print(f" (Only nodes that already have an en.md are checked. The {len(missing_nodes)} missing above have no doc yet.)") + print() + + incomplete_docs = [] + for node in doc_checker.documented_nodes: + completeness = doc_checker.check_doc_completeness(node) + missing_langs = [lang for lang, exists in completeness.items() if not exists] + if missing_langs: + incomplete_docs.append((node, missing_langs)) + + if incomplete_docs: + print(f"⚠️ Found {len(incomplete_docs)} document(s) missing certain language versions:") + print() + for node, missing_langs in sorted(incomplete_docs)[:10]: # Show only first 10 + langs_str = ", ".join(missing_langs) + print(f" - {node}: Missing {langs_str}") + + if len(incomplete_docs) > 10: + print(f" ... and {len(incomplete_docs) - 10} more nodes") + print() + else: + print("✅ Among existing docs: all contain all 7 language versions!") + print(" (The nodes missing documentation above still need en.md generated first.)") + print() + + # 7. Detect changed nodes (documented nodes whose source hash differs from recorded) + print("=" * 80) + print("🔄 Checking for changed nodes (source code hash comparison)") + print("=" * 80) + print(" Note: Extracting latest source code directly from ComfyUI to detect changes") + print() + + version_db_path = NODE_VERSIONS + version_db = {} + if version_db_path.exists(): + with open(version_db_path, 'r', encoding='utf-8') as f: + version_db = json.load(f).get("nodes", {}) + + changed_nodes = [] + extraction_failed = [] + + for node_name in sorted(doc_checker.documented_nodes): + if node_name not in version_db: + continue + recorded_hash = version_db[node_name].get("current_hash") + if not recorded_hash: + continue + + # Extract current source DIRECTLY from ComfyUI source code (not from cache) + # This ensures we detect changes even if ai_input cache is stale + info = scanner.node_info.get(node_name, {}) + if not info: + continue + + file_path = COMFYUI_PATH / info.get('file', '') + if not file_path.exists(): + extraction_failed.append(node_name) + continue + + node_type = info.get('type', 'classic') + class_name = info.get('class_name', node_name) + + # Class-only fingerprint: hash must not move when only contextual/cross-file extraction changes. + current_class_src = extract_node_class_source(file_path, class_name) + if not current_class_src: + extraction_failed.append(node_name) + continue + + current_hash = hashlib.sha256(current_class_src.encode('utf-8')).hexdigest() + if current_hash != recorded_hash: + changed_nodes.append({ + "name": node_name, + "file": info.get('file', 'unknown'), + "type": node_type, + "class_name": class_name, + "old_hash": recorded_hash[:16] + "...", + "new_hash": current_hash[:16] + "...", + }) + + if changed_nodes: + print(f"⚠️ Found {len(changed_nodes)} node(s) with changed source code:") + for n in changed_nodes: + print(f" - {n['name']} ({n['old_hash']} → {n['new_hash']})") + print() + else: + print("✅ No source code changes detected among documented nodes\n") + + if extraction_failed: + print(f"⚠️ Failed to extract source for {len(extraction_failed)} node(s):") + for name in extraction_failed[:5]: + print(f" - {name}") + if len(extraction_failed) > 5: + print(f" ... and {len(extraction_failed) - 5} more") + print() + + # 8. Save results to JSON file + output_file = MISSING_NODES_REPORT + report = { + "scan_time": str(Path(__file__).stat().st_mtime), + "total_nodes": len(scanner.all_nodes), + "documented_nodes": len(doc_checker.documented_nodes), + "missing_count": len(missing_nodes), + "changed_count": len(changed_nodes), + "missing_nodes": [ + { + "name": node, + "file": scanner.node_info.get(node, {}).get('file', 'unknown'), + "type": scanner.node_info.get(node, {}).get('type', 'unknown'), + "class_name": scanner.node_info.get(node, {}).get('class_name', 'unknown'), + } + for node in sorted(missing_nodes) + ], + "changed_nodes": changed_nodes, + "extra_docs": sorted(list(extra_docs)), + "incomplete_docs": [ + {"node": node, "missing_languages": langs} + for node, langs in sorted(incomplete_docs) + ] + } + + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2, ensure_ascii=False) + + print("=" * 80) + print(f"💾 Detailed report saved to: {output_file}") + print("=" * 80) + + +if __name__ == "__main__": + main() + diff --git a/pipeline/scripts/sync_frontend_translations.py b/pipeline/scripts/sync_frontend_translations.py new file mode 100644 index 000000000..7fa27da22 --- /dev/null +++ b/pipeline/scripts/sync_frontend_translations.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +""" +从前端仓库的 nodeDefs.json 同步参数翻译到文档 +用法: python sync_frontend_translations.py +示例: python sync_frontend_translations.py /path/to/ComfyUI_frontend +""" + +import json +import os +import sys +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import NODE_TRANSLATIONS, embedded_docs_dir, load_dotenv + +load_dotenv() + +DOCS_ROOT = embedded_docs_dir() + +# 支持的语言 +SUPPORTED_LANGS = ['en', 'zh', 'zh-TW', 'es', 'fr', 'ja', 'ko', 'ru', 'ar', 'tr', 'pt-BR', 'fa'] + +def load_frontend_translations(frontend_path, lang): + """从前端仓库加载指定语言的翻译""" + locale_file = Path(frontend_path) / 'src' / 'locales' / lang / 'nodeDefs.json' + + if not locale_file.exists(): + print(f"⚠️ 警告: 未找到 {lang} 语言文件: {locale_file}") + return {} + + try: + with open(locale_file, 'r', encoding='utf-8') as f: + return json.load(f) + except Exception as e: + print(f"❌ 读取 {lang} 语言文件失败: {e}") + return {} + +def get_node_translations(frontend_translations, node_name): + """获取节点的翻译信息""" + if node_name not in frontend_translations: + return None + + node_data = frontend_translations[node_name] + translations = { + 'display_name': node_data.get('display_name', ''), + 'description': node_data.get('description', ''), + 'inputs': {}, + 'outputs': {} + } + + # 提取输入参数翻译 + if 'inputs' in node_data: + for param_name, param_data in node_data['inputs'].items(): + if isinstance(param_data, dict): + translations['inputs'][param_name] = { + 'name': param_data.get('name', param_name), + 'tooltip': param_data.get('tooltip', '') + } + + # 提取输出翻译 + if 'outputs' in node_data: + for output_idx, output_data in node_data['outputs'].items(): + if isinstance(output_data, dict): + translations['outputs'][output_idx] = { + 'name': output_data.get('name', ''), + 'tooltip': output_data.get('tooltip', '') + } + + return translations + +def create_translation_report(frontend_path): + """生成翻译对照报告""" + print(f"\n正在从前端仓库加载翻译: {frontend_path}\n") + + # 加载所有语言的翻译 + all_translations = {} + for lang in SUPPORTED_LANGS: + all_translations[lang] = load_frontend_translations(frontend_path, lang) + + # 获取所有节点名称（从文档目录） + node_dirs = [d for d in DOCS_ROOT.iterdir() if d.is_dir()] + + print(f"找到 {len(node_dirs)} 个节点文档目录\n") + print("=" * 80) + + # 为每个节点生成翻译报告 + for node_dir in sorted(node_dirs): + node_name = node_dir.name + + # 检查是否有对应的前端翻译 + has_translation = any(node_name in all_translations[lang] for lang in SUPPORTED_LANGS) + + if not has_translation: + print(f"\n⚠️ {node_name}: 未找到前端翻译") + continue + + print(f"\n✓ {node_name}") + print("-" * 80) + + # 显示各语言的参数翻译 + for lang in SUPPORTED_LANGS: + if node_name in all_translations[lang]: + trans = get_node_translations(all_translations[lang], node_name) + if trans and trans['inputs']: + print(f"\n [{lang.upper()}] 参数翻译:") + for param_name, param_trans in trans['inputs'].items(): + print(f" - {param_name}: {param_trans['name']}") + if param_trans['tooltip']: + print(f" 提示: {param_trans['tooltip'][:60]}...") + +def export_translation_json(frontend_path, output_file=None): + """导出所有节点的翻译为JSON文件，便于后续使用""" + all_translations = {} + for lang in SUPPORTED_LANGS: + all_translations[lang] = load_frontend_translations(frontend_path, lang) + + output_path = NODE_TRANSLATIONS if output_file is None else Path(output_file) + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(all_translations, f, ensure_ascii=False, indent=2) + + print(f"\n✓ 翻译数据已导出到: {output_path}") + +def main(): + if len(sys.argv) < 2: + print("用法: python sync_frontend_translations.py [--export]") + print("示例: python sync_frontend_translations.py /path/to/ComfyUI_frontend") + print("\n选项:") + print(" --export 导出翻译为JSON文件") + sys.exit(1) + + frontend_path = Path(sys.argv[1]) + + if not frontend_path.exists(): + print(f"❌ 错误: 前端仓库路径不存在: {frontend_path}") + sys.exit(1) + + if '--export' in sys.argv: + export_translation_json(frontend_path) + else: + create_translation_report(frontend_path) + +if __name__ == '__main__': + main() + diff --git a/scripts/sync_to_docs.py b/pipeline/scripts/sync_to_comfy_docs.py similarity index 98% rename from scripts/sync_to_docs.py rename to pipeline/scripts/sync_to_comfy_docs.py index 18760f357..563a11ff7 100644 --- a/scripts/sync_to_docs.py +++ b/pipeline/scripts/sync_to_comfy_docs.py @@ -18,9 +18,14 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Set, Tuple -# ALL_NODES_INFO_PATH: optional scanner output (node_name -> { category, ... }). -# When absent, category lookup falls back to ComfyUI source extraction. -ALL_NODES_INFO_PATH = Path(os.getenv("ALL_NODES_INFO", "")) if os.getenv("ALL_NODES_INFO") else Path("") +import runtime # noqa: F401 +from lib.paths import ALL_NODES_INFO, default_embedded_docs_path, load_dotenv + +load_dotenv() + +ALL_NODES_INFO_PATH = ALL_NODES_INFO + +# Cache: node_name -> category (first segment). Loaded from scanner output if available. _nodes_info_cache: Optional[Dict[str, Dict[str, Any]]] = None @@ -41,13 +46,9 @@ def _load_all_nodes_info() -> Dict[str, Dict[str, Any]]: _nodes_info_cache = {} return _nodes_info_cache -# --- Self-contained path resolution ------------------------------------- -# This script is maintained inside the embedded-docs repo (scripts/sync_to_docs.py), -# so the source docs live next to it. Target (Comfy-Org/docs checkout) and -# ComfyUI source are provided via env vars, mirroring the local pipeline setup. -_SCRIPT_DIR = Path(__file__).resolve().parent.parent -EMBEDDED_DOCS_PATH = _SCRIPT_DIR +EMBEDDED_DOCS_PATH = default_embedded_docs_path() COMFYUI_PATH = Path(os.getenv("COMFYUI_PATH", "")) +_SCRIPT_DIR = Path(__file__).resolve().parent.parent TARGET_DOCS = Path(os.getenv("TARGET_DOCS", _SCRIPT_DIR / ".." / "docs")) DOCS_SOURCE = EMBEDDED_DOCS_PATH / "comfyui_embedded_docs" / "docs" diff --git a/pipeline/scripts/update_param_translations.py b/pipeline/scripts/update_param_translations.py new file mode 100644 index 000000000..ae3043e3a --- /dev/null +++ b/pipeline/scripts/update_param_translations.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +""" +Update parameter names in documentation to match frontend translations +Uses node_translations.json exported from frontend nodeDefs +""" + +import json +import os +import re +import sys +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import NODE_TRANSLATIONS, embedded_docs_dir, load_dotenv + +load_dotenv() + +DOCS_ROOT = embedded_docs_dir().resolve() +TRANSLATIONS_FILE = NODE_TRANSLATIONS + +# Supported languages +SUPPORTED_LANGS = ['zh', 'zh-TW', 'es', 'fr', 'ja', 'ko', 'ru', 'ar', 'tr', 'pt-BR', 'fa'] + +def load_frontend_translations(): + """Load frontend translations from exported JSON""" + if not TRANSLATIONS_FILE.exists(): + print(f"❌ Error: {TRANSLATIONS_FILE} not found") + print(" Please run: python sync_frontend_translations.py --export") + sys.exit(1) + + with open(TRANSLATIONS_FILE, 'r', encoding='utf-8') as f: + return json.load(f) + +def extract_table_rows(content, table_type='inputs'): + """Extract rows from Inputs or Outputs table""" + # Find the table section + if table_type == 'inputs': + pattern = r'##\s+(?:输入|輸入|入力|입력|Входы|Entradas|Entrées|Inputs|المدخلات|Girdiler|ورودی‌ها)\s*\n\n(.*?)(?=\n##|\Z)' + else: + pattern = r'##\s+(?:输出|輸出|出力|출력|Выходы|Salidas|Sorties|Outputs|المخرجات|Çıktılar|خروجی‌ها)\s*\n\n(.*?)(?=\n##|\Z)' + + match = re.search(pattern, content, re.DOTALL) + if not match: + return None, None + + table_section = match.group(1) + table_start = match.start(1) + + # Extract table lines + lines = table_section.strip().split('\n') + if len(lines) < 3: # Must have header, separator, and at least one row + return None, None + + return lines, table_start + +def update_parameter_name_in_row(row, old_param_name, new_param_name): + """Update parameter name in a table row while preserving backticks and structure""" + # Parameter name is typically in the first column with backticks + pattern = rf'\|\s*`{re.escape(old_param_name)}`\s*\|' + replacement = f'| `{new_param_name}` |' + return re.sub(pattern, replacement, row) + +def update_doc_with_translations(doc_file, node_name, lang, frontend_translations): + """Update a documentation file with frontend translations""" + + # Get translations for this node and language + if lang not in frontend_translations: + return False, "Language not in translations" + + if node_name not in frontend_translations[lang]: + return False, "Node not in frontend translations" + + node_trans = frontend_translations[lang][node_name] + + # Read current documentation + with open(doc_file, 'r', encoding='utf-8') as f: + content = f.read() + + original_content = content + changes_made = [] + + # Update input parameter names + if 'inputs' in node_trans: + for param_name, param_data in node_trans['inputs'].items(): + if not isinstance(param_data, dict): + continue + frontend_name = param_data.get('name', '') + if frontend_name and frontend_name != param_name: + # Try to find and replace the parameter name in the table + old_pattern = f'`{param_name}`' + if old_pattern in content: + # Only replace in table rows (lines starting with |) + lines = content.split('\n') + for i, line in enumerate(lines): + if line.strip().startswith('|') and old_pattern in line: + # This is a table row with the parameter + lines[i] = line.replace(f'`{param_name}`', f'`{frontend_name}`', 1) + changes_made.append(f"[Input] {param_name} → {frontend_name}") + content = '\n'.join(lines) + + # Update output parameter names + if 'outputs' in node_trans: + lines = content.split('\n') + in_output_section = False + output_row_index = 0 # Track which output row we're on (0-indexed) + + for i, line in enumerate(lines): + # Detect if we're in the Outputs section + if re.match(r'##\s+(?:输出|輸出|出力|출력|Выходы|Salidas|Sorties|Outputs|المخرجات|Çıktılar)', line): + in_output_section = True + output_row_index = 0 + continue + elif line.startswith('##'): + in_output_section = False + continue + + # Update output names in the table (skip header and separator rows) + if in_output_section and line.strip().startswith('|'): + # Skip table header and separator + if '---' in line: + continue + + # Skip any row where the first column doesn't have a backtick (header in any language) + if not re.match(r'\|\s*`', line): + continue + + # This is an actual data row + output_idx_str = str(output_row_index) + if output_idx_str in node_trans['outputs']: + output_data = node_trans['outputs'][output_idx_str] + if not isinstance(output_data, dict): + output_row_index += 1 + continue + frontend_name = output_data.get('name', '') + + if frontend_name: + # Replace the output name in the first column + # Format: | `OldName` | DataType | Description | + old_match = re.match(r'(\|\s*)`([^`]+)`(\s*\|)', line) + if old_match: + old_name = old_match.group(2) + lines[i] = re.sub( + r'(\|\s*)`[^`]+`(\s*\|)', + f'\\1`{frontend_name}`\\2', + line, + count=1 + ) + changes_made.append(f"[Output] {old_name} → {frontend_name}") + + output_row_index += 1 + + content = '\n'.join(lines) + + # Save if changes were made + if content != original_content: + with open(doc_file, 'w', encoding='utf-8') as f: + f.write(content) + return True, changes_made + + return False, [] + +def main(): + """Main function""" + + # Parse arguments + target_lang = None + target_node = None + dry_run = False + + for i, arg in enumerate(sys.argv[1:]): + if arg == '--lang': + target_lang = sys.argv[i + 2] if i + 2 < len(sys.argv) else None + elif arg == '--node': + target_node = sys.argv[i + 2] if i + 2 < len(sys.argv) else None + elif arg == '--dry-run': + dry_run = True + + print("=" * 80) + print("Parameter Translation Updater") + print("=" * 80) + print(f"Docs root: {DOCS_ROOT}") + print(f"Translations file: {TRANSLATIONS_FILE}") + print(f"Target language: {target_lang or 'ALL'}") + print(f"Target node: {target_node or 'ALL'}") + print(f"Mode: {'Dry run (preview only)' if dry_run else 'Update files'}") + print("=" * 80) + print() + + # Load frontend translations + print("📖 Loading frontend translations...") + frontend_trans = load_frontend_translations() + print(f" Loaded translations for {len(SUPPORTED_LANGS)} languages\n") + + # Get list of nodes to process + if target_node: + node_dirs = [DOCS_ROOT / target_node] + if not node_dirs[0].exists(): + print(f"❌ Error: Node directory not found: {node_dirs[0]}") + sys.exit(1) + else: + node_dirs = [d for d in DOCS_ROOT.iterdir() if d.is_dir()] + + # Process each node + total_updated = 0 + total_skipped = 0 + + for node_dir in sorted(node_dirs): + node_name = node_dir.name + + # Determine which languages to process + langs_to_process = [target_lang] if target_lang else SUPPORTED_LANGS + + for lang in langs_to_process: + if lang == 'en': # Skip English (it's the source) + continue + + doc_file = node_dir / f"{lang}.md" + if not doc_file.exists(): + continue + + # Update document + if not dry_run: + updated, changes = update_doc_with_translations(doc_file, node_name, lang, frontend_trans) + + if updated: + print(f"✅ Updated {node_name} ({lang}): {', '.join(changes)}") + total_updated += 1 + else: + total_skipped += 1 + else: + # Dry run - just check + _, changes = update_doc_with_translations(doc_file, node_name, lang, frontend_trans) + if changes: + print(f"🔍 Would update {node_name} ({lang}): {', '.join(changes)}") + total_updated += 1 + else: + total_skipped += 1 + + print("\n" + "=" * 80) + print("📊 Summary") + print("=" * 80) + print(f"✅ Updated: {total_updated}") + print(f"⏭️ Skipped: {total_skipped}") + if dry_run: + print("\n💡 Run without --dry-run to apply changes") + print("=" * 80) + +if __name__ == '__main__': + main() + diff --git a/pipeline/scripts/update_translation_status.py b/pipeline/scripts/update_translation_status.py new file mode 100644 index 000000000..76d5566b0 --- /dev/null +++ b/pipeline/scripts/update_translation_status.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Update translation status in missing_nodes_report.json +After completing translations, remove the language from missing_languages +""" + +import json +import sys +from pathlib import Path + +import runtime # noqa: F401 +from lib.paths import MISSING_NODES_REPORT + +REPORT_FILE = MISSING_NODES_REPORT + + +def update_translation_status(node_name: str, lang: str): + """Remove a language from a node's missing_languages list""" + + if not REPORT_FILE.exists(): + print(f"❌ Error: Report file not found: {REPORT_FILE}") + return False + + with open(REPORT_FILE, 'r', encoding='utf-8') as f: + report = json.load(f) + + # Find the node and remove the language + updated = False + for doc in report.get('incomplete_docs', []): + if doc['node'] == node_name: + if lang in doc.get('missing_languages', []): + doc['missing_languages'].remove(lang) + updated = True + break + + if updated: + # Save updated report + with open(REPORT_FILE, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2, ensure_ascii=False) + return True + + return False + + +def batch_update_translations(completed_nodes: dict): + """ + Batch update translation status + completed_nodes: {lang: [node1, node2, ...]} + """ + + if not REPORT_FILE.exists(): + print(f"❌ Error: Report file not found: {REPORT_FILE}") + return False + + with open(REPORT_FILE, 'r', encoding='utf-8') as f: + report = json.load(f) + + # Update each language's completed nodes + for lang, nodes in completed_nodes.items(): + for node_name in nodes: + for doc in report.get('incomplete_docs', []): + if doc['node'] == node_name: + if lang in doc.get('missing_languages', []): + doc['missing_languages'].remove(lang) + + # Remove nodes that have no missing languages + report['incomplete_docs'] = [ + doc for doc in report.get('incomplete_docs', []) + if len(doc.get('missing_languages', [])) > 0 + ] + + # Save updated report + with open(REPORT_FILE, 'w', encoding='utf-8') as f: + json.dump(report, f, indent=2, ensure_ascii=False) + + print(f"✅ Updated translation status in {REPORT_FILE}") + return True + + +if __name__ == "__main__": + # For testing + if len(sys.argv) > 2: + node = sys.argv[1] + lang = sys.argv[2] + if update_translation_status(node, lang): + print(f"✅ Removed {lang} from {node}'s missing languages") + else: + print(f"⚠️ No update needed for {node} ({lang})") + diff --git a/pipeline/scripts/version_tracker.py b/pipeline/scripts/version_tracker.py new file mode 100644 index 000000000..c242fae2c --- /dev/null +++ b/pipeline/scripts/version_tracker.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +""" +Node Version Tracking and Change Detection Tool +""" + +import os +import json +import hashlib +from pathlib import Path +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +import runtime # noqa: F401 +from lib.paths import AI_INPUT_DIR, NODE_VERSIONS + + +class NodeVersionTracker: + """Node version tracker for detecting source code changes""" + + def __init__(self, ai_input_path: Path): + self.ai_input_path = ai_input_path + self.version_db_path = NODE_VERSIONS + self.version_db = self._load_version_db() + + def _load_version_db(self) -> Dict: + """Load version database from JSON file""" + if self.version_db_path.exists(): + with open(self.version_db_path, 'r', encoding='utf-8') as f: + return json.load(f) + return {"nodes": {}, "last_scan": None} + + def _save_version_db(self): + """Save version database to JSON file""" + with open(self.version_db_path, 'w', encoding='utf-8') as f: + json.dump(self.version_db, f, indent=2, ensure_ascii=False) + + @staticmethod + def calculate_source_hash(source_code: str) -> str: + """Calculate SHA256 hash of source code""" + return hashlib.sha256(source_code.encode('utf-8')).hexdigest() + + @staticmethod + def get_current_timestamp() -> str: + """Get current timestamp in ISO format""" + return datetime.now().isoformat() + + def record_node_version(self, node_name: str, source_code: str, metadata: Dict = None) -> Dict: + """Record node version information with source code hash + + If the hash is the same as current_hash, only update last_updated timestamp. + If the hash is different, append a new version record. + """ + current_hash = self.calculate_source_hash(source_code) + timestamp = self.get_current_timestamp() + + # Save to version database + if node_name not in self.version_db["nodes"]: + self.version_db["nodes"][node_name] = {"versions": []} + + node_data = self.version_db["nodes"][node_name] + existing_hash = node_data.get("current_hash") + + # If hash is the same, only update timestamp (don't create duplicate record) + if existing_hash == current_hash: + node_data["last_updated"] = timestamp + # Return existing version info structure for compatibility + version_info = { + "node_name": node_name, + "extracted_at": timestamp, + "source_hash": current_hash, + "source_length": len(source_code), + "metadata": metadata or {} + } + else: + # Hash is different, create new version record + version_info = { + "node_name": node_name, + "extracted_at": timestamp, + "source_hash": current_hash, + "source_length": len(source_code), + "metadata": metadata or {} + } + node_data["versions"].append(version_info) + node_data["current_hash"] = current_hash + node_data["last_updated"] = timestamp + + self.version_db["last_scan"] = timestamp + self._save_version_db() + return version_info + + def check_node_changed(self, node_name: str, current_source: str) -> Tuple[bool, Optional[str]]: + """Check if node source code has changed + + Returns: + (is_changed, previous_hash) + """ + if node_name not in self.version_db["nodes"]: + return True, None # New node + + current_hash = self.calculate_source_hash(current_source) + previous_hash = self.version_db["nodes"][node_name].get("current_hash") + + return current_hash != previous_hash, previous_hash + + def get_node_version_history(self, node_name: str) -> List[Dict]: + """Get version history of a node""" + if node_name not in self.version_db["nodes"]: + return [] + return self.version_db["nodes"][node_name].get("versions", []) + + def get_changed_nodes(self, comfyui_path: Path) -> List[Dict]: + """Compare each prepared node's recorded hash to live ComfyUI class source (same rule as scan/prepare). + + Uses the latest known ``file_path`` / ``class_name`` in version history metadata; skips nodes + whose bundle was never fingerprinted against a real ``.py`` path. + """ + from lib.node_source_extract import extract_node_class_source + + changed_nodes = [] + root = Path(comfyui_path) + + for node_name, node_data in self.version_db["nodes"].items(): + ai_bundle = self.ai_input_path / node_name / "source_code.py" + if not ai_bundle.exists(): + continue + + meta: Dict = {} + vers = node_data.get("versions") or [] + if vers: + meta = vers[-1].get("metadata") or {} + + rel = meta.get("file_path") + cls_nm = meta.get("class_name") or node_name + if not rel: + continue + + fp = root / rel + if not fp.is_file(): + continue + + class_src = extract_node_class_source(fp, cls_nm) + if not (class_src or "").strip(): + continue + + is_changed, old_hash = self.check_node_changed(node_name, class_src) + if is_changed: + changed_nodes.append( + { + "node_name": node_name, + "old_hash": old_hash, + "new_hash": self.calculate_source_hash(class_src), + "last_updated": node_data.get("last_updated"), + } + ) + + return changed_nodes + + +def compare_versions(version1: Dict, version2: Dict) -> Dict: + """Compare differences between two versions""" + return { + "hash_changed": version1["source_hash"] != version2["source_hash"], + "length_changed": version1["source_length"] != version2["source_length"], + "length_diff": version2["source_length"] - version1["source_length"], + "time_diff": version2["extracted_at"] + } + + +def main(): + """Main function: Check changed nodes""" + import sys + + ai_input_path = AI_INPUT_DIR + tracker = NodeVersionTracker(ai_input_path) + + if len(sys.argv) > 1 and sys.argv[1] == "check": + # Check which nodes have changes + print("🔍 Checking node changes...") + print("=" * 80) + + changed = tracker.get_changed_nodes(Path(os.getenv("COMFYUI_PATH", ""))) + + if changed: + print(f"\n📊 Found {len(changed)} nodes with changes:\n") + for node in changed: + print(f" - {node['node_name']}") + print(f" Old hash: {node['old_hash'][:16]}...") + print(f" New hash: {node['new_hash'][:16]}...") + print(f" Last updated: {node['last_updated']}") + print() + else: + print("\n✅ No nodes have changed") + + elif len(sys.argv) > 1 and sys.argv[1] == "history": + # View history of a specific node + if len(sys.argv) < 3: + print("Usage: python3 version_tracker.py history ") + return + + node_name = sys.argv[2] + history = tracker.get_node_version_history(node_name) + + print(f"📜 Version history for node {node_name}:") + print("=" * 80) + + if history: + for idx, version in enumerate(history, 1): + print(f"\nVersion {idx}:") + print(f" Extracted at: {version['extracted_at']}") + print(f" Source hash: {version['source_hash'][:16]}...") + print(f" Code length: {version['source_length']} characters") + else: + print(f"\nNo version records found for node {node_name}") + + else: + print("Node Version Tracking Tool") + print("=" * 80) + print("\nUsage:") + print(" python3 version_tracker.py check # Check changed nodes") + print(" python3 version_tracker.py history # View node history") + print("\nDatabase location:", tracker.version_db_path) + print(f"Recorded nodes: {len(tracker.version_db['nodes'])}") + if tracker.version_db['last_scan']: + print(f"Last scan time: {tracker.version_db['last_scan']}") + + +if __name__ == "__main__": + main() diff --git a/pipeline/tests/test_doc_title.py b/pipeline/tests/test_doc_title.py new file mode 100644 index 000000000..5f29fd0f7 --- /dev/null +++ b/pipeline/tests/test_doc_title.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Unit tests for lib.doc_title.""" + +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +from lib import doc_title as dt # noqa: E402 + + +class DocTitleTests(unittest.TestCase): + def setUp(self): + dt.load_node_translations.cache_clear() + + def test_strip_leading_h1_single(self): + body = "# Old Title\n\nOverview text.\n\n## Inputs\n" + self.assertEqual(dt.strip_leading_h1(body), "Overview text.\n\n## Inputs\n") + + def test_strip_leading_h1_multiple_and_no_space(self): + body = "# First\n#Second\n\n# Third\n\nOverview.\n" + self.assertEqual(dt.strip_leading_h1(body), "Overview.\n") + + def test_strip_leading_h1_preserves_h2(self): + body = "## Not a title\n\nOverview.\n" + self.assertEqual(dt.strip_leading_h1(body), body) + + def test_ensure_doc_title_fallback_to_node_name(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "node_translations.json" + path.write_text("{}", encoding="utf-8") + with patch.object(dt, "NODE_TRANSLATIONS", path): + dt.load_node_translations.cache_clear() + out = dt.ensure_doc_title("Overview paragraph.", "KSampler", "en") + self.assertEqual(out, "# KSampler\n\nOverview paragraph.") + + def test_get_node_display_name_from_frontend(self): + translations = { + "en": {"KSampler": {"display_name": "KSampler (Advanced)"}}, + "zh": {"KSampler": {"display_name": "K采样器（高级）"}}, + } + self.assertEqual( + dt.get_node_display_name("KSampler", "zh", translations), + "K采样器（高级）", + ) + + def test_get_node_display_name_falls_back_to_english(self): + translations = { + "en": {"KSampler": {"display_name": "KSampler (Advanced)"}}, + "zh": {}, + } + self.assertEqual( + dt.get_node_display_name("KSampler", "zh", translations), + "KSampler (Advanced)", + ) + + def test_get_node_display_name_falls_back_to_class_name(self): + translations = {"en": {}, "zh": {}} + self.assertEqual( + dt.get_node_display_name("UnknownNode", "zh", translations), + "UnknownNode", + ) + + def test_analyze_title_issues(self): + translations = {"en": {"KSampler": {"display_name": "KSampler (Advanced)"}}} + body = "Overview only.\n" + self.assertEqual( + dt.analyze_title_issues(body, "KSampler", "en", translations), + ["missing"], + ) + body_dup = "# A\n# B\n\nOverview.\n" + issues = dt.analyze_title_issues(body_dup, "KSampler", "en", translations) + self.assertIn("duplicate", issues) + self.assertIn("mismatch", issues) + + def test_fix_document_title_preserve_hash(self): + footer = "\n\n---\n**Source fingerprint (SHA-256):** `abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234`\n" + disclaimer = "> AI note [Edit on GitHub](https://example.com)\n" + original = "# Wrong\n\nBody text.\n\n" + disclaimer + footer + translations = {"en": {"KSampler": {"display_name": "KSampler (Advanced)"}}} + out = dt.fix_document_title( + original, "KSampler", "en", translations, hash_mode="preserve" + ) + self.assertIn("# KSampler (Advanced)", out) + self.assertIn("Body text.", out) + self.assertIn("abcd1234abcd1234", out) + self.assertIn("AI note", out) + + def test_ensure_doc_title_replaces_ai_h1(self): + translations = {"en": {"CLIPTextEncode": {"display_name": "CLIP Text Encode"}}} + body = "# AI Guessed Title\n\nDoes encoding.\n" + out = dt.ensure_doc_title(body, "CLIPTextEncode", "en", translations) + self.assertEqual(out, "# CLIP Text Encode\n\nDoes encoding.") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/tests/test_sync_to_comfy_docs.sh b/pipeline/tests/test_sync_to_comfy_docs.sh new file mode 100644 index 000000000..dc6b018ee --- /dev/null +++ b/pipeline/tests/test_sync_to_comfy_docs.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Smoke tests for sync_to_comfy_docs.py (no writes to TARGET_DOCS; uses --dry-run). +# Run from repo root: bash tests/test_sync_to_comfy_docs.sh + +set -e +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +export PYTHONPATH="${REPO_ROOT}${PYTHONPATH:+:$PYTHONPATH}" + +echo "=== 1. Dry-run: 3 nodes, no docs.json ===" +python3 scripts/sync_to_comfy_docs.py --mode test --count 3 --dry-run --no-docs-json + +echo "" +echo "=== 2. Non-existent node: expect 0 nodes synced ===" +python3 scripts/sync_to_comfy_docs.py --node NonExistentNode +# exit 0, "Syncing 0 nodes" + +echo "" +echo "=== 3. Node with space in name (dry-run) ===" +python3 scripts/sync_to_comfy_docs.py --node "Epsilon Scaling" --dry-run --no-docs-json + +echo "" +echo "=== 4. Node with space: Video Slice (dry-run) ===" +python3 scripts/sync_to_comfy_docs.py --node "Video Slice" --dry-run --no-docs-json + +echo "" +echo "=== 5. Mode all dry-run (count only) ===" +python3 scripts/sync_to_comfy_docs.py --mode all --dry-run --no-docs-json 2>&1 | grep -E "^Syncing|^Done" || true + +echo "" +echo "=== All smoke tests passed ===" diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 7d291c49f..000000000 --- a/scripts/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Sync pipeline: embedded-docs → Comfy-Org/docs - -This directory contains the scripts that generate the `built-in-nodes/*` pages on -[docs.comfy.org](https://docs.comfy.org) from the documentation sources in this -repository (`comfyui_embedded_docs/docs//{en,zh,ja,ko}.md`). - -## `sync_to_docs.py` - -Converts every node's `en.md` (+ `zh.md` / `ja.md` / `ko.md` when present) into an -`.mdx` page in a checkout of [Comfy-Org/docs](https://github.com/Comfy-Org/docs), -and updates the `docs.json` navigation (slug casing, category groups, locale tabs). - -### Usage - -```bash -# Point at a Comfy-Org/docs checkout (defaults to ../docs relative to this repo) -export TARGET_DOCS=/path/to/comfy/docs - -# Dry-run (no files written) -python3 scripts/sync_to_docs.py --node Canny --dry-run - -# Sync a single node -python3 scripts/sync_to_docs.py --node Canny - -# Sync everything (all nodes with en.md) -python3 scripts/sync_to_docs.py --mode all -``` - -Optional env vars: - -| Var | Purpose | -|-----|---------| -| `TARGET_DOCS` | Comfy-Org/docs checkout root (default: `../docs` next to this repo) | -| `COMFYUI_PATH` | ComfyUI source checkout, used only to extract node categories when scanner output is absent | -| `ALL_NODES_INFO` | Path to scanner output JSON (`{nodes: {name: {category, ...}}}`), enables category lookup without ComfyUI source | - -### What it generates - -- **Per-locale `.mdx`**: `built-in-nodes/X.mdx`, `zh/built-in-nodes/X.mdx`, `ja/...`, `ko/...` -- **Frontmatter**: title + a **concrete SEO description** extracted from the node's - `en.md` overview first sentence (not a templated string), `sidebarTitle`, icon, wide mode -- **`docs.json` nav**: adds/updates the node slug under the right category group for - all 4 locales, with case-corrected slugs matching the on-disk files -- **Assets**: copies referenced images to `images/built-in-nodes//` - -### MDX safety - -`_normalize_mdx_content()` makes the Markdown source safe for Mintlify's MDX parser: - -- Fenced code blocks are preserved byte-for-byte (never escaped) -- Whitelisted HTML tags (`video`, `source`, `p`, `br`, ...) stay raw -- Paired Mintlify components (`...`, ``, ``, ...) stay raw -- Unknown tags (`` API syntax examples) are escaped to `<bbox>` in pairs -- Orphaned closing tags are escaped (avoids acorn "Unexpected closing slash" errors) -- Comparison operators in prose (`<= 3840`) are escaped - -### Notes - -- Node slugs in `docs.json` must match the on-disk `.mdx` filename **exactly** - (Mintlify routing is case-sensitive). `published_node_name()` resolves the - published name per locale against real directory entries — important on macOS, - where `Path.is_file()` cannot distinguish `CLIPTextEncodeControlnet.mdx` from - `ClipTextEncodeControlnet.mdx` (case-insensitive APFS). -- After syncing, a PR against Comfy-Org/docs is opened separately (the pipeline - itself only writes files and updates `docs.json` locally). From 966c565235155cc3b8245dc18cc068a8c24f6310 Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:13:41 +0800 Subject: [PATCH 04/10] chore: generalize LLM API key config (any OpenAI-compatible provider) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename DEEPSEEK_API_KEY to LLM_API_KEY — the pipeline uses an OpenAI-compatible chat API, so users should be able to point it at any provider (DeepSeek, OpenAI, OpenRouter, local vLLM/Ollama, ...). DEEPSEEK_API_KEY is still accepted as a fallback for existing setups. --- pipeline/README.md | 6 +++--- pipeline/env.example | 4 +++- pipeline/scripts/batch_generate_docs.py | 4 ++-- pipeline/scripts/batch_translate_docs.py | 4 ++-- pipeline/scripts/check_config.py | 8 ++++---- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pipeline/README.md b/pipeline/README.md index 77301a034..4e23af264 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -34,7 +34,7 @@ pipeline/ ```bash cd pipeline cp env.example .env -# edit .env: COMFYUI_PATH, DEEPSEEK_API_KEY, etc. +# edit .env: COMFYUI_PATH, LLM_API_KEY, etc. pip install -r requirements.txt ``` @@ -92,8 +92,8 @@ See `env.example`. Key ones: | Var | Required | Purpose | |-----|----------|---------| | `COMFYUI_PATH` | yes (scan/generate) | ComfyUI source checkout | -| `DEEPSEEK_API_KEY` | yes (LLM steps) | OpenAI-compatible API key | -| `API_BASE_URL` / `API_MODEL` | no | Defaults: DeepSeek | +| `LLM_API_KEY` | yes (LLM steps) | OpenAI-compatible API key (any provider; `DEEPSEEK_API_KEY` also accepted for back-compat) | +| `API_BASE_URL` / `API_MODEL` | no | OpenAI-compatible endpoint + model (defaults: DeepSeek) | | `EMBEDDED_DOCS_PATH` | no | embedded-docs repo root (defaults to repo root) | | `TARGET_DOCS` | sync step | Comfy-Org/docs checkout | | `COMFYUI_FRONTEND_PATH` | param-translation step | ComfyUI frontend repo | diff --git a/pipeline/env.example b/pipeline/env.example index 8c0b78682..c640a567f 100644 --- a/pipeline/env.example +++ b/pipeline/env.example @@ -18,7 +18,9 @@ TARGET_DOCS=/path/to/comfy/docs # --- LLM API configuration -------------------------------------------------- # The pipeline uses an OpenAI-compatible chat API for doc generation & translation. -DEEPSEEK_API_KEY=your_api_key_here +# Any OpenAI-compatible provider works (DeepSeek, OpenAI, OpenRouter, local +# vLLM/Ollama, etc.) — set the base URL and model for your provider. +LLM_API_KEY=your_api_key_here API_BASE_URL=https://api.deepseek.com API_MODEL=deepseek-chat diff --git a/pipeline/scripts/batch_generate_docs.py b/pipeline/scripts/batch_generate_docs.py index 9a5f7f5ac..1948c8f57 100644 --- a/pipeline/scripts/batch_generate_docs.py +++ b/pipeline/scripts/batch_generate_docs.py @@ -32,7 +32,7 @@ load_dotenv() # Configuration -API_KEY = os.getenv('DEEPSEEK_API_KEY') +API_KEY = os.getenv('LLM_API_KEY') or os.getenv('DEEPSEEK_API_KEY') API_BASE_URL = os.getenv('API_BASE_URL', 'https://api.deepseek.com') API_MODEL = os.getenv('API_MODEL', 'deepseek-chat') BATCH_SIZE = int(os.getenv('BATCH_SIZE', '5')) @@ -105,7 +105,7 @@ class AIDocGenerator: def __init__(self): if not API_KEY: - raise ValueError("❌ DEEPSEEK_API_KEY not found, please configure it in .env file") + raise ValueError("❌ LLM_API_KEY not found, please configure it in .env file") self.client = OpenAI( api_key=API_KEY, diff --git a/pipeline/scripts/batch_translate_docs.py b/pipeline/scripts/batch_translate_docs.py index fb9c8ede5..2ee613b76 100644 --- a/pipeline/scripts/batch_translate_docs.py +++ b/pipeline/scripts/batch_translate_docs.py @@ -65,8 +65,8 @@ logger = logging.getLogger(__name__) # AI Configuration -DEFAULT_API_KEY = os.getenv('DEEPSEEK_API_KEY', '') -DEFAULT_BASE_URL = "https://api.deepseek.com" +DEFAULT_API_KEY = os.getenv('LLM_API_KEY') or os.getenv('DEEPSEEK_API_KEY', '') +DEFAULT_BASE_URL = os.getenv('API_BASE_URL', 'https://api.deepseek.com') DEFAULT_MODEL = os.getenv('API_MODEL', 'deepseek-chat') DEFAULT_BATCH_SIZE = 5 diff --git a/pipeline/scripts/check_config.py b/pipeline/scripts/check_config.py index d5775f283..a6a74eefe 100644 --- a/pipeline/scripts/check_config.py +++ b/pipeline/scripts/check_config.py @@ -92,11 +92,11 @@ def check_config(): print("-" * 80) # Check API key - api_key = os.getenv('DEEPSEEK_API_KEY') - if api_key and api_key != 'your_deepseek_api_key_here': - print(f"✅ DEEPSEEK_API_KEY: {'*' * 20}{api_key[-4:]}") + api_key = os.getenv('LLM_API_KEY') or os.getenv('DEEPSEEK_API_KEY') + if api_key and api_key != 'your_api_key_here': + print(f"✅ LLM_API_KEY: {'*' * 20}{api_key[-4:]}") else: - print(f"❌ DEEPSEEK_API_KEY not configured") + print(f"❌ LLM_API_KEY not configured") all_ok = False # Check API settings From 28dae5c897ea3300f62391bccf1439ed632f989a Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:17:38 +0800 Subject: [PATCH 05/10] chore: localize pipeline UI/comment strings to English MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Translate interactive prompts, print messages, docstrings, and comments from Chinese to English across the pipeline scripts (main.py menu, sync_frontend_translations, check_md_links, cleanup_duplicate_hashes, replace_placeholders, check_outputs). Functional data is preserved: language-name maps, per-language heading maps, multilingual regexes, docs.json nav strings, and disclaimer texts must stay in their target languages by design. Also remove scripts/fix_translations.py — the file was corrupt since its initial commit (truncated mid-statement with garbled content) and is unreferenced by any pipeline step. --- pipeline/main.py | 206 +++++++++--------- pipeline/scripts/batch_translate_docs.py | 2 +- pipeline/scripts/check_md_links.py | 46 ++-- pipeline/scripts/check_outputs.py | 4 +- pipeline/scripts/cleanup_duplicate_hashes.py | 16 +- pipeline/scripts/fix_translations.py | 77 ------- pipeline/scripts/replace_placeholders.py | 20 +- .../scripts/sync_frontend_translations.py | 56 ++--- 8 files changed, 175 insertions(+), 252 deletions(-) delete mode 100644 pipeline/scripts/fix_translations.py diff --git a/pipeline/main.py b/pipeline/main.py index db43b6791..dbc4aa8a0 100644 --- a/pipeline/main.py +++ b/pipeline/main.py @@ -717,25 +717,25 @@ def _prompt_yes_no(text: str, default: bool = False) -> bool: def run_interactive(workflow: DocumentationWorkflow) -> bool: """Run interactive menu-driven workflow.""" print("\n" + "=" * 60) - print(" ComfyUI 文档自动化 - 交互式菜单") + print(" ComfyUI Documentation Automation - Interactive Menu") print(" Documentation Automation - Interactive Menu") print("=" * 60) while True: - print("\n请选择操作 / Choose action:") - print(" 1) 仅扫描 (Scan only)") - print(" 2) 生成英文文档 (Generate English docs)") - print(" 3) 翻译（子菜单：单语 / 全部语言；全部语言里可选「强制全量重译」）") + print("\nChoose an action:") + print(" 1) Scan only") + print(" 2) Generate English docs") + print(" 3) Translate (submenu: one language / all languages; force full re-translate option)") print(" (Translate: one lang / all langs; submenu includes force-retranslate-all)") - print(" 4) 生成缺失文档并全部翻译 (Generate missing + translate all)") - print(" 5) 同步到 Comfy 文档 (Sync to Comfy docs)") - print(" 6) 更新变更节点文档 (Regenerate docs for changed nodes)") - print(" 7) 全量重跑英文（可选随后全语言翻译）(FULL en.md; optional all-lang translate)") - print(" 8) 强制全语言重译全部节点（每个 en.md → 11 语覆盖；等同 CLI --retranslate-all-languages）") + print(" 4) Generate missing docs + translate all") + print(" 5) Sync to Comfy docs") + print(" 6) Regenerate docs for changed nodes") + print(" 7) Full en.md regeneration (optional all-lang translate)") + print(" 8) Force full re-translate of all nodes (all en.md -> 11 languages; same as CLI --retranslate-all-languages)") print(" (Force-retranslate ALL langs for EVERY node with en.md; API-heavy)") - print(" 9) 修复已有文档 (Fix existing docs — no AI)") - print(" 0) 退出 (Exit)") - choice = _prompt("选项 / Choice", "0").strip() + print(" 9) Fix existing docs (no AI)") + print(" 0) Exit") + choice = _prompt("Choice", "0").strip() if choice == "0": print("Bye.") @@ -743,32 +743,32 @@ def run_interactive(workflow: DocumentationWorkflow) -> bool: if choice == "1": ok = workflow.scan_nodes() - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok if choice == "2": - print("\n--- 生成模式 ---") - print(" 1) test - 生成指定数量的缺失节点 (default 20)") - print(" 2) all - 生成所有缺失节点") - print(" 3) node - 仅生成单个节点") - sub = _prompt("模式 (1/2/3)", "1").strip() + print("\n--- Generation Mode ---") + print(" 1) test - generate N missing nodes (default 20)") + print(" 2) all - generate all missing nodes") + print(" 3) node - generate a single node") + sub = _prompt("Mode (1/2/3)", "1").strip() if sub == "2": mode = "all" count = 20 node_name = None elif sub == "3": mode = "node" - node_name = _prompt("节点名称 (Node name)").strip() + node_name = _prompt("Node name").strip() if not node_name: - print(" 未输入节点名，已取消。") + print(" No node name entered; cancelled.") continue count = None else: mode = "test" - count = _prompt_int("生成数量 (Count)", 20) + count = _prompt_int("Count", 20) node_name = None - force = _prompt_yes_no("是否覆盖已有文档 (Force overwrite)?", False) + force = _prompt_yes_no("Force overwrite existing docs?", False) print() if mode == "node": ok = ( @@ -779,86 +779,86 @@ def run_interactive(workflow: DocumentationWorkflow) -> bool: ) else: ok = workflow.run_full_workflow(mode=mode, count=count, force=force) - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok if choice == "3": - print("\n--- 翻译 ---") - print(" 1) 单语言 (One language)") - print(" 2) 全部语言 (All languages)") - tr_choice = _prompt("1 或 2", "1").strip() + print("\n--- Translation ---") + print(" 1) One language") + print(" 2) All languages") + tr_choice = _prompt("1 or 2", "1").strip() if tr_choice == "2": - print(" 1) 全部缺失 (all) - 按报告仅翻译当前缺失（每种语言整批缺失）") - print(" 2) 指定数量 (test) - 每种语言只翻译缺失队列前 N 条") - print(" 3) 强制全量重译 - 每个有 en.md 的节点全部语种覆盖（忽略缺失报告；CLI: --retranslate-all-languages）") + print(" 1) all - translate all currently missing per the report") + print(" 2) test - translate the first N missing per language") + print(" 3) Force full re-translate - overwrite all languages for every node with en.md (CLI: --retranslate-all-languages)") all_or_count = _prompt("1 / 2 / 3", "1").strip() if all_or_count == "3": return workflow.run_all_languages_translation(mode="all", count=20, force=True, force_all_nodes=True) if all_or_count == "2": - count = _prompt_int("每种语言处理数量 (Count per language)", 20) - force = _prompt_yes_no("是否覆盖已有翻译 (Force overwrite)?", False) + count = _prompt_int("Count per language", 20) + force = _prompt_yes_no("Force overwrite existing translations?", False) return workflow.run_all_languages_translation(mode="test", count=count, force=force) - force = _prompt_yes_no("是否覆盖已有翻译 (Force overwrite)?", False) + force = _prompt_yes_no("Force overwrite existing translations?", False) return workflow.run_all_languages_translation(mode="all", count=20, force=force) - print("\n可选语言:") + print("\nAvailable languages:") for i, lang in enumerate(LANGUAGES, 1): print(f" {i:2}) {lang} {LANG_NAMES.get(lang, '')}") - lang_idx = _prompt_int("语言编号 (1-11)", 1) + lang_idx = _prompt_int("Language number (1-11)", 1) if not (1 <= lang_idx <= len(LANGUAGES)): - print(" 无效编号。") + print(" Invalid number.") continue lang = LANGUAGES[lang_idx - 1] - print("\n 1) test - 翻译指定数量 (默认 20)") - print(" 2) all - 翻译全部缺失") - tm = _prompt("模式 (1/2)", "1").strip() + print("\n 1) test - translate N (default 20)") + print(" 2) all - translate all missing") + tm = _prompt("Mode (1/2)", "1").strip() mode = "all" if tm == "2" else "test" - count = _prompt_int("数量 (test 时)", 20) if mode == "test" else 20 - force = _prompt_yes_no("是否覆盖已有翻译 (Force overwrite)?", False) + count = _prompt_int("Count (for test)", 20) if mode == "test" else 20 + force = _prompt_yes_no("Force overwrite existing translations?", False) print() ok = workflow.run_translation_workflow(lang=lang, mode=mode, count=count, force=force) - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok if choice == "4": - print("\n--- 生成缺失文档并全部翻译（一次性跑完，中间不再确认）---") - print(" 1) test - 先生成指定数量的缺失英文文档，再对全部语言翻译同样数量") - print(" 2) all - 先生成所有缺失英文文档，再对全部语言翻译所有缺失（推荐，一次性完成）") - sub = _prompt("模式 (1/2)", "2").strip() + print("\n--- Generate missing docs + translate all (runs to completion) ---") + print(" 1) test - generate N missing English docs, then translate the same count for all languages") + print(" 2) all - generate all missing English docs, then translate all missing for every language (recommended)") + sub = _prompt("Mode (1/2)", "2").strip() if sub == "2": gen_mode, gen_count = "all", 20 tr_mode, tr_count = "all", 10 else: gen_mode = "test" - gen_count = _prompt_int("生成数量 (Count)", 20) + gen_count = _prompt_int("Count", 20) tr_mode = "test" tr_count = gen_count - force_gen = _prompt_yes_no("是否覆盖已有英文文档 (Force overwrite)?", False) - force_tr = _prompt_yes_no("是否覆盖已有翻译 (Force overwrite)?", False) - print("\n将一次性执行：先生成英文文档 → 再全部语言翻译，中间不再询问。") - print("[Step 1/2] 生成英文文档...") + force_gen = _prompt_yes_no("Force overwrite existing English docs?", False) + force_tr = _prompt_yes_no("Force overwrite existing translations?", False) + print("\nThis will run end-to-end: generate English docs, then translate all languages, without further prompts.") + print("[Step 1/2] Generating English docs...") if not workflow.run_full_workflow(mode=gen_mode, count=gen_count, force=force_gen): - print(" 生成失败，已取消。") - if _prompt_yes_no("继续操作? (Continue?)", False): + print(" Generation failed; cancelled.") + if _prompt_yes_no("Continue?", False): continue return False - print("\n[Step 2/2] 全部语言翻译（自动连续执行）...") + print("\n[Step 2/2] Translating all languages (automatic)...") ok = workflow.run_all_languages_translation(mode=tr_mode, count=tr_count, force=force_tr) - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok if choice == "5": - print("\n--- 同步到 Comfy 文档 ---") - print(" 将 embedded-docs 的 en.md/zh.md 与图片同步到 comfy/docs (built-in-nodes)。") - print(" 1) test - 同步前 N 个节点 (默认 10)") - print(" 2) all - 同步所有有 en.md 的节点") - sub = _prompt("模式 (1/2)", "1").strip() + print("\n--- Sync to Comfy docs ---") + print(" Sync embedded-docs en.md/zh.md and images to comfy/docs (built-in-nodes).") + print(" 1) test - sync first N nodes (default 10)") + print(" 2) all - sync all nodes with en.md") + sub = _prompt("Mode (1/2)", "1").strip() mode = "all" if sub == "2" else "test" - count = _prompt_int("数量 (test 时)", 10) if mode == "test" else 10 - dry = _prompt_yes_no("仅预览不写入 (Dry run)?", False) - no_json = _prompt_yes_no("不更新 docs.json (No docs.json)?", False) + count = _prompt_int("Count (for test)", 10) if mode == "test" else 10 + dry = _prompt_yes_no("Dry run (no writes)?", False) + no_json = _prompt_yes_no("Skip docs.json update?", False) args = ["--mode", mode] if mode == "test": args.extend(["--count", str(count)]) @@ -872,85 +872,85 @@ def run_interactive(workflow: DocumentationWorkflow) -> bool: args, "Sync to Comfy docs (built-in-nodes + docs.json)" ) - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok if choice == "6": - print("\n--- 更新变更节点文档 ---") - print(" 扫描源码变更 → 重新生成有变动节点的英文文档。") - force = _prompt_yes_no("是否强制覆盖已有文档 (Force overwrite)?", True) + print("\n--- Update changed node docs ---") + print(" Scan source changes -> regenerate English docs for changed nodes.") + force = _prompt_yes_no("Force overwrite existing docs?", True) print() ok = workflow.run_changed_workflow(force=force) - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok if choice == "7": - print("\n--- 全量英文文档重生 ---") - print(" 会：扫描 → 对所有已扫描节点跑 prepare_ai_input → batch_generate_docs all --force。") - print(" ⚠️ 耗时长；会重写每个节点的 en.md 并占用大量 API。") - print(" 默认会在英文完成后继续全语言翻译；若只在交互里改了英文不想动翻译，选 n（或 CLI 仅用 --mode regenerate-all 不加翻译）。") + print("\n--- Full English docs regeneration ---") + print(" Runs: scan -> prepare_ai_input for all scanned nodes -> batch_generate_docs all --force.") + print(" Warning: long-running; rewrites every node en.md and consumes significant API quota.") + print(" By default continues with all-language translation after English; choose n to skip (or use CLI --mode regenerate-all without translation).") also_tr = _prompt_yes_no( - "英文完成后是否继续「全语言翻译」（mode=all + 强制覆盖翻译）? (Also translate all langs?)", + "Also translate all languages after English (mode=all + force overwrite)?", True, ) - if not _prompt_yes_no("确认继续？", False): - print(" 已取消。") + if not _prompt_yes_no("Confirm to continue?", False): + print(" Cancelled.") continue - lim_raw = _prompt("仅先做前 N 个节点（调试，留空=全部）Prepare limit / Enter for all").strip() + lim_raw = _prompt("Limit to first N nodes (debug, empty=all) Prepare limit / Enter for all").strip() prepare_limit = int(lim_raw) if lim_raw else None if prepare_limit is not None and prepare_limit <= 0: - print(" 无效数量。") + print(" Invalid count.") continue print() ok = workflow.run_regenerate_all_workflow( prepare_limit=prepare_limit, translate_all_languages=also_tr, ) - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok if choice == "8": - print("\n--- 强制全语言重译全部节点 ---") - print(" 会对「每个已有 en.md 的节点」在全部 11 种语言上覆盖写入翻译（忽略缺失报告）。") - print(" ⚠️ API 与时间消耗极大；等同于: python3 main.py --retranslate-all-languages") - if not _prompt_yes_no("确认执行？", False): - print(" 已取消。") + print("\n--- Force full re-translate of all nodes ---") + print(" Overwrites translations for every node with en.md across all 11 languages (ignores missing report).") + print(" Warning: heavy API and time usage; equivalent to: python3 main.py --retranslate-all-languages") + if not _prompt_yes_no("Confirm execution?", False): + print(" Cancelled.") continue print() ok = workflow.run_all_languages_translation( mode="all", count=20, force=True, force_all_nodes=True ) - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok if choice == "9": - print("\n--- 修复已有文档 (Fix) ---") - print(" 1) 文档标题 — 缺失 / 重复 / 与前端 display_name 不一致") + print("\n--- Fix existing docs ---") + print(" 1) Doc titles - missing / duplicated / mismatched with frontend display_name") print(" (Doc titles from frontend nodeDefs; no AI)") - sub = _prompt("选项 (1)", "1").strip() + sub = _prompt("Option (1)", "1").strip() if sub != "1": - print(" 暂仅支持 1) 文档标题。") + print(" Only option 1 (doc titles) is supported.") continue - print("\n Hash 处理 / SHA footer:") - print(" 1) preserve - 保留原 disclaimer + SHA（推荐，翻译已对齐时）") - print(" 2) update - 从 en.md / ai_input 重新写入 SHA（并重写 disclaimer）") + print("\n Hash handling / SHA footer:") + print(" 1) preserve - keep original disclaimer + SHA (recommended when translations are aligned)") + print(" 2) update - rewrite SHA from en.md / ai_input (and rewrite disclaimer)") hash_choice = _prompt("Hash (1/2)", "1").strip() hash_mode = "update" if hash_choice == "2" else "preserve" - print("\n 1) test - 扫描前 N 个文件 (默认 20)") - print(" 2) all - 扫描全部已有 .md") - scope = _prompt("范围 (1/2)", "2").strip() + print("\n 1) test - scan first N files (default 20)") + print(" 2) all - scan all existing .md files") + scope = _prompt("Scope (1/2)", "2").strip() fix_mode = "all" if scope == "2" else "test" - fix_count = _prompt_int("文件数量 (test 时)", 20) if fix_mode == "test" else 20 - dry_run = _prompt_yes_no("仅预览不写入 (Dry run)?", True) - sync_fe = _prompt_yes_no("先同步前端 nodeDefs 翻译? (Sync frontend)", True) - node_name = _prompt("仅单个节点 (留空=全部) Node name").strip() or None - lang_raw = _prompt("仅单语言代码 en/zh/... (留空=全部) Lang").strip() or None + fix_count = _prompt_int("File count (for test)", 20) if fix_mode == "test" else 20 + dry_run = _prompt_yes_no("Dry run (no writes)?", True) + sync_fe = _prompt_yes_no("Sync frontend nodeDefs translations first?", True) + node_name = _prompt("Single node only (empty = all) Node name").strip() or None + lang_raw = _prompt("Single language code en/zh/... (empty = all) Lang").strip() or None if lang_raw and lang_raw not in (["en"] + LANGUAGES): - print(f" 无效语言: {lang_raw}") + print(f" Invalid language: {lang_raw}") continue print() ok = workflow.run_fix_doc_titles_workflow( @@ -962,11 +962,11 @@ def run_interactive(workflow: DocumentationWorkflow) -> bool: sync_frontend=sync_fe, hash_mode=hash_mode, ) - if ok and _prompt_yes_no("继续操作? (Continue?)", False): + if ok and _prompt_yes_no("Continue?", False): continue return ok - print(" 请输入 0–9。") + print(" Please enter 0-9.") def main(): diff --git a/pipeline/scripts/batch_translate_docs.py b/pipeline/scripts/batch_translate_docs.py index 2ee613b76..6d5dc4131 100644 --- a/pipeline/scripts/batch_translate_docs.py +++ b/pipeline/scripts/batch_translate_docs.py @@ -273,7 +273,7 @@ def translate_document(node_name, target_lang, lang_config, api_key, base_url, m content = strip_trailing_fingerprint_section(content) # Post-process: enforce English output names. The AI sometimes translates output names - # (e.g. 'positive' → '正向', 'négatif') causing duplicates in the Outputs table. + # (e.g. 'positive' translated to the same word as 'negative') causing duplicates in the Outputs table. # We parse the en.md Outputs table and force the first column to match. content = _fix_output_names_in_translation(content, full_en) diff --git a/pipeline/scripts/check_md_links.py b/pipeline/scripts/check_md_links.py index f729b63e2..b50a53abf 100644 --- a/pipeline/scripts/check_md_links.py +++ b/pipeline/scripts/check_md_links.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ -检查 Markdown 文档中的链接有效性和占位符 -用法: python check_md_links.py [--fix-placeholders] +Check link validity and placeholders in Markdown docs +Usage: python check_md_links.py [--fix-placeholders] """ import os @@ -22,7 +22,7 @@ HTML_SRC_RE = re.compile(r'<(?:img|video|audio|source)[^>]+src=["\']([^"\'>]+)["\']', re.IGNORECASE) PLACEHOLDER_RE = re.compile(r'\{(heading_\w+)\}') -# 不同语言的标题映射 +# Heading mapping per language HEADING_TRANSLATIONS = { 'en': { 'heading_overview': '## Overview', @@ -76,17 +76,17 @@ } def get_language_from_filename(filename): - """从文件名获取语言代码""" + """Get the language code from a filename""" stem = Path(filename).stem return stem if stem in HEADING_TRANSLATIONS else None def is_local_link(link): - """只检查本地相对路径（非 http/https/data: 开头）""" + """Only check local relative paths (not http/https/data: prefixed)""" link = link.strip() return not (link.startswith('http://') or link.startswith('https://') or link.startswith('data:')) def find_links_in_line(line): - """提取行中的所有本地链接""" + """Extract all local links in a line""" links = [] for m in MD_LINK_RE.finditer(line): for g in m.groups(): @@ -99,11 +99,11 @@ def find_links_in_line(line): return links def find_placeholders_in_content(content): - """查找内容中的占位符""" + """Find placeholders in content""" return PLACEHOLDER_RE.findall(content) def check_file(fpath, fix_placeholders=False): - """检查单个文件的链接和占位符""" + """Check links and placeholders in a single file""" errors = [] placeholder_issues = [] rel_fpath = fpath.relative_to(DOCS_ROOT.parent.parent) @@ -113,12 +113,12 @@ def check_file(fpath, fix_placeholders=False): content = f.read() lines = content.split('\n') - # 检查占位符 + # Check placeholders placeholders = find_placeholders_in_content(content) if placeholders: - placeholder_issues.append(f"{rel_fpath}: 发现占位符 {placeholders}") + placeholder_issues.append(f"{rel_fpath}: found placeholders {placeholders}") - # 如果需要修复且能识别语言 + # Fix if requested and language is detectable if fix_placeholders and lang: translations = HEADING_TRANSLATIONS[lang] modified = False @@ -131,9 +131,9 @@ def check_file(fpath, fix_placeholders=False): if modified: with open(fpath, 'w', encoding='utf-8') as f: f.write(content) - placeholder_issues[-1] += " [已修复]" + placeholder_issues[-1] += " [fixed]" - # 检查链接 + # Check links for idx, line in enumerate(lines, 1): for link in find_links_in_line(line): link_path = link.split('#')[0].split('?')[0] @@ -153,20 +153,20 @@ def check_file(fpath, fix_placeholders=False): abs_path = (fpath.parent / link_path).absolute() if not abs_path.exists(): - errors.append(f"[链接失效] {rel_fpath}:{idx}: {link}") + errors.append(f"[broken link] {rel_fpath}:{idx}: {link}") return errors, placeholder_issues def check_links(): if not DOCS_ROOT.exists(): - print(f"错误: 文档目录不存在: {DOCS_ROOT}") + print(f"Error: docs directory does not exist: {DOCS_ROOT}") sys.exit(1) fix_placeholders = '--fix-placeholders' in sys.argv link_errors = [] placeholder_issues = [] - print(f"正在检查 {DOCS_ROOT} 下的所有文档...") + print(f"Checking all docs under {DOCS_ROOT}...") for root, _, files in os.walk(DOCS_ROOT): for fname in files: @@ -181,30 +181,30 @@ def check_links(): if placeholder_issues: has_issues = True print("\n" + "=" * 80) - print(f"发现 {len(placeholder_issues)} 个文件包含占位符：") + print(f"Found {len(placeholder_issues)} files with placeholders:") print("=" * 80) for issue in placeholder_issues: print(f" {issue}") if fix_placeholders: - print("\n✓ 占位符已自动修复") + print("\n✓ Placeholders auto-fixed") else: - print("\n提示: 运行 --fix-placeholders 参数来自动替换占位符") + print("\nTip: run with --fix-placeholders to auto-replace placeholders") if link_errors: has_issues = True print("\n" + "=" * 80) - print(f"发现 {len(link_errors)} 个无效链接：") + print(f"Found {len(link_errors)} broken links:") print("=" * 80) for i, err in enumerate(link_errors): if i < 10: print(f" {err}") elif i == 10: - print(f"\n ... 还有 {len(link_errors) - 10} 个错误（仅显示前10个）") + print(f"\n ... {len(link_errors) - 10} more errors (showing first 10 only)") break - print("\n请修正上述链接问题。") + print("\nPlease fix the link issues above.") if not has_issues: - print("\n✓ 所有检查通过！") + print("\n✓ All checks passed!") else: sys.exit(1) diff --git a/pipeline/scripts/check_outputs.py b/pipeline/scripts/check_outputs.py index a7897d42e..0b419489a 100644 --- a/pipeline/scripts/check_outputs.py +++ b/pipeline/scripts/check_outputs.py @@ -12,8 +12,8 @@ if v.get('outputs'): nodes_with_outputs.append((n, v.get('outputs', {}))) -print(f'有输出翻译的节点数: {len(nodes_with_outputs)}') -print('\n示例:') +print(f'Nodes with output translations: {len(nodes_with_outputs)}') +print('\nExamples:') for n, o in nodes_with_outputs[:5]: print(f'\n{n}:') print(json.dumps(o, ensure_ascii=False, indent=2)) diff --git a/pipeline/scripts/cleanup_duplicate_hashes.py b/pipeline/scripts/cleanup_duplicate_hashes.py index 162ad743b..238f35444 100644 --- a/pipeline/scripts/cleanup_duplicate_hashes.py +++ b/pipeline/scripts/cleanup_duplicate_hashes.py @@ -55,7 +55,7 @@ def cleanup_duplicate_hashes(): if removed_count > 0: nodes_cleaned.append(node_name) - print(f" ✅ {node_name}: 移除了 {removed_count} 个重复记录 (保留 {len(unique_versions)} 个)") + print(f" ✅ {node_name}: removed {removed_count} duplicate records (kept {len(unique_versions)})") node_data["versions"] = unique_versions # Update current_hash to match the last version (most recent) @@ -65,24 +65,24 @@ def cleanup_duplicate_hashes(): if total_removed > 0: print("=" * 80) - print(f"\n📊 清理完成:") - print(f" - 清理了 {len(nodes_cleaned)} 个节点") - print(f" - 移除了 {total_removed} 个重复的 hash 记录") + print(f"\n📊 Cleanup finished:") + print(f" - cleaned {len(nodes_cleaned)} nodes") + print(f" - removed {total_removed} duplicate hash records") # Backup original file backup_path = version_db_path.with_suffix('.json.backup') - print(f"\n💾 备份原文件到: {backup_path}") + print(f"\n💾 Backup of original file at: {backup_path}") with open(backup_path, 'w', encoding='utf-8') as f: json.dump(data, f, indent=2, ensure_ascii=False) # Save cleaned data - print(f"💾 保存清理后的数据...") + print(f"💾 Saving cleaned data...") with open(version_db_path, 'w', encoding='utf-8') as f: json.dump(data, f, indent=2, ensure_ascii=False) - print(f"\n✅ 清理完成！") + print(f"\n✅ Cleanup complete!") else: - print("\n✅ 没有发现重复记录，无需清理") + print("\n✅ No duplicate records found; nothing to clean") if __name__ == "__main__": diff --git a/pipeline/scripts/fix_translations.py b/pipeline/scripts/fix_translations.py deleted file mode 100644 index 0937e65bf..000000000 --- a/pipeline/scripts/fix_translations.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -""" -修复翻译文件中的占位符和参数名称问题 -""" - -import json -import re -from pathlib import Path - -import runtime # noqa: F401 -from lib.paths import NODE_TRANSLATIONS, TRANSLATION_CONFIG, embedded_docs_dir - -DOCS_PATH = embedded_docs_dir() -TRANSLATION_CONFIG_FILE = TRANSLATION_CONFIG -TRANSLATIONS_FILE = NODE_TRANSLATIONS - -def fix_heading_placeholders(content, lang_config): - """替换标题占位符""" - content = content.replace('{heading_overview}', f"## {lang_config.get('heading_overview', 'Overview')}") - content = content.replace('{heading_inputs}', f"## {lang_config.get('heading_inputs', 'Inputs')}") - content = content.replace('{heading_outputs}', f"## {lang_config.get('heading_outputs', 'Outputs')}") - return content - -def update_param_names(content, node_name, lang, frontend_translations): - """更新参数名称""" - if lang not in frontend_translations: - return content, False - - if node_name not in frontend_translations[lang]: - return content, False - - node_trans = frontend_translations[lang][node_name] - original_content = content - changes_made = [] - - # 更新输入参数名 - if 'inputs' in node_trans: - for param_name, param_data in node_trans['inputs'].items(): - frontend_name = param_data.get('name', '') - if frontend_name and frontend_name != param_name: - # 替换表格中的参数名 - pattern = rf'\|\s*`{re.escape(param_name)}`\s*\|' - replacement = f'| `{frontend_name}` |' - if re.search(pattern, content): - content = re.sub(pattern, replacement, content) - changes_made.append(f"{param_name} → {frontend_name}") - - # 更新输出参数名 - if 'outputs' in node_trans: - lines = content.split('\n') - in_output_section = False - output_row_index = 0 - - for i, line in enumerate(lines): - # 检测输出部分 - if re.match(r'##\s+(?:输出|輸出|出力|출력|Выходы|Salidas|Sorties|Outputs|المخرجات|Çıktılar)', line): - in_output_section = True - output_row_index = 0 - continue - elif line.startswith('##'): - in_output_section = False - continue - - # 更新输出名称 - if in_output_section and line.strip().startswith('|'): - if 'Output Name' in line or 'Data Type' in line or '---' in line: - continue - - output_idx_str = str(output_row_index) - if output_idx_str in node_trans['outputs']: - output_data = node_trans['outputs'][output_idx_str] - frontend_name = output_data.get('name', '') - - if frontend_name: - old_match = re.match(r'(\|\s*)`([^`]+)`(\s*\|)', line) - if old_match: - old_name = old_match.group aufgrund eines Errors unterbrochen wurde, ich muss fortfahren diff --git a/pipeline/scripts/replace_placeholders.py b/pipeline/scripts/replace_placeholders.py index 86644b975..44f9af263 100644 --- a/pipeline/scripts/replace_placeholders.py +++ b/pipeline/scripts/replace_placeholders.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ -替换文档中的占位符为对应语言的标题 -用法: python replace_placeholders.py [--check-only] +Replace placeholders in docs with language-specific headings +Usage: python replace_placeholders.py [--check-only] """ import os @@ -14,7 +14,7 @@ DOCS_ROOT = embedded_docs_dir() -# 不同语言的标题映射 +# Heading mapping per language HEADING_TRANSLATIONS = { 'en': { 'heading_overview': '## Overview', @@ -68,12 +68,12 @@ } def get_language_from_filename(filename): - """从文件名获取语言代码""" + """Get the language code from a filename""" stem = Path(filename).stem return stem if stem in HEADING_TRANSLATIONS else None def replace_placeholders_in_file(filepath, check_only=False): - """替换文件中的占位符""" + """Replace placeholders in a file""" lang = get_language_from_filename(filepath.name) if not lang: return False, [] @@ -117,18 +117,18 @@ def main(): if files_with_placeholders: if check_only: - print(f"\n发现 {len(files_with_placeholders)} 个文件包含占位符：") + print(f"\nFound {len(files_with_placeholders)} files with placeholders:") for repl in all_replacements: print(repl) - print(f"\n运行不带 --check-only 参数来替换这些占位符。") + print(f"\nRun without --check-only to replace these placeholders.") sys.exit(1) else: - print(f"\n已替换 {len(files_with_placeholders)} 个文件中的占位符：") + print(f"\nReplaced placeholders in {len(files_with_placeholders)} files:") for repl in all_replacements: print(repl) - print(f"\n✓ 完成！") + print(f"\n✓ Done!") else: - print("✓ 未发现需要替换的占位符。") + print("✓ No placeholders found to replace.") if __name__ == '__main__': main() diff --git a/pipeline/scripts/sync_frontend_translations.py b/pipeline/scripts/sync_frontend_translations.py index 7fa27da22..138583ac0 100644 --- a/pipeline/scripts/sync_frontend_translations.py +++ b/pipeline/scripts/sync_frontend_translations.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """ -从前端仓库的 nodeDefs.json 同步参数翻译到文档 -用法: python sync_frontend_translations.py -示例: python sync_frontend_translations.py /path/to/ComfyUI_frontend +Sync parameter translations from the frontend repo's nodeDefs.json into the docs +Usage: python sync_frontend_translations.py +Example: python sync_frontend_translations.py /path/to/ComfyUI_frontend """ import json @@ -17,26 +17,26 @@ DOCS_ROOT = embedded_docs_dir() -# 支持的语言 +# Supported languages SUPPORTED_LANGS = ['en', 'zh', 'zh-TW', 'es', 'fr', 'ja', 'ko', 'ru', 'ar', 'tr', 'pt-BR', 'fa'] def load_frontend_translations(frontend_path, lang): - """从前端仓库加载指定语言的翻译""" + """Load translations for a language from the frontend repo""" locale_file = Path(frontend_path) / 'src' / 'locales' / lang / 'nodeDefs.json' if not locale_file.exists(): - print(f"⚠️ 警告: 未找到 {lang} 语言文件: {locale_file}") + print(f"⚠️ Warning: language file not found for {lang}: {locale_file}") return {} try: with open(locale_file, 'r', encoding='utf-8') as f: return json.load(f) except Exception as e: - print(f"❌ 读取 {lang} 语言文件失败: {e}") + print(f"❌ Failed to read language file for {lang}: {e}") return {} def get_node_translations(frontend_translations, node_name): - """获取节点的翻译信息""" + """Get translation info for a node""" if node_name not in frontend_translations: return None @@ -48,7 +48,7 @@ def get_node_translations(frontend_translations, node_name): 'outputs': {} } - # 提取输入参数翻译 + # Extract input parameter translations if 'inputs' in node_data: for param_name, param_data in node_data['inputs'].items(): if isinstance(param_data, dict): @@ -57,7 +57,7 @@ def get_node_translations(frontend_translations, node_name): 'tooltip': param_data.get('tooltip', '') } - # 提取输出翻译 + # Extract output translations if 'outputs' in node_data: for output_idx, output_data in node_data['outputs'].items(): if isinstance(output_data, dict): @@ -69,47 +69,47 @@ def get_node_translations(frontend_translations, node_name): return translations def create_translation_report(frontend_path): - """生成翻译对照报告""" - print(f"\n正在从前端仓库加载翻译: {frontend_path}\n") + """Generate a translation comparison report""" + print(f"\nLoading translations from frontend repo: {frontend_path}\n") - # 加载所有语言的翻译 + # Load translations for all languages all_translations = {} for lang in SUPPORTED_LANGS: all_translations[lang] = load_frontend_translations(frontend_path, lang) - # 获取所有节点名称（从文档目录） + # Get all node names (from the docs directory) node_dirs = [d for d in DOCS_ROOT.iterdir() if d.is_dir()] - print(f"找到 {len(node_dirs)} 个节点文档目录\n") + print(f"Found {len(node_dirs)} node doc directories\n") print("=" * 80) - # 为每个节点生成翻译报告 + # Generate a translation report per node for node_dir in sorted(node_dirs): node_name = node_dir.name - # 检查是否有对应的前端翻译 + # Check whether a frontend translation exists has_translation = any(node_name in all_translations[lang] for lang in SUPPORTED_LANGS) if not has_translation: - print(f"\n⚠️ {node_name}: 未找到前端翻译") + print(f"\n⚠️ {node_name}: no frontend translation found") continue print(f"\n✓ {node_name}") print("-" * 80) - # 显示各语言的参数翻译 + # Show per-language parameter translations for lang in SUPPORTED_LANGS: if node_name in all_translations[lang]: trans = get_node_translations(all_translations[lang], node_name) if trans and trans['inputs']: - print(f"\n [{lang.upper()}] 参数翻译:") + print(f"\n [{lang.upper()}] Parameter translations:") for param_name, param_trans in trans['inputs'].items(): print(f" - {param_name}: {param_trans['name']}") if param_trans['tooltip']: - print(f" 提示: {param_trans['tooltip'][:60]}...") + print(f" Tooltip: {param_trans['tooltip'][:60]}...") def export_translation_json(frontend_path, output_file=None): - """导出所有节点的翻译为JSON文件，便于后续使用""" + """Export all node translations to a JSON file for later use""" all_translations = {} for lang in SUPPORTED_LANGS: all_translations[lang] = load_frontend_translations(frontend_path, lang) @@ -118,20 +118,20 @@ def export_translation_json(frontend_path, output_file=None): with open(output_path, 'w', encoding='utf-8') as f: json.dump(all_translations, f, ensure_ascii=False, indent=2) - print(f"\n✓ 翻译数据已导出到: {output_path}") + print(f"\n✓ Translation data exported to: {output_path}") def main(): if len(sys.argv) < 2: - print("用法: python sync_frontend_translations.py [--export]") - print("示例: python sync_frontend_translations.py /path/to/ComfyUI_frontend") - print("\n选项:") - print(" --export 导出翻译为JSON文件") + print("Usage: python sync_frontend_translations.py [--export]") + print("Example: python sync_frontend_translations.py /path/to/ComfyUI_frontend") + print("\nOptions:") + print(" --export export translations to JSON file") sys.exit(1) frontend_path = Path(sys.argv[1]) if not frontend_path.exists(): - print(f"❌ 错误: 前端仓库路径不存在: {frontend_path}") + print(f"❌ Error: frontend repo path does not exist: {frontend_path}") sys.exit(1) if '--export' in sys.argv: From 0f5d1d945df850002fd21ed7fdf2d130b7a1ef50 Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:19:51 +0800 Subject: [PATCH 06/10] chore: rename pipeline/ to docs-generation/ More descriptive name for the doc automation tooling, mirroring the comfyui_embedded_docs/ content directory it maintains. All path resolution is based on __file__, so nothing else changes. --- {pipeline => docs-generation}/.gitignore | 0 {pipeline => docs-generation}/README.md | 6 +++--- {pipeline => docs-generation}/config/doc_rules.txt | 0 .../config/translation_config.json | 0 {pipeline => docs-generation}/config/translation_rules.txt | 0 {pipeline => docs-generation}/env.example | 0 {pipeline => docs-generation}/lib/__init__.py | 0 {pipeline => docs-generation}/lib/doc_disclaimer.py | 0 {pipeline => docs-generation}/lib/doc_title.py | 0 {pipeline => docs-generation}/lib/hash_footer.py | 0 {pipeline => docs-generation}/lib/node_source_extract.py | 0 {pipeline => docs-generation}/lib/paths.py | 0 {pipeline => docs-generation}/main.py | 0 {pipeline => docs-generation}/requirements.txt | 0 .../scripts/batch_generate_docs.py | 0 .../scripts/batch_translate_docs.py | 0 {pipeline => docs-generation}/scripts/check_config.py | 0 {pipeline => docs-generation}/scripts/check_md_links.py | 0 {pipeline => docs-generation}/scripts/check_outputs.py | 0 .../scripts/cleanup_duplicate_hashes.py | 0 {pipeline => docs-generation}/scripts/fix_doc_titles.py | 0 {pipeline => docs-generation}/scripts/generate_docs.py | 0 .../scripts/migrate_docs_format.py | 0 {pipeline => docs-generation}/scripts/prepare_ai_input.py | 0 .../scripts/prepare_translation.py | 0 .../scripts/replace_placeholders.py | 0 {pipeline => docs-generation}/scripts/runtime.py | 0 {pipeline => docs-generation}/scripts/scan_missing_nodes.py | 0 .../scripts/sync_frontend_translations.py | 0 {pipeline => docs-generation}/scripts/sync_to_comfy_docs.py | 0 .../scripts/update_param_translations.py | 0 .../scripts/update_translation_status.py | 0 {pipeline => docs-generation}/scripts/version_tracker.py | 0 {pipeline => docs-generation}/tests/test_doc_title.py | 0 .../tests/test_sync_to_comfy_docs.sh | 0 35 files changed, 3 insertions(+), 3 deletions(-) rename {pipeline => docs-generation}/.gitignore (100%) rename {pipeline => docs-generation}/README.md (98%) rename {pipeline => docs-generation}/config/doc_rules.txt (100%) rename {pipeline => docs-generation}/config/translation_config.json (100%) rename {pipeline => docs-generation}/config/translation_rules.txt (100%) rename {pipeline => docs-generation}/env.example (100%) rename {pipeline => docs-generation}/lib/__init__.py (100%) rename {pipeline => docs-generation}/lib/doc_disclaimer.py (100%) rename {pipeline => docs-generation}/lib/doc_title.py (100%) rename {pipeline => docs-generation}/lib/hash_footer.py (100%) rename {pipeline => docs-generation}/lib/node_source_extract.py (100%) rename {pipeline => docs-generation}/lib/paths.py (100%) rename {pipeline => docs-generation}/main.py (100%) rename {pipeline => docs-generation}/requirements.txt (100%) rename {pipeline => docs-generation}/scripts/batch_generate_docs.py (100%) rename {pipeline => docs-generation}/scripts/batch_translate_docs.py (100%) rename {pipeline => docs-generation}/scripts/check_config.py (100%) rename {pipeline => docs-generation}/scripts/check_md_links.py (100%) rename {pipeline => docs-generation}/scripts/check_outputs.py (100%) rename {pipeline => docs-generation}/scripts/cleanup_duplicate_hashes.py (100%) rename {pipeline => docs-generation}/scripts/fix_doc_titles.py (100%) rename {pipeline => docs-generation}/scripts/generate_docs.py (100%) rename {pipeline => docs-generation}/scripts/migrate_docs_format.py (100%) rename {pipeline => docs-generation}/scripts/prepare_ai_input.py (100%) rename {pipeline => docs-generation}/scripts/prepare_translation.py (100%) rename {pipeline => docs-generation}/scripts/replace_placeholders.py (100%) rename {pipeline => docs-generation}/scripts/runtime.py (100%) rename {pipeline => docs-generation}/scripts/scan_missing_nodes.py (100%) rename {pipeline => docs-generation}/scripts/sync_frontend_translations.py (100%) rename {pipeline => docs-generation}/scripts/sync_to_comfy_docs.py (100%) rename {pipeline => docs-generation}/scripts/update_param_translations.py (100%) rename {pipeline => docs-generation}/scripts/update_translation_status.py (100%) rename {pipeline => docs-generation}/scripts/version_tracker.py (100%) rename {pipeline => docs-generation}/tests/test_doc_title.py (100%) rename {pipeline => docs-generation}/tests/test_sync_to_comfy_docs.sh (100%) diff --git a/pipeline/.gitignore b/docs-generation/.gitignore similarity index 100% rename from pipeline/.gitignore rename to docs-generation/.gitignore diff --git a/pipeline/README.md b/docs-generation/README.md similarity index 98% rename from pipeline/README.md rename to docs-generation/README.md index 4e23af264..6256c58e9 100644 --- a/pipeline/README.md +++ b/docs-generation/README.md @@ -11,7 +11,7 @@ translates node documentation into 11 languages, and publishes it. ## Layout ``` -pipeline/ +docs-generation/ ├── main.py # CLI entry point (full workflows) ├── scripts/ │ ├── scan_missing_nodes.py # Scan ComfyUI source: find new/changed nodes @@ -32,7 +32,7 @@ pipeline/ ## Setup ```bash -cd pipeline +cd docs-generation cp env.example .env # edit .env: COMFYUI_PATH, LLM_API_KEY, etc. pip install -r requirements.txt @@ -45,7 +45,7 @@ pip install -r requirements.txt cd /path/to/ComfyUI && git fetch origin master && git rebase origin/master # 2. Scan + regenerate changed node docs (never skip, even if scan says "no changes") -cd /path/to/embedded-docs/pipeline +cd /path/to/embedded-docs/docs-generation python3 main.py --mode changed # 3. Generate docs for new nodes diff --git a/pipeline/config/doc_rules.txt b/docs-generation/config/doc_rules.txt similarity index 100% rename from pipeline/config/doc_rules.txt rename to docs-generation/config/doc_rules.txt diff --git a/pipeline/config/translation_config.json b/docs-generation/config/translation_config.json similarity index 100% rename from pipeline/config/translation_config.json rename to docs-generation/config/translation_config.json diff --git a/pipeline/config/translation_rules.txt b/docs-generation/config/translation_rules.txt similarity index 100% rename from pipeline/config/translation_rules.txt rename to docs-generation/config/translation_rules.txt diff --git a/pipeline/env.example b/docs-generation/env.example similarity index 100% rename from pipeline/env.example rename to docs-generation/env.example diff --git a/pipeline/lib/__init__.py b/docs-generation/lib/__init__.py similarity index 100% rename from pipeline/lib/__init__.py rename to docs-generation/lib/__init__.py diff --git a/pipeline/lib/doc_disclaimer.py b/docs-generation/lib/doc_disclaimer.py similarity index 100% rename from pipeline/lib/doc_disclaimer.py rename to docs-generation/lib/doc_disclaimer.py diff --git a/pipeline/lib/doc_title.py b/docs-generation/lib/doc_title.py similarity index 100% rename from pipeline/lib/doc_title.py rename to docs-generation/lib/doc_title.py diff --git a/pipeline/lib/hash_footer.py b/docs-generation/lib/hash_footer.py similarity index 100% rename from pipeline/lib/hash_footer.py rename to docs-generation/lib/hash_footer.py diff --git a/pipeline/lib/node_source_extract.py b/docs-generation/lib/node_source_extract.py similarity index 100% rename from pipeline/lib/node_source_extract.py rename to docs-generation/lib/node_source_extract.py diff --git a/pipeline/lib/paths.py b/docs-generation/lib/paths.py similarity index 100% rename from pipeline/lib/paths.py rename to docs-generation/lib/paths.py diff --git a/pipeline/main.py b/docs-generation/main.py similarity index 100% rename from pipeline/main.py rename to docs-generation/main.py diff --git a/pipeline/requirements.txt b/docs-generation/requirements.txt similarity index 100% rename from pipeline/requirements.txt rename to docs-generation/requirements.txt diff --git a/pipeline/scripts/batch_generate_docs.py b/docs-generation/scripts/batch_generate_docs.py similarity index 100% rename from pipeline/scripts/batch_generate_docs.py rename to docs-generation/scripts/batch_generate_docs.py diff --git a/pipeline/scripts/batch_translate_docs.py b/docs-generation/scripts/batch_translate_docs.py similarity index 100% rename from pipeline/scripts/batch_translate_docs.py rename to docs-generation/scripts/batch_translate_docs.py diff --git a/pipeline/scripts/check_config.py b/docs-generation/scripts/check_config.py similarity index 100% rename from pipeline/scripts/check_config.py rename to docs-generation/scripts/check_config.py diff --git a/pipeline/scripts/check_md_links.py b/docs-generation/scripts/check_md_links.py similarity index 100% rename from pipeline/scripts/check_md_links.py rename to docs-generation/scripts/check_md_links.py diff --git a/pipeline/scripts/check_outputs.py b/docs-generation/scripts/check_outputs.py similarity index 100% rename from pipeline/scripts/check_outputs.py rename to docs-generation/scripts/check_outputs.py diff --git a/pipeline/scripts/cleanup_duplicate_hashes.py b/docs-generation/scripts/cleanup_duplicate_hashes.py similarity index 100% rename from pipeline/scripts/cleanup_duplicate_hashes.py rename to docs-generation/scripts/cleanup_duplicate_hashes.py diff --git a/pipeline/scripts/fix_doc_titles.py b/docs-generation/scripts/fix_doc_titles.py similarity index 100% rename from pipeline/scripts/fix_doc_titles.py rename to docs-generation/scripts/fix_doc_titles.py diff --git a/pipeline/scripts/generate_docs.py b/docs-generation/scripts/generate_docs.py similarity index 100% rename from pipeline/scripts/generate_docs.py rename to docs-generation/scripts/generate_docs.py diff --git a/pipeline/scripts/migrate_docs_format.py b/docs-generation/scripts/migrate_docs_format.py similarity index 100% rename from pipeline/scripts/migrate_docs_format.py rename to docs-generation/scripts/migrate_docs_format.py diff --git a/pipeline/scripts/prepare_ai_input.py b/docs-generation/scripts/prepare_ai_input.py similarity index 100% rename from pipeline/scripts/prepare_ai_input.py rename to docs-generation/scripts/prepare_ai_input.py diff --git a/pipeline/scripts/prepare_translation.py b/docs-generation/scripts/prepare_translation.py similarity index 100% rename from pipeline/scripts/prepare_translation.py rename to docs-generation/scripts/prepare_translation.py diff --git a/pipeline/scripts/replace_placeholders.py b/docs-generation/scripts/replace_placeholders.py similarity index 100% rename from pipeline/scripts/replace_placeholders.py rename to docs-generation/scripts/replace_placeholders.py diff --git a/pipeline/scripts/runtime.py b/docs-generation/scripts/runtime.py similarity index 100% rename from pipeline/scripts/runtime.py rename to docs-generation/scripts/runtime.py diff --git a/pipeline/scripts/scan_missing_nodes.py b/docs-generation/scripts/scan_missing_nodes.py similarity index 100% rename from pipeline/scripts/scan_missing_nodes.py rename to docs-generation/scripts/scan_missing_nodes.py diff --git a/pipeline/scripts/sync_frontend_translations.py b/docs-generation/scripts/sync_frontend_translations.py similarity index 100% rename from pipeline/scripts/sync_frontend_translations.py rename to docs-generation/scripts/sync_frontend_translations.py diff --git a/pipeline/scripts/sync_to_comfy_docs.py b/docs-generation/scripts/sync_to_comfy_docs.py similarity index 100% rename from pipeline/scripts/sync_to_comfy_docs.py rename to docs-generation/scripts/sync_to_comfy_docs.py diff --git a/pipeline/scripts/update_param_translations.py b/docs-generation/scripts/update_param_translations.py similarity index 100% rename from pipeline/scripts/update_param_translations.py rename to docs-generation/scripts/update_param_translations.py diff --git a/pipeline/scripts/update_translation_status.py b/docs-generation/scripts/update_translation_status.py similarity index 100% rename from pipeline/scripts/update_translation_status.py rename to docs-generation/scripts/update_translation_status.py diff --git a/pipeline/scripts/version_tracker.py b/docs-generation/scripts/version_tracker.py similarity index 100% rename from pipeline/scripts/version_tracker.py rename to docs-generation/scripts/version_tracker.py diff --git a/pipeline/tests/test_doc_title.py b/docs-generation/tests/test_doc_title.py similarity index 100% rename from pipeline/tests/test_doc_title.py rename to docs-generation/tests/test_doc_title.py diff --git a/pipeline/tests/test_sync_to_comfy_docs.sh b/docs-generation/tests/test_sync_to_comfy_docs.sh similarity index 100% rename from pipeline/tests/test_sync_to_comfy_docs.sh rename to docs-generation/tests/test_sync_to_comfy_docs.sh From f386c1d9d3a47bc5e599807391f656da4dbb0504 Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:21:57 +0800 Subject: [PATCH 07/10] docs: update root README for docs-generation pipeline Point the 'Syncing to Comfy docs' section at docs-generation/scripts/ instead of the old doc_automation/ paths, list the pipeline components, and document TARGET_DOCS in the example commands. --- README.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 675ce0ad2..a0f3ddddf 100644 --- a/README.md +++ b/README.md @@ -46,35 +46,45 @@ The publishing workflow: ## Syncing to Comfy docs -The script `doc_automation/sync_to_comfy_docs.py` syncs embedded-docs (en.md, zh.md, and assets) to the [comfy/docs](https://github.com/Comfy-Org/comfy/tree/main/docs) repository as built-in node MDX files and updates the navigation (`docs.json`). +The `docs-generation` pipeline syncs embedded-docs (en.md, zh.md, ja.md, ko.md, and assets) to the [Comfy-Org/docs](https://github.com/Comfy-Org/docs) repository as built-in node MDX files and updates the navigation (`docs.json`). + +The pipeline lives in [`docs-generation/`](docs-generation/README.md) and includes: + +- `docs-generation/scripts/scan_missing_nodes.py` – scan the ComfyUI codebase, detect new/changed nodes +- `docs-generation/scripts/batch_generate_docs.py` + `batch_translate_docs.py` – LLM-based doc generation and 11-language translation +- `docs-generation/scripts/update_param_translations.py` – reconcile parameter names with the ComfyUI frontend i18n +- `docs-generation/scripts/sync_to_comfy_docs.py` – generate `built-in-nodes/*.mdx` + update `docs.json` navigation +- `docs-generation/scripts/version_tracker.py` – per-node source hash tracking + +See [docs-generation/README.md](docs-generation/README.md) for full setup and workflow. **Environment variables (optional):** -- `EMBEDDED_DOCS_PATH` – Path to this repo (default: parent of `doc_automation`) +- `EMBEDDED_DOCS_PATH` – Path to this repo (default: the repo this pipeline lives in) - `COMFYUI_PATH` – Path to the ComfyUI repo (used to read node category from source) - `TARGET_DOCS` – Path to the comfy/docs root (e.g. `/path/to/comfy/docs`) -**Category mapping:** The sync script uses each node’s ComfyUI category to put it in the right docs.json group. For the most complete categories (including API nodes and nodes that get category from a base class), run the node scanner once so it can write `doc_automation/all_nodes_info.json`; the sync script will prefer that file when present. +**Category mapping:** The sync script uses each node's ComfyUI category to put it in the right docs.json group. For the most complete categories (including API nodes and nodes that get category from a base class), run the node scanner once so it can write `docs-generation/data/all_nodes_info.json`; the sync script will prefer that file when present. ```sh # Optional: run scanner first to build all_nodes_info.json (better category coverage) -python doc_automation/scan_missing_nodes.py +python docs-generation/scripts/scan_missing_nodes.py ``` **Run from repo root:** ```sh # Test mode: sync first 10 nodes (dry run: no writes) -python doc_automation/sync_to_comfy_docs.py --mode test --count 10 --dry-run +TARGET_DOCS=/path/to/comfy/docs python docs-generation/scripts/sync_to_comfy_docs.py --mode test --count 10 --dry-run # Sync all nodes with en.md and update docs.json -python doc_automation/sync_to_comfy_docs.py --mode all +TARGET_DOCS=/path/to/comfy/docs python docs-generation/scripts/sync_to_comfy_docs.py --mode all # Sync a single node -python doc_automation/sync_to_comfy_docs.py --node Load3D +TARGET_DOCS=/path/to/comfy/docs python docs-generation/scripts/sync_to_comfy_docs.py --node Load3D ``` -You can also use the interactive menu: run `python doc_automation/main.py` and choose option **5) 同步到 Comfy 文档 (Sync to Comfy docs)**. +You can also use the interactive menu: run `python docs-generation/main.py` and choose option **5) Sync to Comfy docs**. ## Linting From 5d0e58bfd1ca9295d86c31cd0d1aca8c957495a7 Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:22:09 +0800 Subject: [PATCH 08/10] chore: update lib docstrings to docs-generation naming --- docs-generation/lib/__init__.py | 2 +- docs-generation/lib/paths.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-generation/lib/__init__.py b/docs-generation/lib/__init__.py index f39ed0020..b0d202ff3 100644 --- a/docs-generation/lib/__init__.py +++ b/docs-generation/lib/__init__.py @@ -1 +1 @@ -"""Shared libraries for doc_automation.""" +"""Shared libraries for the docs-generation pipeline.""" diff --git a/docs-generation/lib/paths.py b/docs-generation/lib/paths.py index 6b2a85aa5..5611c5c7c 100644 --- a/docs-generation/lib/paths.py +++ b/docs-generation/lib/paths.py @@ -1,4 +1,4 @@ -"""Central path configuration for doc_automation.""" +"""Central path configuration for the docs-generation pipeline.""" from __future__ import annotations From 7ecf56d0016a35934f3ed56e354571bcd2cf25bb Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:34:11 +0800 Subject: [PATCH 09/10] =?UTF-8?q?fix:=20address=20CodeRabbit=20review=20?= =?UTF-8?q?=E2=80=94=2017=20findings=20in=20sync=5Fto=5Fcomfy=5Fdocs.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup: - Remove unused helpers (_remove_group_from_pages, flatten_builtin_pages, _build_reverse_map/_GROUP_LABEL_TO_CATEGORY_ROOT, collect_page_keys prefix arg) - Modernize type annotations to built-in generics (dict/list/set/tuple) Correctness: - _purge_noncanonical_nav_pages: replace noncanonical keys in place when a matching .mdx exists (locate list before removal), keep unknown keys instead of silently dropping them - _rebuild_wrapper_groups: pass locale_code into published_node_name so slug casing is resolved per locale - resolve_source_node: canonical name first, deterministic sorted aliases, deduped checks - main: exit non-zero when --node does not resolve to en.md; reject negative --count; drop stray f-string - docs.json: atomic write via temp file + os.replace, preserve trailing newline Perf: - Cache ComfyUI source file listing/reads (_comfyui_source_files) - Memoize per-locale .mdx dir listings (_locale_mdx_names) and scanner_node_key Description (GEO): - Per-locale description extraction with English fallback (was: all locales used the English description) - Skip image/table lines; split on '. ' only before uppercase (protects e.g.); align truncation to 180 chars MDX safety: - Stash inline backtick spans in addition to fenced code blocks - Preserve whitespace after escaped < (incl. newlines) - Escape literal curly braces in prose (code stays untouched) - copy_assets_and_rewrite: stash code blocks before rewriting refs; resolve basename collisions with subdirectory prefix Tests: add tests/test_sync_helpers.py (17 new tests covering the above). --- docs-generation/scripts/sync_to_comfy_docs.py | 409 +++++++++++------- docs-generation/tests/test_sync_helpers.py | 138 ++++++ 2 files changed, 392 insertions(+), 155 deletions(-) create mode 100644 docs-generation/tests/test_sync_helpers.py diff --git a/docs-generation/scripts/sync_to_comfy_docs.py b/docs-generation/scripts/sync_to_comfy_docs.py index 563a11ff7..5a32fb45f 100644 --- a/docs-generation/scripts/sync_to_comfy_docs.py +++ b/docs-generation/scripts/sync_to_comfy_docs.py @@ -15,8 +15,9 @@ import re import shutil import sys +from functools import lru_cache from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Optional import runtime # noqa: F401 from lib.paths import ALL_NODES_INFO, default_embedded_docs_path, load_dotenv @@ -26,10 +27,10 @@ ALL_NODES_INFO_PATH = ALL_NODES_INFO # Cache: node_name -> category (first segment). Loaded from scanner output if available. -_nodes_info_cache: Optional[Dict[str, Dict[str, Any]]] = None +_nodes_info_cache: Optional[dict[str, dict[str, Any]]] = None -def _load_all_nodes_info() -> Dict[str, Dict[str, Any]]: +def _load_all_nodes_info() -> dict[str, dict[str, Any]]: """Load all_nodes_info.json from scanner (node_name -> { file, category?, ... }).""" global _nodes_info_cache if _nodes_info_cache is not None: @@ -60,7 +61,7 @@ def _load_all_nodes_info() -> Dict[str, Dict[str, Any]]: DOCS_JSON = TARGET_DOCS / "docs.json" # Locale sync + docs.json navigation (language code -> config) -LOCALE_CONFIGS: List[Dict[str, Any]] = [ +LOCALE_CONFIGS: list[dict[str, Any]] = [ { "code": "en", "md_file": "en.md", @@ -150,7 +151,7 @@ def _load_all_nodes_info() -> Dict[str, Dict[str, Any]]: # - replacement nodes in nodes_replacements.py (no category field) # - deprecated / aliased nodes not found in current source # - partner API nodes with flat MDX files but no scanner entry -_FALLBACK_CATEGORY: Dict[str, str] = { +_FALLBACK_CATEGORY: dict[str, str] = { # Replacement nodes (nodes_replacements.py) — inherit from their original "BatchImagesNode": "image", "ConditioningAverage": "conditioning", @@ -223,19 +224,6 @@ def _load_all_nodes_info() -> Dict[str, Dict[str, Any]]: "VAEEncodeTiled": "latent", } -# Reverse map: group label (EN or ZH) -> ComfyUI category root string -# Used by _restructure_group_by_category to look up which category root each group corresponds to. -_GROUP_LABEL_TO_CATEGORY_ROOT: Dict[str, str] = {} -def _build_reverse_map() -> None: - seen: Dict[str, str] = {} - for cat_key, labels in CATEGORY_TO_GROUP.items(): - for label in labels: - if label not in seen: - seen[label] = cat_key - _GROUP_LABEL_TO_CATEGORY_ROOT.update(seen) -_build_reverse_map() - - def _default_group_for_lang(lang_idx: int) -> str: return (DEFAULT_GROUP_EN, DEFAULT_GROUP_ZH, DEFAULT_GROUP_JA, DEFAULT_GROUP_KO)[lang_idx] @@ -247,14 +235,14 @@ def _group_label_for_category(first_segment: str, lang_idx: int) -> str: return _seg_to_label(first_segment) -def _find_lang_entry(nav: Dict[str, Any], lang_code: str) -> Optional[Dict[str, Any]]: +def _find_lang_entry(nav: dict[str, Any], lang_code: str) -> Optional[dict[str, Any]]: for entry in nav.get("navigation", {}).get("languages", []): if entry.get("language") == lang_code: return entry return None -def _find_tab_pages(lang_entry: Dict[str, Any], tab_name: str) -> Optional[List[Any]]: +def _find_tab_pages(lang_entry: dict[str, Any], tab_name: str) -> Optional[list[Any]]: for tab in lang_entry.get("tabs", []): if tab.get("tab") == tab_name and "pages" in tab: return tab["pages"] @@ -262,11 +250,18 @@ def _find_tab_pages(lang_entry: Dict[str, Any], tab_name: str) -> Optional[List[ def _seg_to_label(seg: str) -> str: - """Convert a path segment like 'custom_sampling' to a display label 'Custom Sampling'.""" - return seg.replace("_", " ").replace("-", " ").title() + """Convert a path segment like 'custom_sampling' to a display label 'Custom Sampling'. + + Preserves already-uppercase tokens (acronyms such as BFL, SDXL, API) unchanged + instead of title-casing them into 'Sdxl' / 'Api'. + """ + return " ".join( + token if token.isupper() and len(token) > 1 else token.capitalize() + for token in seg.replace("_", " ").replace("-", " ").split() + ) -def _category_to_group_and_sub(full_category_path: str, lang_idx: int = 0) -> Tuple[str, Optional[str]]: +def _category_to_group_and_sub(full_category_path: str, lang_idx: int = 0) -> tuple[str, Optional[str]]: """Return (group label, sub-group label or None) for the given locale index. Always uses the FIRST segment to determine the top-level group so that the @@ -295,7 +290,7 @@ def is_local_link(href: str) -> bool: ) -def _class_name_variants(node_name: str) -> List[str]: +def _class_name_variants(node_name: str) -> list[str]: """Return possible class names in source (e.g. ClipTextEncode <-> CLIPTextEncode).""" variants = [node_name] if node_name.startswith("Clip") and len(node_name) > 4: @@ -340,31 +335,38 @@ def _category_from_schema_node_id(text: str, node_name: str, first_segment_only: return None -def extract_category_full_from_comfyui(node_name: str) -> Optional[str]: - """Extract full category string from ComfyUI source (e.g. 'api node/image/ByteDance').""" +@lru_cache(maxsize=None) +def _comfyui_source_files() -> tuple[Path, ...]: + """Cached list of ComfyUI Python source files under SCAN_PATHS (read once).""" + files: list[Path] = [] for base in SCAN_PATHS: if not base.exists(): continue - files = [base] if base.is_file() else list(base.rglob("*.py")) - for path in files: - if path.suffix != ".py": - continue - try: - text = path.read_text(encoding="utf-8", errors="ignore") - except Exception: - continue - if node_name not in text and not any(v in text for v in _class_name_variants(node_name)): - continue - cat = _category_from_class_block(text, node_name, first_segment_only=False) - if cat: - return cat - cat = _category_from_schema_node_id(text, node_name, first_segment_only=False) - if cat: - return cat + files.extend([base] if base.is_file() else list(base.rglob("*.py"))) + return tuple(files) + + +def _read_comfyui_source(path: Path) -> str: + """Read a ComfyUI source file, tolerating decode errors (cached per file).""" + return path.read_text(encoding="utf-8", errors="ignore") + + +def extract_category_full_from_comfyui(node_name: str) -> Optional[str]: + """Extract full category string from ComfyUI source (e.g. 'api node/image/ByteDance').""" + for path in _comfyui_source_files(): + text = _read_comfyui_source(path) + if node_name not in text and not any(v in text for v in _class_name_variants(node_name)): + continue + cat = _category_from_class_block(text, node_name, first_segment_only=False) + if cat: + return cat + cat = _category_from_schema_node_id(text, node_name, first_segment_only=False) + if cat: + return cat return None -def _resolve_node_info_key(nodes: Dict[str, Dict], node_name: str) -> Optional[Dict[str, Any]]: +def _resolve_node_info_key(nodes: dict[str, dict], node_name: str) -> Optional[dict[str, Any]]: """Get node info from all_nodes_info by node_name, or by class variant, or by node_id.""" if node_name in nodes: return nodes[node_name] @@ -377,6 +379,7 @@ def _resolve_node_info_key(nodes: Dict[str, Dict], node_name: str) -> Optional[D return None +@lru_cache(maxsize=None) def scanner_node_key(node_name: str) -> Optional[str]: """Return the all_nodes_info.json dict key for this node, if known to the scanner.""" nodes = _load_all_nodes_info() @@ -404,10 +407,10 @@ def canonical_node_name(node_name: str) -> str: return scanner_node_key(node_name) or node_name -def node_name_nav_aliases(node_name: str) -> Set[str]: +def node_name_nav_aliases(node_name: str) -> set[str]: """All page-key basename variants that should collapse to the same scanner node.""" canonical = canonical_node_name(node_name) - aliases: Set[str] = {node_name, canonical} + aliases: set[str] = {node_name, canonical} for variant in _class_name_variants(node_name): aliases.add(variant) for variant in _class_name_variants(canonical): @@ -427,6 +430,16 @@ def _locale_code_for_page_key(page_key: str) -> str: return "en" +@lru_cache(maxsize=None) +def _locale_mdx_names(builtin_dir: str) -> frozenset: + """Cached set of published .mdx basenames (without extension) for a locale directory.""" + d = Path(builtin_dir) + try: + return frozenset(e[:-4] for e in os.listdir(d) if e.endswith(".mdx")) + except OSError: + return frozenset() + + def published_node_name(node_name: str, locale_code: Optional[str] = None) -> str: """MDX filename and docs.json page basename — keep existing on-disk name when already published. @@ -446,11 +459,9 @@ def published_node_name(node_name: str, locale_code: Optional[str] = None) -> st d = locale["builtin_dir"] if not d.is_dir(): continue - try: - entries = os.listdir(d) - except OSError: + mdx_names = set(_locale_mdx_names(str(d))) + if not mdx_names: continue - mdx_names = {e[:-4] for e in entries if e.endswith(".mdx")} # 1) exact (case-sensitive) alias match against real on-disk names for alias in aliases: if alias in mdx_names: @@ -474,25 +485,39 @@ def canonical_page_key(page_key: str) -> str: return "/".join([*parts[:-1], published]) -def resolve_source_node(node_name: str) -> Tuple[str, Path]: - """Return (scanner canonical name, embedded-docs dir) for reading en.md / zh.md / ja.md.""" +def resolve_source_node(node_name: str) -> tuple[str, Path]: + """Return (scanner canonical name, embedded-docs dir) for reading en.md / zh.md / ja.md. + + Checks the canonical name first, then the remaining nav aliases in deterministic + sorted order (avoiding duplicate checks), then a case-insensitive fallback scan. + """ canonical = canonical_node_name(node_name) - for candidate in node_name_nav_aliases(node_name): + seen: set[str] = set() + candidates: list[str] = [] + for candidate in sorted(node_name_nav_aliases(node_name)): + if candidate in seen: + continue + seen.add(candidate) + if candidate == canonical: + candidates.insert(0, candidate) # canonical first + else: + candidates.append(candidate) + for candidate in candidates: node_dir = DOCS_SOURCE / candidate if (node_dir / "en.md").exists(): return canonical, node_dir if DOCS_SOURCE.exists(): lower = canonical.lower() - for node_dir in DOCS_SOURCE.iterdir(): + for node_dir in sorted(DOCS_SOURCE.iterdir()): if node_dir.is_dir() and node_dir.name.lower() == lower and (node_dir / "en.md").exists(): return canonical, node_dir return canonical, DOCS_SOURCE / canonical -def list_nodes_with_en_md() -> List[str]: - """List embedded-docs nodes deduped by scanner canonical name.""" - seen: Set[str] = set() - nodes: List[str] = [] +def list_nodes_with_en_md() -> list[str]: + """list embedded-docs nodes deduped by scanner canonical name.""" + seen: set[str] = set() + nodes: list[str] = [] for node_dir in sorted(DOCS_SOURCE.iterdir()): if not node_dir.is_dir() or not (node_dir / "en.md").exists(): continue @@ -524,57 +549,34 @@ def get_category_for_node(node_name: str) -> Optional[str]: def extract_category_from_comfyui(node_name: str) -> Optional[str]: """Find node in ComfyUI source and extract CATEGORY/category for this node only. Returns first segment (e.g. sampling from sampling/custom_sampling).""" - for base in SCAN_PATHS: - if not base.exists(): + for path in _comfyui_source_files(): + text = _read_comfyui_source(path) + if node_name not in text and not any(v in text for v in _class_name_variants(node_name)): continue - files = [base] if base.is_file() else list(base.rglob("*.py")) - for path in files: - if path.suffix != ".py": - continue - try: - text = path.read_text(encoding="utf-8", errors="ignore") - except Exception: - continue - if node_name not in text and not any(v in text for v in _class_name_variants(node_name)): - continue - # Prefer: category from the class block that defines this node - cat = _category_from_class_block(text, node_name) - if cat: - return cat - # Fallback: Schema with node_id (e.g. node_id="AddNoise" ... category="...") - cat = _category_from_schema_node_id(text, node_name) - if cat: - return cat + # Prefer: category from the class block that defines this node + cat = _category_from_class_block(text, node_name) + if cat: + return cat + # Fallback: Schema with node_id (e.g. node_id="AddNoise" ... category="...") + cat = _category_from_schema_node_id(text, node_name) + if cat: + return cat return None -def collect_page_keys(pages: List[Any], prefix: str = "") -> Set[str]: +def collect_page_keys(pages: list[Any]) -> set[str]: """Recursively collect all page string keys from a tab's pages.""" - out: Set[str] = set() + out: set[str] = set() for item in pages: if isinstance(item, str): out.add(item) elif isinstance(item, dict): if "pages" in item: - out |= collect_page_keys(item["pages"], prefix) + out |= collect_page_keys(item["pages"]) return out -def flatten_builtin_pages(pages: List[Any], key_prefix: str) -> List[str]: - """ - Recursively collect built-in-nodes page keys (e.g. built-in-nodes/NodeName or zh/built-in-nodes/NodeName) - and return a sorted flat list. Used for flat/collapsed sidebar (no groups). - """ - keys: Set[str] = set() - for item in pages: - if isinstance(item, str) and (item == key_prefix or item.startswith(key_prefix + "/")): - keys.add(item) - elif isinstance(item, dict) and "pages" in item: - keys |= set(flatten_builtin_pages(item["pages"], key_prefix)) - return sorted(keys) - - -def find_group_in_pages(pages: List[Any], group_label: str) -> Optional[List[Any]]: +def find_group_in_pages(pages: list[Any], group_label: str) -> Optional[list[Any]]: """Find the top-level group with 'group' == group_label and return its 'pages' list.""" for item in pages: if isinstance(item, dict) and item.get("group") == group_label and "pages" in item: @@ -582,17 +584,17 @@ def find_group_in_pages(pages: List[Any], group_label: str) -> Optional[List[Any return None -def find_or_create_group_in_pages(pages: List[Any], group_label: str) -> List[Any]: +def find_or_create_group_in_pages(pages: list[Any], group_label: str) -> list[Any]: """Find group with group_label in pages, or create it and append. Return that group's 'pages' list.""" for item in pages: if isinstance(item, dict) and item.get("group") == group_label and "pages" in item: return item["pages"] - new_group: Dict[str, Any] = {"group": group_label, "pages": []} + new_group: dict[str, Any] = {"group": group_label, "pages": []} pages.append(new_group) return new_group["pages"] -def remove_page_from_pages(pages: List[Any], page_key: str) -> None: +def remove_page_from_pages(pages: list[Any], page_key: str) -> None: """Remove page_key from pages tree in place (recursive).""" i = 0 while i < len(pages): @@ -606,15 +608,7 @@ def remove_page_from_pages(pages: List[Any], page_key: str) -> None: i += 1 -def _remove_group_from_pages(pages: List[Any], group_label: str) -> None: - """Remove the first top-level group with given label from pages (in place).""" - for i, item in enumerate(pages): - if isinstance(item, dict) and item.get("group") == group_label: - pages.pop(i) - return - - -def _sort_pages_alphabetically(pages: List[Any], groups_first: bool = True) -> None: +def _sort_pages_alphabetically(pages: list[Any], groups_first: bool = True) -> None: """Sort pages array in place: recursively sort nested 'pages', then sort this level. groups_first=True → sub-groups before flat page strings (used inside category groups @@ -636,8 +630,8 @@ def _sort_pages_alphabetically(pages: List[Any], groups_first: bool = True) -> N def _rebuild_wrapper_groups( - wrapper_pages: List[Any], - node_cat_map: Dict[str, str], + wrapper_pages: list[Any], + node_cat_map: dict[str, str], lang_idx: int = 0, ) -> None: """Completely rebuild all non-API groups inside wrapper_pages from scratch. @@ -649,29 +643,30 @@ def _rebuild_wrapper_groups( lang_idx: 0 = EN, 1 = zh, 2 = ja, 3 = ko labels from CATEGORY_TO_GROUP. """ # Preserve the API Node group as-is - api_node_item: Optional[Dict[str, Any]] = None + api_node_item: Optional[dict[str, Any]] = None for item in wrapper_pages: if isinstance(item, dict) and item.get("group") == "API Node": api_node_item = item break - api_keys: Set[str] = set(collect_page_keys(api_node_item["pages"])) if api_node_item else set() - all_keys: Set[str] = set(collect_page_keys(wrapper_pages)) + api_keys: set[str] = set(collect_page_keys(api_node_item["pages"])) if api_node_item else set() + all_keys: set[str] = set(collect_page_keys(wrapper_pages)) non_api_keys = {canonical_page_key(k) for k in (all_keys - api_keys)} # Build case-insensitive lookup for node_cat_map (handles ClipLoader → CLIPLoader mismatches) - lower_cat_map: Dict[str, str] = {k.lower(): v for k, v in node_cat_map.items()} + lower_cat_map: dict[str, str] = {k.lower(): v for k, v in node_cat_map.items()} # Build case-insensitive lookup for fallback map - lower_fallback: Dict[str, str] = {k.lower(): v for k, v in _FALLBACK_CATEGORY.items()} + lower_fallback: dict[str, str] = {k.lower(): v for k, v in _FALLBACK_CATEGORY.items()} # Rebuild wrapper: clear everything, re-add API Node, then re-place all other keys wrapper_pages.clear() if api_node_item is not None: wrapper_pages.append(api_node_item) + locale_code = LOCALE_CONFIGS[lang_idx]["code"] if 0 <= lang_idx < len(LOCALE_CONFIGS) else "en" for key in sorted(non_api_keys): key_parts = Path(key).parts # e.g. ('built-in-nodes', 'conditioning', 'video-models', 'wan-vace-to-video') - node_name = published_node_name(key_parts[-1]) + node_name = published_node_name(key_parts[-1], locale_code) if key_parts[-1] != node_name: key = "/".join([*key_parts[:-1], node_name]) @@ -755,7 +750,7 @@ def _rebuild_wrapper_groups( target.append(key) -def _remove_empty_groups(pages: List[Any]) -> None: +def _remove_empty_groups(pages: list[Any]) -> None: """Remove groups with empty 'pages' in place (recursive, bottom-up).""" i = 0 while i < len(pages): @@ -768,13 +763,13 @@ def _remove_empty_groups(pages: List[Any]) -> None: i += 1 -def _migrate_toplevel_groups_to_wrapper(pages: List[Any], wrapper_label: str) -> None: +def _migrate_toplevel_groups_to_wrapper(pages: list[Any], wrapper_label: str) -> None: """Move any top-level dict groups (other than wrapper_label) inside the wrapper group. This ensures previously added top-level groups (3D, API Node, etc.) become nested inside the wrapper so Mintlify renders them as collapsible entries. """ - orphans: List[Dict[str, Any]] = [] + orphans: list[dict[str, Any]] = [] i = 0 while i < len(pages): item = pages[i] @@ -804,7 +799,7 @@ def get_description_from_content(content: str) -> str: Skips AI-generated disclaimer blockquote lines and headings. """ lines = content.split("\n") - first_para: List[str] = [] + first_para: list[str] = [] for line in lines: line_stripped = line.strip() if not line_stripped: @@ -817,6 +812,9 @@ def get_description_from_content(content: str) -> str: break if line_stripped.startswith("#"): continue + # Skip Markdown images and table rows — they are not prose + if line_stripped.startswith("![") or line_stripped.startswith("|"): + continue if line_stripped.startswith("> ") and ( "AI-generated" in line_stripped or "AI 生成" in line_stripped @@ -826,18 +824,23 @@ def get_description_from_content(content: str) -> str: continue first_para.append(line_stripped) paragraph = " ".join(first_para) if first_para else "" - # Return only the first sentence (up to first ". " or end of paragraph) - for sep in (". ", "。"): - idx = paragraph.find(sep) - if idx != -1: - return paragraph[: idx + 1] - return paragraph[:160] if paragraph else "" - - -def find_local_asset_refs(content: str, doc_dir: Path) -> List[Tuple[str, Path]]: + # Return only the first sentence. For English, split on ". " only when the + # next char is uppercase (avoids cutting "e.g." / "i.e." abbreviations); + # CJK sentence ender "。" splits unconditionally. + m = re.search(r"\.\s+(?=[A-Z])", paragraph) + if m: + return paragraph[: m.end()] + idx = paragraph.find("。") + if idx != -1: + return paragraph[: idx + 1] + # Align fallback truncation with the 180-char frontmatter limit + return paragraph[:180] if paragraph else "" + + +def find_local_asset_refs(content: str, doc_dir: Path) -> list[tuple[str, Path]]: """Return list of (original_ref, resolved_absolute_path) for local assets.""" - refs: List[Tuple[str, Path]] = [] - seen: Set[str] = set() + refs: list[tuple[str, Path]] = [] + seen: set[str] = set() for m in MD_IMAGE_RE.finditer(content): href = m.group(1).strip() if not is_local_link(href): @@ -862,11 +865,32 @@ def copy_assets_and_rewrite( images_out_dir: Path, dry_run: bool, ) -> str: - """Copy referenced assets to images_out_dir and rewrite refs to /images/built-in-nodes/NodeName/xxx.""" + """Copy referenced assets to images_out_dir and rewrite refs to /images/built-in-nodes/NodeName/xxx. + + - Resolves basename collisions by prefixing the relative subdirectory when two + distinct sources share a basename, so neither overwrites the other. + - Rewrites only prose references: fenced code blocks are stashed first so their + content stays byte-identical. + """ refs = find_local_asset_refs(content, doc_dir) - out = content + # Stash fenced code blocks so refs inside them are never rewritten + _stashed: list[str] = [] + + def _stash_code_blocks(m: "re.Match[str]") -> str: + _stashed.append(m.group(0)) + return f"\x00CODEBLOCK{len(_stashed) - 1}\x00" + + out = re.sub(r"```.*?```", _stash_code_blocks, content, flags=re.DOTALL) + + used_names: dict[str, Path] = {} for orig, abs_path in refs: filename = abs_path.name + rel = abs_path.relative_to(doc_dir) + # Deterministic destination name: include the source subdirectory on collision + if filename in used_names and used_names[filename] != abs_path: + stem, ext = filename.rsplit(".", 1) if "." in filename else (filename, "") + filename = f"{Path(rel).parent.name}_{stem}.{ext}" if ext else f"{Path(rel).parent.name}_{stem}" + used_names[filename] = abs_path new_ref = f"/images/built-in-nodes/{node_name}/{filename}" if not dry_run: images_out_dir.mkdir(parents=True, exist_ok=True) @@ -875,6 +899,12 @@ def copy_assets_and_rewrite( shutil.copy2(abs_path, dest) # Replace in content (use orig as-is to avoid re-escaping) out = out.replace(orig, new_ref) + + # Restore stashed code blocks verbatim + def _restore(m: "re.Match[str]") -> str: + return _stashed[int(m.group(1))] + + out = re.sub(r"\x00CODEBLOCK(\d+)\x00", _restore, out) return out @@ -892,14 +922,15 @@ def _normalize_mdx_content(content: str) -> str: # Strip leading H1 (# Title) and any blank lines after it content = re.sub(r'^#\s+[^\n]*\n?\n*', '', content, count=1) - # Protect fenced code blocks from ALL escaping below: inside ``` blocks the - # content must stay byte-identical (CommonMark renders code blocks verbatim, - # so <= would show literally instead of <=). - _code_blocks: List[str] = [] + # Protect fenced code blocks AND inline backtick spans from ALL escaping below: + # inside ``` blocks (and `code`) the content must stay byte-identical + # (CommonMark renders code verbatim, so <= would show literally instead of <=). + _code_blocks: list[str] = [] def _stash_code(m: "re.Match[str]") -> str: _code_blocks.append(m.group(0)) return f"\x00CODEBLOCK{len(_code_blocks) - 1}\x00" content = re.sub(r"```.*?```", _stash_code, content, flags=re.DOTALL) + content = re.sub(r"`[^`\n]+`", _stash_code, content) content = content.replace("
", "
") content = re.sub(r"(]+)>", r"\1 />", content) @@ -908,8 +939,9 @@ def _stash_code(m: "re.Match[str]") -> str: # Must escape <= first, then already handled above). content = re.sub(r"<=", r"<=", content) content = re.sub(r"<(\d)", r"<\1", content) - # Escape any remaining bare < that is not an HTML/JSX tag (e.g. and are intentional paired components # (e.g. Mintlify's ..., , ) — keep them raw. # Only orphaned tags (opening without closing, or closing without opening) get escaped. @@ -939,6 +971,11 @@ def _stash_code(m: "re.Match[str]") -> str: ) content = re.sub(r"<([a-zA-Z_][a-zA-Z0-9]*)", lambda m: m.group(0) if m.group(1) in _keep_raw else "<" + m.group(1), content) + # Escape literal curly braces in prose (MDX treats { as a JSX expression + # boundary). Code blocks and inline code are already stashed, so they are + # unaffected. + content = content.replace("{", "{").replace("}", "}") + # Restore code blocks verbatim def _restore_code(m: "re.Match[str]") -> str: return _code_blocks[int(m.group(1))] @@ -981,22 +1018,57 @@ def _normalize_category(raw: Optional[str]) -> str: return raw -def _purge_noncanonical_nav_pages(tab_pages: List[Any]) -> None: - """Remove duplicate docs.json page keys (e.g. CLIPLoader when ClipLoader.mdx is already published).""" +def _purge_noncanonical_nav_pages(tab_pages: list[Any]) -> None: + """Replace noncanonical docs.json page keys with the published on-disk spelling. + + When a key's basename differs from the published name (e.g. ClipLoader vs + CLIPLoader.mdx on disk), replace it in place at the same position with the + canonical key rather than silently removing it. Keys whose locale has no + matching .mdx file are left unchanged so no navigation entry is lost. + """ for key in sorted(collect_page_keys(tab_pages)): parts = key.split("/") if not parts: continue locale_code = _locale_code_for_page_key(key) - if parts[-1] != published_node_name(parts[-1], locale_code): - remove_page_from_pages(tab_pages, key) + published = published_node_name(parts[-1], locale_code) + if parts[-1] == published: + continue + # Only rewrite when a real .mdx file exists for this locale + locale = next((c for c in LOCALE_CONFIGS if c["code"] == locale_code), None) + if locale is None: + continue + target_mdx = locale["builtin_dir"] / f"{published}.mdx" + if not target_mdx.exists(): + continue + # Locate the containing pages list BEFORE removing the key, then + # re-insert the canonical spelling at the same logical position. + target_pages = _find_tab_pages_for_key(tab_pages, key) + remove_page_from_pages(tab_pages, key) + canonical_key = "/".join([*parts[:-1], published]) + if target_pages is not None and canonical_key not in target_pages: + target_pages.append(canonical_key) + + +def _find_tab_pages_for_key(pages: list[Any], page_key: str) -> Optional[list[Any]]: + """Return the innermost 'pages' list containing page_key (recursive), or None.""" + for item in pages: + if isinstance(item, str): + if item == page_key: + return pages + elif isinstance(item, dict) and "pages" in item: + if page_key in collect_page_keys(item["pages"]): + found = _find_tab_pages_for_key(item["pages"], page_key) + if found is not None: + return found + return None def _place_page_in_nav( - tab_pages: List[Any], + tab_pages: list[Any], page_key: str, full_category: str, - locale: Dict[str, Any], + locale: dict[str, Any], ) -> None: """Insert page_key under the correct Built-in Nodes group for one locale.""" page_key = canonical_page_key(page_key) @@ -1035,7 +1107,7 @@ def _place_page_in_nav( def sync_node( node_name: str, dry_run: bool, -) -> Tuple[bool, Optional[str], List[str]]: +) -> tuple[bool, Optional[str], list[str]]: """Sync one node: en.md (+ zh.md / ja.md when present) -> MDX and copy assets.""" scanner_name, node_dir = resolve_source_node(node_name) en_md = node_dir / "en.md" @@ -1046,8 +1118,8 @@ def sync_node( published_en = published_node_name(scanner_name, "en") images_out = IMAGES_TARGET / published_en content_en = en_md.read_text(encoding="utf-8") - description = get_description_from_content(content_en) - synced_locales: List[str] = [] + description_en = get_description_from_content(content_en) + synced_locales: list[str] = [] for locale in LOCALE_CONFIGS: md_path = node_dir / locale["md_file"] @@ -1061,7 +1133,11 @@ def sync_node( content = md_path.read_text(encoding="utf-8") content = copy_assets_and_rewrite(content, node_dir, published_en, images_out, dry_run) content = _normalize_mdx_content(content) - mdx = build_frontmatter(scanner_name, description or f"Documentation for {scanner_name} node.") + content + # Description: prefer the localized overview first sentence, fall back to English. + locale_desc = get_description_from_content(content) + if not locale_desc: + locale_desc = description_en + mdx = build_frontmatter(scanner_name, locale_desc or f"Documentation for {scanner_name} node.") + content target_mdx = locale["builtin_dir"] / f"{published}.mdx" if not dry_run: @@ -1090,9 +1166,16 @@ def main(): print(f"ERROR: TARGET_DOCS not found: {TARGET_DOCS}") sys.exit(1) + if args.count < 0: + print("ERROR: --count must be >= 0") + sys.exit(1) + if args.node: _canonical, _src = resolve_source_node(args.node) - nodes = [_canonical] if (_src / "en.md").exists() else [] + if not (_src / "en.md").exists(): + print(f"ERROR: node '{args.node}' does not resolve to a directory containing en.md: {_src}") + sys.exit(1) + nodes = [_canonical] else: nodes = list_nodes_with_en_md() if args.mode == "test": @@ -1103,8 +1186,8 @@ def main(): if update_docs_json: print(f" docs.json path: {DOCS_JSON}") if not DOCS_JSON.exists(): - print(f" WARNING: docs.json not found at above path; navigation will not be updated.") - synced: List[Tuple[str, Optional[str], List[str]]] = [] + print(" WARNING: docs.json not found at above path; navigation will not be updated.") + synced: list[tuple[str, Optional[str], list[str]]] = [] for node_name in nodes: ok, category, synced_locales = sync_node(node_name, args.dry_run) if ok: @@ -1116,7 +1199,7 @@ def main(): else: with open(DOCS_JSON, "r", encoding="utf-8") as f: nav = json.load(f) - added: Dict[str, List[str]] = {cfg["code"]: [] for cfg in LOCALE_CONFIGS} + added: dict[str, list[str]] = {cfg["code"]: [] for cfg in LOCALE_CONFIGS} for node_name, full_category, synced_locales in synced: scanner_name = canonical_node_name(node_name) for locale in LOCALE_CONFIGS: @@ -1144,7 +1227,7 @@ def main(): _purge_noncanonical_nav_pages(tab_pages) _migrate_toplevel_groups_to_wrapper(tab_pages, locale["wrapper"]) - node_cat_map: Dict[str, str] = { + node_cat_map: dict[str, str] = { name: info.get("category", "") for name, info in _load_all_nodes_info().items() } @@ -1170,8 +1253,24 @@ def main(): _remove_empty_groups(tab_pages) _sort_pages_alphabetically(tab_pages, groups_first=False) - with open(DOCS_JSON, "w", encoding="utf-8") as f: - json.dump(nav, f, indent=2, ensure_ascii=False) + # Atomic write: serialize to a temp file in the same directory, then + # os.replace so a failed serialization never leaves docs.json truncated. + had_trailing_newline = False + try: + with open(DOCS_JSON, "r", encoding="utf-8") as f: + had_trailing_newline = f.read().endswith("\n") + except Exception: + had_trailing_newline = True + tmp_path = DOCS_JSON.with_suffix(".json.tmp") + try: + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(nav, f, indent=2, ensure_ascii=False) + if had_trailing_newline: + f.write("\n") + os.replace(tmp_path, DOCS_JSON) + finally: + if tmp_path.exists(): + tmp_path.unlink() any_added = any(added[code] for code in added) if any_added: print(f"docs.json: updated {DOCS_JSON}") diff --git a/docs-generation/tests/test_sync_helpers.py b/docs-generation/tests/test_sync_helpers.py new file mode 100644 index 000000000..61a233e45 --- /dev/null +++ b/docs-generation/tests/test_sync_helpers.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Unit tests for scripts.sync_to_comfy_docs normalization helpers. + +Covers the CodeRabbit review fixes: inline-code stashing, curly-brace +escaping, whitespace-preserving < escaping, description extraction +(image/table skipping, e.g. protection), acronym-preserving labels, +and non-destructive nav purge. +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +os.environ.setdefault("TARGET_DOCS", str(REPO_ROOT / ".." / "docs")) +os.environ.setdefault("COMFYUI_PATH", "") + +from scripts import sync_to_comfy_docs as sc # noqa: E402 + + +class SegToLabelTests(unittest.TestCase): + def test_title_cases_underscores(self): + self.assertEqual(sc._seg_to_label("custom_sampling"), "Custom Sampling") + + def test_preserves_acronyms(self): + self.assertEqual(sc._seg_to_label("SDXL"), "SDXL") + self.assertEqual(sc._seg_to_label("API_node"), "API Node") + self.assertEqual(sc._seg_to_label("bfl_flux"), "Bfl Flux") + + +class NormalizeMdxTests(unittest.TestCase): + def test_inline_code_preserved(self): + out = sc._normalize_mdx_content("Use `x <= 10` for limits.") + self.assertIn("`x <= 10`", out) + self.assertNotIn("<=", out.split("`")[1]) + + def test_fenced_code_preserved_verbatim(self): + out = sc._normalize_mdx_content("```python\nif x <= 10: print({1})\n```") + self.assertIn("if x <= 10: print({1})", out) + + def test_curly_braces_escaped_in_prose(self): + out = sc._normalize_mdx_content("The bbox {x, y} syntax.") + self.assertIn("{x, y}", out) + + def test_curly_braces_not_escaped_in_code(self): + out = sc._normalize_mdx_content("```json\n{\"a\": 1}\n```") + self.assertIn('{"a": 1}', out) + + def test_whitespace_after_lt_preserved(self): + out = sc._normalize_mdx_content("text < more") + self.assertIn("text < more", out) + + def test_paired_mintlify_component_kept(self): + out = sc._normalize_mdx_content("\nHeads up.\n") + self.assertIn("", out) + self.assertIn("", out) + + def test_orphaned_closing_tag_escaped(self): + out = sc._normalize_mdx_content("text more") + self.assertIn("</Note>", out) + + +class DescriptionExtractionTests(unittest.TestCase): + def test_skips_image_and_table_lines(self): + content = "![img](x.png)\n\n| A | B |\n|---|---|\n\nThis is the first sentence. Second sentence." + desc = sc.get_description_from_content(content) + self.assertTrue(desc.startswith("This is the first sentence.")) + + def test_does_not_cut_abbreviations(self): + desc = sc.get_description_from_content("This is a test e.g. with abbreviations. Second sentence.") + self.assertTrue(desc.startswith("This is a test e.g.")) + + def test_cjk_sentence_break(self): + desc = sc.get_description_from_content("这是第一句话。第二句话。") + self.assertEqual(desc, "这是第一句话。") + + def test_truncation_aligned_to_180(self): + long = "Word " * 100 + desc = sc.get_description_from_content(long) + self.assertLessEqual(len(desc), 180) + + +class PurgeNavTests(unittest.TestCase): + def _fake_locale(self): + """Point the en locale's builtin_dir at a temp dir with a real .mdx file.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + builtin_dir = Path(tmp.name) / "built-in-nodes" + builtin_dir.mkdir() + (builtin_dir / "CLIPTextEncodeControlnet.mdx").write_text("---\n---\n", encoding="utf-8") + orig = sc.LOCALE_CONFIGS[0]["builtin_dir"] + sc.LOCALE_CONFIGS[0]["builtin_dir"] = builtin_dir + self.addCleanup(lambda: sc.LOCALE_CONFIGS[0].__setitem__("builtin_dir", orig)) + # Drop any cached listing for the original dir so the fake dir is used + sc._locale_mdx_names.cache_clear() + return builtin_dir + + def test_replaces_canonical_when_mdx_exists(self): + self._fake_locale() + nav = [{ + "group": "Nodes", + "pages": [ + "built-in-nodes/ClipTextEncodeControlnet", + "built-in-nodes/KSampler", + ], + }] + sc._purge_noncanonical_nav_pages(nav) + keys = sc.collect_page_keys(nav) + self.assertIn("built-in-nodes/CLIPTextEncodeControlnet", keys) + self.assertNotIn("built-in-nodes/ClipTextEncodeControlnet", keys) + self.assertIn("built-in-nodes/KSampler", keys) + + def test_keeps_unknown_key_without_mdx(self): + self._fake_locale() + nav = [{"group": "Nodes", "pages": ["built-in-nodes/NonexistentNodeXYZ"]}] + sc._purge_noncanonical_nav_pages(nav) + keys = sc.collect_page_keys(nav) + self.assertIn("built-in-nodes/NonexistentNodeXYZ", keys) + + +class FrontmatterTests(unittest.TestCase): + def test_concrete_description_used(self): + fm = sc.build_frontmatter("Canny", "Extract all edge lines from photos.") + self.assertIn("Extract all edge lines from photos.", fm) + self.assertNotIn("Complete documentation for the Canny node", fm) + + def test_template_fallback_when_empty(self): + fm = sc.build_frontmatter("Canny", "") + self.assertIn("Complete documentation for the Canny node", fm) + + +if __name__ == "__main__": + unittest.main() From 73ca188a852cf9f5cf38b03f81258b922f3c25fe Mon Sep 17 00:00:00 2001 From: lin-bot23 Date: Wed, 12 Aug 2026 21:44:14 +0800 Subject: [PATCH 10/10] chore: drop provider-specific defaults; require API_BASE_URL/API_MODEL from env Remove the DeepSeek-default base URL and model from env.example and the scripts. The pipeline is provider-agnostic (any OpenAI-compatible API), so users must set API_BASE_URL/API_MODEL for their provider. Add a clear validation error in batch_generate_docs when either is missing. --- docs-generation/README.md | 2 +- docs-generation/env.example | 6 +++--- docs-generation/scripts/batch_generate_docs.py | 10 ++++++++-- docs-generation/scripts/batch_translate_docs.py | 4 ++-- docs-generation/scripts/check_config.py | 4 ++-- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/docs-generation/README.md b/docs-generation/README.md index 6256c58e9..46944f7c5 100644 --- a/docs-generation/README.md +++ b/docs-generation/README.md @@ -93,7 +93,7 @@ See `env.example`. Key ones: |-----|----------|---------| | `COMFYUI_PATH` | yes (scan/generate) | ComfyUI source checkout | | `LLM_API_KEY` | yes (LLM steps) | OpenAI-compatible API key (any provider; `DEEPSEEK_API_KEY` also accepted for back-compat) | -| `API_BASE_URL` / `API_MODEL` | no | OpenAI-compatible endpoint + model (defaults: DeepSeek) | +| `API_BASE_URL` / `API_MODEL` | yes (LLM steps) | OpenAI-compatible endpoint + model for your provider (no default; must be set) | | `EMBEDDED_DOCS_PATH` | no | embedded-docs repo root (defaults to repo root) | | `TARGET_DOCS` | sync step | Comfy-Org/docs checkout | | `COMFYUI_FRONTEND_PATH` | param-translation step | ComfyUI frontend repo | diff --git a/docs-generation/env.example b/docs-generation/env.example index c640a567f..ea6bf8e56 100644 --- a/docs-generation/env.example +++ b/docs-generation/env.example @@ -19,10 +19,10 @@ TARGET_DOCS=/path/to/comfy/docs # --- LLM API configuration -------------------------------------------------- # The pipeline uses an OpenAI-compatible chat API for doc generation & translation. # Any OpenAI-compatible provider works (DeepSeek, OpenAI, OpenRouter, local -# vLLM/Ollama, etc.) — set the base URL and model for your provider. +# vLLM/Ollama, etc.): set the base URL and model for your provider. LLM_API_KEY=your_api_key_here -API_BASE_URL=https://api.deepseek.com -API_MODEL=deepseek-chat +API_BASE_URL= +API_MODEL= # --- Batch processing ------------------------------------------------------- BATCH_SIZE=5 diff --git a/docs-generation/scripts/batch_generate_docs.py b/docs-generation/scripts/batch_generate_docs.py index 1948c8f57..5d0311419 100644 --- a/docs-generation/scripts/batch_generate_docs.py +++ b/docs-generation/scripts/batch_generate_docs.py @@ -33,8 +33,8 @@ # Configuration API_KEY = os.getenv('LLM_API_KEY') or os.getenv('DEEPSEEK_API_KEY') -API_BASE_URL = os.getenv('API_BASE_URL', 'https://api.deepseek.com') -API_MODEL = os.getenv('API_MODEL', 'deepseek-chat') +API_BASE_URL = os.getenv('API_BASE_URL', '').strip() +API_MODEL = os.getenv('API_MODEL', '').strip() BATCH_SIZE = int(os.getenv('BATCH_SIZE', '5')) MAX_RETRIES = int(os.getenv('MAX_RETRIES', '3')) DELAY_BETWEEN_REQUESTS = int(os.getenv('DELAY_BETWEEN_REQUESTS', '2')) @@ -106,6 +106,12 @@ class AIDocGenerator: def __init__(self): if not API_KEY: raise ValueError("❌ LLM_API_KEY not found, please configure it in .env file") + if not API_BASE_URL or not API_MODEL: + raise ValueError( + "❌ API_BASE_URL / API_MODEL not configured. Point them at your " + "OpenAI-compatible provider (e.g. DeepSeek, OpenAI, OpenRouter, " + "local vLLM/Ollama) in .env." + ) self.client = OpenAI( api_key=API_KEY, diff --git a/docs-generation/scripts/batch_translate_docs.py b/docs-generation/scripts/batch_translate_docs.py index 6d5dc4131..a56ac838d 100644 --- a/docs-generation/scripts/batch_translate_docs.py +++ b/docs-generation/scripts/batch_translate_docs.py @@ -66,8 +66,8 @@ # AI Configuration DEFAULT_API_KEY = os.getenv('LLM_API_KEY') or os.getenv('DEEPSEEK_API_KEY', '') -DEFAULT_BASE_URL = os.getenv('API_BASE_URL', 'https://api.deepseek.com') -DEFAULT_MODEL = os.getenv('API_MODEL', 'deepseek-chat') +DEFAULT_BASE_URL = os.getenv('API_BASE_URL', '').strip() +DEFAULT_MODEL = os.getenv('API_MODEL', '').strip() DEFAULT_BATCH_SIZE = 5 # Custom log level for success diff --git a/docs-generation/scripts/check_config.py b/docs-generation/scripts/check_config.py index a6a74eefe..866ae6073 100644 --- a/docs-generation/scripts/check_config.py +++ b/docs-generation/scripts/check_config.py @@ -100,8 +100,8 @@ def check_config(): all_ok = False # Check API settings - api_base = os.getenv('API_BASE_URL', 'https://api.deepseek.com') - api_model = os.getenv('API_MODEL', 'deepseek-chat') + api_base = os.getenv('API_BASE_URL', '').strip() + api_model = os.getenv('API_MODEL', '').strip() print(f"✅ API_BASE_URL: {api_base}") print(f"✅ API_MODEL: {api_model}")