From 51635cb885a40d0c4f21410f711d0bd640ed7674 Mon Sep 17 00:00:00 2001 From: buxinzi2233 Date: Fri, 14 Aug 2026 12:40:09 +0800 Subject: [PATCH 1/2] feat: expose V100 LoRA FP32 protection in WebUI --- tests/test_config_service.py | 105 ++++++++++++++++++++++ webui/explanations/guides/cn/_fields.json | 1 + webui/explanations/guides/en/_fields.json | 1 + webui/explanations/guides/ja/_fields.json | 1 + webui/explanations/guides/ko/_fields.json | 1 + webui/services/config_service.py | 50 ++++++++++- 6 files changed, 157 insertions(+), 2 deletions(-) diff --git a/tests/test_config_service.py b/tests/test_config_service.py index ba1525c62..49370f955 100644 --- a/tests/test_config_service.py +++ b/tests/test_config_service.py @@ -105,6 +105,111 @@ def test_sample_decode_inline_validation_rejects_unknown_mode(): ] +def test_lora_fp32_compute_is_editable_tri_state_and_round_trips( + monkeypatch, tmp_path +): + configs = tmp_path / "configs" + methods = configs / "gui-methods" + overlays = configs / "custom" / "variants" + methods.mkdir(parents=True) + overlays.mkdir(parents=True) + (configs / "base.toml").write_text( + 'network_module = "networks.lora_anima"\n', encoding="utf-8" + ) + (configs / "model.toml").write_text("", encoding="utf-8") + (configs / "presets.toml").write_text("[default]\n", encoding="utf-8") + (methods / "lora.toml").write_text("network_dim = 16\n", encoding="utf-8") + + monkeypatch.setattr(config_service, "CONFIGS_DIR", configs) + monkeypatch.setattr(config_service, "GUI_METHODS_DIR", methods) + monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml") + monkeypatch.setattr(config_service, "CUSTOM_VARIANTS_DIR", overlays) + + result = config_service.build_merged_config("lora", "default", lang="en") + field = next(f for f in result["fields"] if f["key"] == "lora_fp32_compute") + assert field["value"] == "auto" + assert field["origin"] == "base" + assert field["field_type"] == "select" + assert field["group"] == "Performance" + assert field["read_only"] is False + assert field["options"] == ["auto", "true", "false"] + assert field["description"] + + config_service.save_variant_config("lora", {"lora_fp32_compute": "true"}) + saved = toml.loads((overlays / "lora.toml").read_text(encoding="utf-8")) + assert saved["lora_fp32_compute"] is True + + merged, origin = config_service.merged_gui_variant_preset("lora", "default") + assert merged["lora_fp32_compute"] is True + assert origin["lora_fp32_compute"] == "method" + + config_service.save_variant_config("lora", {"lora_fp32_compute": "false"}) + saved = toml.loads((overlays / "lora.toml").read_text(encoding="utf-8")) + assert saved["lora_fp32_compute"] is False + + config_service.save_variant_config("lora", {"lora_fp32_compute": "auto"}) + saved = toml.loads((overlays / "lora.toml").read_text(encoding="utf-8")) + assert "lora_fp32_compute" not in saved + + +def test_lora_fp32_compute_not_shown_for_non_lora_network(monkeypatch, tmp_path): + configs = tmp_path / "configs" + methods = configs / "gui-methods" + methods.mkdir(parents=True) + (configs / "base.toml").write_text( + 'network_module = "networks.lora_anima"\n', encoding="utf-8" + ) + (configs / "model.toml").write_text("", encoding="utf-8") + (configs / "presets.toml").write_text("[default]\n", encoding="utf-8") + (methods / "easycontrol.toml").write_text( + 'network_module = "networks.methods.easycontrol"\n', encoding="utf-8" + ) + + monkeypatch.setattr(config_service, "CONFIGS_DIR", configs) + monkeypatch.setattr(config_service, "GUI_METHODS_DIR", methods) + monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml") + monkeypatch.setattr( + config_service, "CUSTOM_VARIANTS_DIR", configs / "custom" / "variants" + ) + + result = config_service.build_merged_config( + "easycontrol", "default", lang="en" + ) + assert "lora_fp32_compute" not in {field["key"] for field in result["fields"]} + + +def test_lora_fp32_compute_validation_rejects_unknown_mode(): + assert validate_config({"lora_fp32_compute": "sometimes"}) == [ + "lora_fp32_compute must be auto, true, or false" + ] + + +def test_create_custom_preset_normalizes_lora_fp32_compute(monkeypatch, tmp_path): + configs = tmp_path / "configs" + custom_presets = configs / "custom" / "presets" + custom_presets.mkdir(parents=True) + (configs / "presets.toml").write_text("[default]\n", encoding="utf-8") + + monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml") + monkeypatch.setattr(config_service, "CUSTOM_PRESETS_DIR", custom_presets) + + config_service.create_custom_preset( + "auto-protection", {"lora_fp32_compute": "auto"} + ) + auto_saved = toml.loads( + (custom_presets / "auto-protection.toml").read_text(encoding="utf-8") + ) + assert "lora_fp32_compute" not in auto_saved + + config_service.create_custom_preset( + "forced-protection", {"lora_fp32_compute": "true"} + ) + forced_saved = toml.loads( + (custom_presets / "forced-protection.toml").read_text(encoding="utf-8") + ) + assert forced_saved["lora_fp32_compute"] is True + + def test_create_custom_preset_strips_neutral_resume_defaults(monkeypatch, tmp_path): """A fresh WebUI preset must not turn an empty warm-start into a path.""" configs = tmp_path / "configs" diff --git a/webui/explanations/guides/cn/_fields.json b/webui/explanations/guides/cn/_fields.json index 56f8ec551..3facf1907 100644 --- a/webui/explanations/guides/cn/_fields.json +++ b/webui/explanations/guides/cn/_fields.json @@ -56,6 +56,7 @@ "torch_compile": "为前向传播启用torch.compile。初始编译后训练速度加快。以原生形状编译每个DiT块 — 每个令牌数桶系列一张计算图,无静态填充。", "compile_dynamic_seq": "在每个已编译DiT块内部仅将原生令牌序列轴标记为动态。free-fit形状必须启用,使同一分辨率层级复用一张图,而不是每个令牌数各编译一张。", "v100_flash_stability": "V100 FlashAttention诊断模式。off使用完整Flash;hybrid将交叉注意力路由到Torch SDPA;safe保留Flash并检查注意力边界有限性。只有本机全部验证门禁通过后才生成使用off的生产预设;safe仅用于诊断。", + "lora_fp32_compute": "LoRA 系列的 FP32 保护模式。auto:仅在 V100/sm_70 + FP16 训练时自动开启;true:强制使用 FP32 适配器计算;false:明确关闭。保护生效时,适配器参数、优化器输入和保存的检查点张量会保持 FP32,冻结的 DiT 权重仍使用模型精度。通常保持 auto。", "debug_finite_checks": "当注意力输入/输出、块残差、loss或梯度出现NaN/Inf时立即失败。它只用于诊断,不修复或隐藏无效值。", "cache_llm_adapter_outputs": "将LLM适配器层输出缓存到磁盘。避免每轮重新计算文本编码器投影。", "masked_loss": "仅在未被遮罩的区域(如排除文字气泡)应用损失。需要post_image_dataset/masks/(或旧版masks/merged/)中的遮罩文件 — 运行`make mask`生成。", diff --git a/webui/explanations/guides/en/_fields.json b/webui/explanations/guides/en/_fields.json index 90cdf0924..06b2e39d2 100644 --- a/webui/explanations/guides/en/_fields.json +++ b/webui/explanations/guides/en/_fields.json @@ -56,6 +56,7 @@ "torch_compile": "Enable torch.compile for the forward pass. Faster training after initial compilation. Compiles each DiT block at native shapes — one graph per token-count bucket family, no static padding.", "compile_dynamic_seq": "Mark only the native token-sequence axis dynamic inside each compiled DiT block. Required with free-fit shapes so a tier reuses one graph instead of compiling one graph per token count.", "v100_flash_stability": "V100 FlashAttention diagnostic mode. off uses full Flash; hybrid routes cross-attention to Torch SDPA; safe keeps Flash and enables attention-boundary finite checks. A generated production preset uses off only after every local validation gate passes; safe is diagnostic only.", + "lora_fp32_compute": "FP32 protection for the LoRA family. auto enables it only for V100/sm_70 FP16 training; true forces FP32 adapter compute; false explicitly disables it. When active, adapter parameters, optimizer inputs, and saved checkpoint tensors stay FP32 while frozen DiT weights keep the model dtype. Usually leave this on auto.", "debug_finite_checks": "Fail immediately if attention inputs/outputs, block residuals, loss, or gradients contain NaN or Inf. Diagnostic guard only; it never repairs or hides invalid values.", "cache_llm_adapter_outputs": "Cache the LLM adapter layer outputs to disk. Avoids recomputing text encoder projections each epoch.", "masked_loss": "Apply loss only to non-masked regions (e.g., exclude text bubbles). Requires mask files in post_image_dataset/masks/ (or legacy masks/merged/) — run `make mask` to generate them.", diff --git a/webui/explanations/guides/ja/_fields.json b/webui/explanations/guides/ja/_fields.json index 9199ad51f..be60a493f 100644 --- a/webui/explanations/guides/ja/_fields.json +++ b/webui/explanations/guides/ja/_fields.json @@ -56,6 +56,7 @@ "torch_compile": "フォワードパスに torch.compile を有効化します。初期コンパイル後は学習が高速化されます。各 DiT ブロックをネイティブ形状でコンパイルします — トークン数バケットファミリーごとに 1 グラフ、静的パディングなし。", "compile_dynamic_seq": "コンパイル済みDiTブロック内でネイティブトークン列軸だけを動的にします。free-fit形状で、トークン数ごとの再コンパイルを避けるために必要です。", "v100_flash_stability": "V100 FlashAttention診断モード。offは完全なFlash、hybridはクロスアテンションをTorch SDPAへ送り、safeはFlashのまま有限値検査を追加します。ローカル検証ゲートをすべて通過した場合のみoffの本番プリセットを生成し、safeは診断専用です。", + "lora_fp32_compute": "LoRA 系列の FP32 保護モード。auto は V100/sm_70 + FP16 学習時のみ自動的に有効化し、true は FP32 アダプター計算を強制、false は明示的に無効化します。有効時はアダプターパラメーター、オプティマイザー入力、保存チェックポイントを FP32 に保ち、凍結 DiT 重みはモデル精度のままです。通常は auto を使用してください。", "debug_finite_checks": "アテンション入出力、ブロック残差、loss、勾配にNaN/Infが現れた時点で失敗します。無効値を修復または隠蔽しません。", "cache_llm_adapter_outputs": "LLM アダプターレイヤーの出力をディスクにキャッシュします。エポックごとにテキストエンコーダープロジェクションを再計算するのを避けます。", "masked_loss": "マスクされていない領域 (例: テキストバブルを除く) にのみロスを適用します。post_image_dataset/masks/ (またはレガシー masks/merged/) にマスクファイルが必要です — `make mask` で生成してください。", diff --git a/webui/explanations/guides/ko/_fields.json b/webui/explanations/guides/ko/_fields.json index d93cf3708..5d8228dd6 100644 --- a/webui/explanations/guides/ko/_fields.json +++ b/webui/explanations/guides/ko/_fields.json @@ -56,6 +56,7 @@ "torch_compile": "포워드 패스에 torch.compile 활성화. 초기 컴파일 후 학습 속도 향상. 각 DiT 블록을 네이티브 형상으로 컴파일 — 토큰 수 버킷 패밀리당 하나의 그래프, 정적 패딩 없음.", "compile_dynamic_seq": "컴파일된 DiT 블록에서 네이티브 토큰 시퀀스 축만 동적으로 표시합니다. free-fit 형상에서 토큰 수마다 다시 컴파일하지 않고 티어당 하나의 그래프를 재사용하려면 필요합니다.", "v100_flash_stability": "V100 FlashAttention 진단 모드. off는 전체 Flash, hybrid는 크로스 어텐션을 Torch SDPA로 보내고, safe는 Flash를 유지하며 유한성 검사를 추가합니다. 모든 로컬 검증 게이트를 통과한 경우에만 off 운영 프리셋을 생성하며 safe는 진단 전용입니다.", + "lora_fp32_compute": "LoRA 계열 FP32 보호 모드입니다. auto는 V100/sm_70 + FP16 학습에서만 자동 활성화하고, true는 FP32 어댑터 계산을 강제하며, false는 명시적으로 비활성화합니다. 활성화되면 어댑터 파라미터, 옵티마이저 입력, 저장 체크포인트 텐서는 FP32를 유지하고 동결된 DiT 가중치는 모델 정밀도를 유지합니다. 일반적으로 auto를 사용하세요.", "debug_finite_checks": "어텐션 입출력, 블록 잔차, loss 또는 그래디언트에 NaN/Inf가 나타나면 즉시 실패합니다. 잘못된 값을 수정하거나 숨기지 않습니다.", "cache_llm_adapter_outputs": "LLM 어댑터 레이어 출력을 디스크에 캐싱. 매 에폭 텍스트 인코더 투영 재계산 회피.", "masked_loss": "마스크되지 않은 영역에만 손실 적용 (예: 말풍선 제외). post_image_dataset/masks/ (또는 레거시 masks/merged/) 에 마스크 파일 필요 — `make mask` 로 생성.", diff --git a/webui/services/config_service.py b/webui/services/config_service.py index 84804e32e..7b1591aeb 100644 --- a/webui/services/config_service.py +++ b/webui/services/config_service.py @@ -117,6 +117,7 @@ def get_path_overrides( _ATTN_MODES = ["flash", "torch", "mem_efficient", "sageattn", "flex", "xformers"] _SAMPLER_CHOICES = ["euler", "er_sde"] _SAMPLE_DECODE_INLINE_CHOICES = ["auto", "true", "false"] +_LORA_FP32_COMPUTE_CHOICES = ["auto", "true", "false"] _BASE_COMPUTE_CHOICES = ["bf16", "w8a16_convrot", "w8a8_convrot"] _CONVROT_HADAMARD_CHOICES = ["sylvester", "regular"] _CONVROT_SCOPE_CHOICES = [ @@ -197,6 +198,7 @@ def get_path_overrides( ], "sample_sampler": ["euler", "er_sde"], "sample_decode_inline": _SAMPLE_DECODE_INLINE_CHOICES, + "lora_fp32_compute": _LORA_FP32_COMPUTE_CHOICES, } _GROUPS = { @@ -275,6 +277,7 @@ def get_path_overrides( "cache_llm_adapter_outputs", "masked_loss", "mixed_precision", + "lora_fp32_compute", "vae_chunk_size", "vae_disable_cache", "cache_latents", @@ -324,7 +327,11 @@ def get_path_overrides( # Most inherited fields are intentionally read-only in the WebUI. Preview decode # timing is safe to override per variant, and save_variant_config persists that # choice in the user overlay without touching base.toml. -_EDITABLE_INHERITED_KEYS = {"sample_decode_inline", *_CONVROT_FIELDS} +_EDITABLE_INHERITED_KEYS = { + "sample_decode_inline", + "lora_fp32_compute", + *_CONVROT_FIELDS, +} # Resume & Warm-start defaults. These keys are NOT in base.toml (which is # overwritten by `make update`), so merged_gui_variant_preset injects them with @@ -353,6 +360,21 @@ def _strip_neutral_resume_defaults(data: dict) -> dict: } +def _normalize_lora_fp32_compute(data: dict) -> dict: + """Convert the WebUI tri-state sentinel to the trainer's optional bool.""" + out = dict(data) + if "lora_fp32_compute" not in out: + return out + normalized = str(out["lora_fp32_compute"]).strip().lower() + if normalized not in _LORA_FP32_COMPUTE_CHOICES: + raise ValueError("lora_fp32_compute must be auto, true, or false") + if normalized == "auto": + out.pop("lora_fp32_compute", None) + else: + out["lora_fp32_compute"] = normalized == "true" + return out + + _BASIC = { "base_compute", "learning_rate", @@ -552,7 +574,7 @@ def create_custom_preset(name: str, data: dict) -> list[str]: CUSTOM_PRESETS_DIR.mkdir(parents=True, exist_ok=True) _save( CUSTOM_PRESETS_DIR / f"{name}.toml", - _strip_neutral_resume_defaults(data), + _normalize_lora_fp32_compute(_strip_neutral_resume_defaults(data)), ) return list_presets() @@ -685,6 +707,17 @@ def _merge_layer(layer: dict, src: str) -> None: _merge_layer(pset, "preset") _merge_layer(meth, "method") + # The V100 FP16 protection is intentionally tri-state. An omitted + # lora_fp32_compute lets train.py auto-enable it only for V100/sm_70 FP16; + # explicit true/false force the user's choice. Surface the omitted state as + # "auto" in the WebUI without writing that sentinel into the training TOML. + if ( + merged.get("network_module") == "networks.lora_anima" + and "lora_fp32_compute" not in merged + ): + merged["lora_fp32_compute"] = "auto" + origin["lora_fp32_compute"] = "base" + # Inject virtual keys from [[datasets]] variant_override = _validation_enabled_from_datasets(meth.get("datasets")) if variant_override is not None: @@ -1101,6 +1134,14 @@ def validate_config(data: dict) -> list[str]: ) if not valid: errors.append("sample_decode_inline must be auto, true, or false") + if "lora_fp32_compute" in data: + value = data["lora_fp32_compute"] + valid = isinstance(value, bool) or ( + isinstance(value, str) + and value.strip().lower() in _LORA_FP32_COMPUTE_CHOICES + ) + if not valid: + errors.append("lora_fp32_compute must be auto, true, or false") if "base_compute" in data: value = str(data["base_compute"] or "").strip().lower() if value not in _BASE_COMPUTE_CHOICES: @@ -1257,6 +1298,9 @@ def save_variant_config(variant: str, data: dict) -> None: else: current[key] = normalized == "true" continue + if key == "lora_fp32_compute": + current[key] = value + continue if _is_neutral_resume_value(key, value): current.pop(key, None) continue @@ -1266,6 +1310,8 @@ def save_variant_config(variant: str, data: dict) -> None: if extras: current.update(extras) + current = _normalize_lora_fp32_compute(current) + # Validate the prospective effective method config, not only the sparse # overlay. Built-in LoKR flags live in gui-methods/lokr.toml while a user # overlay may contain only network_dim, so validating the overlay alone From 3a490b66ac02010067daf69629ecd9537acbb388 Mon Sep 17 00:00:00 2001 From: LingyeSoul <57278745+LingyeSoul@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:52:48 +0800 Subject: [PATCH 2/2] fix: preserve LoRA FP32 tri-state semantics --- library/config/io.py | 37 ++++++++++- tests/test_config_service.py | 72 ++++++++++++++++++++- webui/explanations/guides/cn/_fields.json | 2 +- webui/explanations/guides/en/_fields.json | 2 +- webui/explanations/guides/ja/_fields.json | 2 +- webui/explanations/guides/ko/_fields.json | 2 +- webui/services/config_service.py | 78 +++++++++++++++++------ 7 files changed, 169 insertions(+), 26 deletions(-) diff --git a/library/config/io.py b/library/config/io.py index e7f339854..35ff7cbb1 100644 --- a/library/config/io.py +++ b/library/config/io.py @@ -42,6 +42,7 @@ # on a typo — a path-string typo fails loudly (file not found), but a # comparison typo would route a gui variant through the flat-methods path. GUI_METHODS_SUBDIR = "gui-methods" +_LORA_FP32_COMPUTE_KEY = "lora_fp32_compute" _SNAPSHOT_SUFFIX = ".snapshot.toml" _EMPTY_STRING_IS_UNSET = frozenset({"network_weights"}) _DUMP_SKIP_KEYS = { @@ -241,6 +242,31 @@ def _resolve_method_path( return os.path.join(configs_dir, methods_subdir, f"{method}.toml") +def _canonicalize_lora_fp32_compute_layer(data: dict) -> dict: + """Move legacy ``network_args`` values into normal layer precedence.""" + out = dict(data) + network_args = out.get("network_args") + if not isinstance(network_args, list): + return out + + cleaned: list = [] + legacy_value = None + for item in network_args: + if isinstance(item, str) and "=" in item: + key, value = item.split("=", 1) + if key.strip() == _LORA_FP32_COMPUTE_KEY: + legacy_value = value.strip().lower() + continue + cleaned.append(item) + if legacy_value is not None: + out[_LORA_FP32_COMPUTE_KEY] = legacy_value + if cleaned: + out["network_args"] = cleaned + else: + out.pop("network_args", None) + return out + + def _load_gui_method_merged(configs_dir: str, method: str) -> tuple[dict, str]: """Load a gui-methods variant as ``{**builtin, **overlay}``. @@ -261,10 +287,12 @@ def _load_gui_method_merged(configs_dir: str, method: str) -> tuple[dict, str]: merged: dict = {} if os.path.exists(builtin_path): with open(builtin_path, "r", encoding="utf-8") as f: - merged.update(toml.load(f)) + merged.update(_canonicalize_lora_fp32_compute_layer(toml.load(f))) if os.path.exists(overlay_path): with open(overlay_path, "r", encoding="utf-8") as f: - merged.update(toml.load(f)) # overlay wins on conflict + merged.update( + _canonicalize_lora_fp32_compute_layer(toml.load(f)) + ) # overlay wins on conflict # source_path: absolute path for _flatten_toml's schema line lookup # (prefers the overlay so line numbers point at the user-owned file). # source_tag: display path for provenance / print-config output. @@ -796,6 +824,7 @@ def load_method_preset( for kind, path, tag, raw in _iter_method_preset_layers( preset, configs_dir, methods_subdir, method, require_files=True ): + raw = _canonicalize_lora_fp32_compute_layer(raw) # Preset sections are flat scalar tables, so wrap them as # ``{preset: section}`` to mirror base/method top-level section tables — # ``_flatten_toml`` then descends one level into the section contents. @@ -806,6 +835,10 @@ def load_method_preset( merged[k] = v provenance[k] = tag + if str(merged.get(_LORA_FP32_COMPUTE_KEY, "")).strip().lower() == "auto": + merged.pop(_LORA_FP32_COMPUTE_KEY, None) + provenance.pop(_LORA_FP32_COMPUTE_KEY, None) + _drop_unset_config_values(merged, provenance) if return_provenance: diff --git a/tests/test_config_service.py b/tests/test_config_service.py index 49370f955..a5a882153 100644 --- a/tests/test_config_service.py +++ b/tests/test_config_service.py @@ -149,18 +149,79 @@ def test_lora_fp32_compute_is_editable_tri_state_and_round_trips( config_service.save_variant_config("lora", {"lora_fp32_compute": "auto"}) saved = toml.loads((overlays / "lora.toml").read_text(encoding="utf-8")) - assert "lora_fp32_compute" not in saved + assert saved["lora_fp32_compute"] == "auto" + + training = load_method_preset( + "lora", "default", configs_dir=str(configs), methods_subdir="gui-methods" + ) + assert "lora_fp32_compute" not in training + + +def test_lora_fp32_auto_clears_preset_and_legacy_network_args(monkeypatch, tmp_path): + configs = tmp_path / "configs" + methods = configs / "gui-methods" + overlays = configs / "custom" / "variants" + custom_presets = configs / "custom" / "presets" + methods.mkdir(parents=True) + overlays.mkdir(parents=True) + custom_presets.mkdir(parents=True) + (configs / "base.toml").write_text( + 'network_module = "networks.lora_anima"\n', encoding="utf-8" + ) + (configs / "model.toml").write_text("", encoding="utf-8") + (configs / "presets.toml").write_text("[default]\n", encoding="utf-8") + (custom_presets / "forced.toml").write_text( + 'network_args = ["lora_fp32_compute=true"]\n', encoding="utf-8" + ) + (methods / "lora.toml").write_text( + 'network_args = ["use_timestep_mask=true", "lora_fp32_compute=false"]\n', + encoding="utf-8", + ) + + monkeypatch.setattr(config_service, "CONFIGS_DIR", configs) + monkeypatch.setattr(config_service, "GUI_METHODS_DIR", methods) + monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml") + monkeypatch.setattr(config_service, "CUSTOM_PRESETS_DIR", custom_presets) + monkeypatch.setattr(config_service, "CUSTOM_VARIANTS_DIR", overlays) + + result = config_service.build_merged_config("lora", "forced", lang="en") + field = next(f for f in result["fields"] if f["key"] == "lora_fp32_compute") + assert field["value"] == "false" + assert field["origin"] == "method" + + config_service.save_variant_config("lora", {"lora_fp32_compute": "true"}) + training = load_method_preset( + "lora", "forced", configs_dir=str(configs), methods_subdir="gui-methods" + ) + assert str(training["lora_fp32_compute"]).lower() == "true" + assert training["network_args"] == ["use_timestep_mask=true"] + + config_service.save_variant_config("lora", {"lora_fp32_compute": "auto"}) + merged, origin = config_service.merged_gui_variant_preset("lora", "forced") + assert merged["lora_fp32_compute"] == "auto" + assert origin["lora_fp32_compute"] == "method" + + training = load_method_preset( + "lora", "forced", configs_dir=str(configs), methods_subdir="gui-methods" + ) + assert "lora_fp32_compute" not in training + assert training["network_args"] == ["use_timestep_mask=true"] def test_lora_fp32_compute_not_shown_for_non_lora_network(monkeypatch, tmp_path): configs = tmp_path / "configs" methods = configs / "gui-methods" + custom_presets = configs / "custom" / "presets" methods.mkdir(parents=True) + custom_presets.mkdir(parents=True) (configs / "base.toml").write_text( 'network_module = "networks.lora_anima"\n', encoding="utf-8" ) (configs / "model.toml").write_text("", encoding="utf-8") (configs / "presets.toml").write_text("[default]\n", encoding="utf-8") + (custom_presets / "forced.toml").write_text( + "lora_fp32_compute = true\n", encoding="utf-8" + ) (methods / "easycontrol.toml").write_text( 'network_module = "networks.methods.easycontrol"\n', encoding="utf-8" ) @@ -168,12 +229,13 @@ def test_lora_fp32_compute_not_shown_for_non_lora_network(monkeypatch, tmp_path) monkeypatch.setattr(config_service, "CONFIGS_DIR", configs) monkeypatch.setattr(config_service, "GUI_METHODS_DIR", methods) monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml") + monkeypatch.setattr(config_service, "CUSTOM_PRESETS_DIR", custom_presets) monkeypatch.setattr( config_service, "CUSTOM_VARIANTS_DIR", configs / "custom" / "variants" ) result = config_service.build_merged_config( - "easycontrol", "default", lang="en" + "easycontrol", "forced", lang="en" ) assert "lora_fp32_compute" not in {field["key"] for field in result["fields"]} @@ -184,6 +246,12 @@ def test_lora_fp32_compute_validation_rejects_unknown_mode(): ] +def test_lora_fp32_compute_help_documents_reliable_mode_override(): + description = config_service._field_desc("lora_fp32_compute", "en") + assert description is not None + assert "reliable block-swap mode overrides false" in description + + def test_create_custom_preset_normalizes_lora_fp32_compute(monkeypatch, tmp_path): configs = tmp_path / "configs" custom_presets = configs / "custom" / "presets" diff --git a/webui/explanations/guides/cn/_fields.json b/webui/explanations/guides/cn/_fields.json index 3facf1907..ce2396c21 100644 --- a/webui/explanations/guides/cn/_fields.json +++ b/webui/explanations/guides/cn/_fields.json @@ -56,7 +56,7 @@ "torch_compile": "为前向传播启用torch.compile。初始编译后训练速度加快。以原生形状编译每个DiT块 — 每个令牌数桶系列一张计算图,无静态填充。", "compile_dynamic_seq": "在每个已编译DiT块内部仅将原生令牌序列轴标记为动态。free-fit形状必须启用,使同一分辨率层级复用一张图,而不是每个令牌数各编译一张。", "v100_flash_stability": "V100 FlashAttention诊断模式。off使用完整Flash;hybrid将交叉注意力路由到Torch SDPA;safe保留Flash并检查注意力边界有限性。只有本机全部验证门禁通过后才生成使用off的生产预设;safe仅用于诊断。", - "lora_fp32_compute": "LoRA 系列的 FP32 保护模式。auto:仅在 V100/sm_70 + FP16 训练时自动开启;true:强制使用 FP32 适配器计算;false:明确关闭。保护生效时,适配器参数、优化器输入和保存的检查点张量会保持 FP32,冻结的 DiT 权重仍使用模型精度。通常保持 auto。", + "lora_fp32_compute": "LoRA 系列的 FP32 保护模式。auto:仅在 V100/sm_70 + FP16 训练时自动开启;true:强制使用 FP32 适配器计算;false:通常关闭,但 V100 可靠块交换模式会为维持安全约束而覆盖 false。保护生效时,适配器参数、优化器输入和保存的检查点张量会保持 FP32,冻结的 DiT 权重仍使用模型精度。通常保持 auto。", "debug_finite_checks": "当注意力输入/输出、块残差、loss或梯度出现NaN/Inf时立即失败。它只用于诊断,不修复或隐藏无效值。", "cache_llm_adapter_outputs": "将LLM适配器层输出缓存到磁盘。避免每轮重新计算文本编码器投影。", "masked_loss": "仅在未被遮罩的区域(如排除文字气泡)应用损失。需要post_image_dataset/masks/(或旧版masks/merged/)中的遮罩文件 — 运行`make mask`生成。", diff --git a/webui/explanations/guides/en/_fields.json b/webui/explanations/guides/en/_fields.json index 06b2e39d2..1d0a5e486 100644 --- a/webui/explanations/guides/en/_fields.json +++ b/webui/explanations/guides/en/_fields.json @@ -56,7 +56,7 @@ "torch_compile": "Enable torch.compile for the forward pass. Faster training after initial compilation. Compiles each DiT block at native shapes — one graph per token-count bucket family, no static padding.", "compile_dynamic_seq": "Mark only the native token-sequence axis dynamic inside each compiled DiT block. Required with free-fit shapes so a tier reuses one graph instead of compiling one graph per token count.", "v100_flash_stability": "V100 FlashAttention diagnostic mode. off uses full Flash; hybrid routes cross-attention to Torch SDPA; safe keeps Flash and enables attention-boundary finite checks. A generated production preset uses off only after every local validation gate passes; safe is diagnostic only.", - "lora_fp32_compute": "FP32 protection for the LoRA family. auto enables it only for V100/sm_70 FP16 training; true forces FP32 adapter compute; false explicitly disables it. When active, adapter parameters, optimizer inputs, and saved checkpoint tensors stay FP32 while frozen DiT weights keep the model dtype. Usually leave this on auto.", + "lora_fp32_compute": "FP32 protection for the LoRA family. auto enables it only for V100/sm_70 FP16 training; true forces FP32 adapter compute; false disables it except that V100 reliable block-swap mode overrides false to preserve its safety contract. When active, adapter parameters, optimizer inputs, and saved checkpoint tensors stay FP32 while frozen DiT weights keep the model dtype. Usually leave this on auto.", "debug_finite_checks": "Fail immediately if attention inputs/outputs, block residuals, loss, or gradients contain NaN or Inf. Diagnostic guard only; it never repairs or hides invalid values.", "cache_llm_adapter_outputs": "Cache the LLM adapter layer outputs to disk. Avoids recomputing text encoder projections each epoch.", "masked_loss": "Apply loss only to non-masked regions (e.g., exclude text bubbles). Requires mask files in post_image_dataset/masks/ (or legacy masks/merged/) — run `make mask` to generate them.", diff --git a/webui/explanations/guides/ja/_fields.json b/webui/explanations/guides/ja/_fields.json index be60a493f..ef1e54709 100644 --- a/webui/explanations/guides/ja/_fields.json +++ b/webui/explanations/guides/ja/_fields.json @@ -56,7 +56,7 @@ "torch_compile": "フォワードパスに torch.compile を有効化します。初期コンパイル後は学習が高速化されます。各 DiT ブロックをネイティブ形状でコンパイルします — トークン数バケットファミリーごとに 1 グラフ、静的パディングなし。", "compile_dynamic_seq": "コンパイル済みDiTブロック内でネイティブトークン列軸だけを動的にします。free-fit形状で、トークン数ごとの再コンパイルを避けるために必要です。", "v100_flash_stability": "V100 FlashAttention診断モード。offは完全なFlash、hybridはクロスアテンションをTorch SDPAへ送り、safeはFlashのまま有限値検査を追加します。ローカル検証ゲートをすべて通過した場合のみoffの本番プリセットを生成し、safeは診断専用です。", - "lora_fp32_compute": "LoRA 系列の FP32 保護モード。auto は V100/sm_70 + FP16 学習時のみ自動的に有効化し、true は FP32 アダプター計算を強制、false は明示的に無効化します。有効時はアダプターパラメーター、オプティマイザー入力、保存チェックポイントを FP32 に保ち、凍結 DiT 重みはモデル精度のままです。通常は auto を使用してください。", + "lora_fp32_compute": "LoRA 系列の FP32 保護モード。auto は V100/sm_70 + FP16 学習時のみ自動的に有効化し、true は FP32 アダプター計算を強制、false は通常無効化しますが、V100 の reliable block-swap モードでは安全性契約を維持するため false を上書きします。有効時はアダプターパラメーター、オプティマイザー入力、保存チェックポイントを FP32 に保ち、凍結 DiT 重みはモデル精度のままです。通常は auto を使用してください。", "debug_finite_checks": "アテンション入出力、ブロック残差、loss、勾配にNaN/Infが現れた時点で失敗します。無効値を修復または隠蔽しません。", "cache_llm_adapter_outputs": "LLM アダプターレイヤーの出力をディスクにキャッシュします。エポックごとにテキストエンコーダープロジェクションを再計算するのを避けます。", "masked_loss": "マスクされていない領域 (例: テキストバブルを除く) にのみロスを適用します。post_image_dataset/masks/ (またはレガシー masks/merged/) にマスクファイルが必要です — `make mask` で生成してください。", diff --git a/webui/explanations/guides/ko/_fields.json b/webui/explanations/guides/ko/_fields.json index 5d8228dd6..3a89bae18 100644 --- a/webui/explanations/guides/ko/_fields.json +++ b/webui/explanations/guides/ko/_fields.json @@ -56,7 +56,7 @@ "torch_compile": "포워드 패스에 torch.compile 활성화. 초기 컴파일 후 학습 속도 향상. 각 DiT 블록을 네이티브 형상으로 컴파일 — 토큰 수 버킷 패밀리당 하나의 그래프, 정적 패딩 없음.", "compile_dynamic_seq": "컴파일된 DiT 블록에서 네이티브 토큰 시퀀스 축만 동적으로 표시합니다. free-fit 형상에서 토큰 수마다 다시 컴파일하지 않고 티어당 하나의 그래프를 재사용하려면 필요합니다.", "v100_flash_stability": "V100 FlashAttention 진단 모드. off는 전체 Flash, hybrid는 크로스 어텐션을 Torch SDPA로 보내고, safe는 Flash를 유지하며 유한성 검사를 추가합니다. 모든 로컬 검증 게이트를 통과한 경우에만 off 운영 프리셋을 생성하며 safe는 진단 전용입니다.", - "lora_fp32_compute": "LoRA 계열 FP32 보호 모드입니다. auto는 V100/sm_70 + FP16 학습에서만 자동 활성화하고, true는 FP32 어댑터 계산을 강제하며, false는 명시적으로 비활성화합니다. 활성화되면 어댑터 파라미터, 옵티마이저 입력, 저장 체크포인트 텐서는 FP32를 유지하고 동결된 DiT 가중치는 모델 정밀도를 유지합니다. 일반적으로 auto를 사용하세요.", + "lora_fp32_compute": "LoRA 계열 FP32 보호 모드입니다. auto는 V100/sm_70 + FP16 학습에서만 자동 활성화하고, true는 FP32 어댑터 계산을 강제하며, false는 일반적으로 비활성화하지만 V100 reliable block-swap 모드에서는 안전 계약을 유지하기 위해 false를 덮어씁니다. 활성화되면 어댑터 파라미터, 옵티마이저 입력, 저장 체크포인트 텐서는 FP32를 유지하고 동결된 DiT 가중치는 모델 정밀도를 유지합니다. 일반적으로 auto를 사용하세요.", "debug_finite_checks": "어텐션 입출력, 블록 잔차, loss 또는 그래디언트에 NaN/Inf가 나타나면 즉시 실패합니다. 잘못된 값을 수정하거나 숨기지 않습니다.", "cache_llm_adapter_outputs": "LLM 어댑터 레이어 출력을 디스크에 캐싱. 매 에폭 텍스트 인코더 투영 재계산 회피.", "masked_loss": "마스크되지 않은 영역에만 손실 적용 (예: 말풍선 제외). post_image_dataset/masks/ (또는 레거시 masks/merged/) 에 마스크 파일 필요 — `make mask` 로 생성.", diff --git a/webui/services/config_service.py b/webui/services/config_service.py index f2bb00c41..b07faa7f8 100644 --- a/webui/services/config_service.py +++ b/webui/services/config_service.py @@ -361,16 +361,44 @@ def _strip_neutral_resume_defaults(data: dict) -> dict: } -def _normalize_lora_fp32_compute(data: dict) -> dict: - """Convert the WebUI tri-state sentinel to the trainer's optional bool.""" +def _canonicalize_lora_fp32_compute_layer(data: dict) -> dict: + """Promote the legacy network_args spelling into the normal merge layer.""" out = dict(data) + network_args = out.get("network_args") + if not isinstance(network_args, list): + return out + + cleaned: list = [] + legacy_value = None + for item in network_args: + if isinstance(item, str) and "=" in item: + key, value = item.split("=", 1) + if key.strip() == "lora_fp32_compute": + legacy_value = value.strip().lower() + continue + cleaned.append(item) + if legacy_value is not None: + out["lora_fp32_compute"] = legacy_value + if cleaned: + out["network_args"] = cleaned + else: + out.pop("network_args", None) + return out + + +def _normalize_lora_fp32_compute(data: dict, *, keep_auto: bool = False) -> dict: + """Convert the WebUI tri-state sentinel to the trainer's optional bool.""" + out = _canonicalize_lora_fp32_compute_layer(data) if "lora_fp32_compute" not in out: return out normalized = str(out["lora_fp32_compute"]).strip().lower() if normalized not in _LORA_FP32_COMPUTE_CHOICES: raise ValueError("lora_fp32_compute must be auto, true, or false") if normalized == "auto": - out.pop("lora_fp32_compute", None) + if keep_auto: + out["lora_fp32_compute"] = "auto" + else: + out.pop("lora_fp32_compute", None) else: out["lora_fp32_compute"] = normalized == "true" return out @@ -679,14 +707,22 @@ def merged_gui_variant_preset(variant: str, preset: str) -> tuple[dict, dict[str _safe_variant(variant) from library.config.model_paths import load_model_config - base = _load(CONFIGS_DIR / "base.toml") - model = load_model_config(CONFIGS_DIR) - pset = _load_all_presets().get(preset, {}) + base = _canonicalize_lora_fp32_compute_layer(_load(CONFIGS_DIR / "base.toml")) + model = _canonicalize_lora_fp32_compute_layer(load_model_config(CONFIGS_DIR)) + pset = _canonicalize_lora_fp32_compute_layer( + _load_all_presets().get(preset, {}) + ) if variant.startswith("custom/"): - meth = _load(_resolve_variant_path(variant)) + meth = _canonicalize_lora_fp32_compute_layer( + _load(_resolve_variant_path(variant)) + ) else: - builtin = _load(GUI_METHODS_DIR / f"{variant}.toml") - overlay = _load(CUSTOM_VARIANTS_DIR / f"{variant}.toml") + builtin = _canonicalize_lora_fp32_compute_layer( + _load(GUI_METHODS_DIR / f"{variant}.toml") + ) + overlay = _canonicalize_lora_fp32_compute_layer( + _load(CUSTOM_VARIANTS_DIR / f"{variant}.toml") + ) meth = {**builtin, **overlay} merged: dict = {} origin: dict[str, str] = {} @@ -711,13 +747,15 @@ def _merge_layer(layer: dict, src: str) -> None: # The V100 FP16 protection is intentionally tri-state. An omitted # lora_fp32_compute lets train.py auto-enable it only for V100/sm_70 FP16; # explicit true/false force the user's choice. Surface the omitted state as - # "auto" in the WebUI without writing that sentinel into the training TOML. - if ( - merged.get("network_module") == "networks.lora_anima" - and "lora_fp32_compute" not in merged - ): - merged["lora_fp32_compute"] = "auto" - origin["lora_fp32_compute"] = "base" + # "auto" in the WebUI; the training merge consumes that sentinel before + # network kwargs are built. + if merged.get("network_module") == "networks.lora_anima": + if "lora_fp32_compute" not in merged: + merged["lora_fp32_compute"] = "auto" + origin["lora_fp32_compute"] = "base" + else: + merged.pop("lora_fp32_compute", None) + origin.pop("lora_fp32_compute", None) # Inject virtual keys from [[datasets]] variant_override = _validation_enabled_from_datasets(meth.get("datasets")) @@ -1238,7 +1276,7 @@ def save_variant_config(variant: str, data: dict) -> None: else: # Built-in variant: always save to custom overlay, never to template path = CUSTOM_VARIANTS_DIR / f"{variant}.toml" - current = _load(path) + current = _canonicalize_lora_fp32_compute_layer(_load(path)) # Handle extra_args: parse as TOML and merge as overrides extra_text = data.pop("extra_args", None) @@ -1311,7 +1349,11 @@ def save_variant_config(variant: str, data: dict) -> None: if extras: current.update(extras) - current = _normalize_lora_fp32_compute(current) + # Keep an explicit auto sentinel at the method layer so it can clear a + # custom preset's true/false. The training config loader consumes this + # sentinel after all layers have merged and does not forward it as a + # network kwarg. + current = _normalize_lora_fp32_compute(current, keep_auto=True) # Validate the prospective effective method config, not only the sparse # overlay. Built-in LoKR flags live in gui-methods/lokr.toml while a user