Skip to content
This repository was archived by the owner on Oct 25, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 35 additions & 2 deletions library/config/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
# on a typo — a path-string typo fails loudly (file not found), but a
# comparison typo would route a gui variant through the flat-methods path.
GUI_METHODS_SUBDIR = "gui-methods"
_LORA_FP32_COMPUTE_KEY = "lora_fp32_compute"
_SNAPSHOT_SUFFIX = ".snapshot.toml"
_EMPTY_STRING_IS_UNSET = frozenset({"network_weights"})
_DUMP_SKIP_KEYS = {
Expand Down Expand Up @@ -241,6 +242,31 @@ def _resolve_method_path(
return os.path.join(configs_dir, methods_subdir, f"{method}.toml")


def _canonicalize_lora_fp32_compute_layer(data: dict) -> dict:
"""Move legacy ``network_args`` values into normal layer precedence."""
out = dict(data)
network_args = out.get("network_args")
if not isinstance(network_args, list):
return out

cleaned: list = []
legacy_value = None
for item in network_args:
if isinstance(item, str) and "=" in item:
key, value = item.split("=", 1)
if key.strip() == _LORA_FP32_COMPUTE_KEY:
legacy_value = value.strip().lower()
continue
cleaned.append(item)
if legacy_value is not None:
out[_LORA_FP32_COMPUTE_KEY] = legacy_value
if cleaned:
out["network_args"] = cleaned
else:
out.pop("network_args", None)
return out


def _load_gui_method_merged(configs_dir: str, method: str) -> tuple[dict, str]:
"""Load a gui-methods variant as ``{**builtin, **overlay}``.

Expand All @@ -261,10 +287,12 @@ def _load_gui_method_merged(configs_dir: str, method: str) -> tuple[dict, str]:
merged: dict = {}
if os.path.exists(builtin_path):
with open(builtin_path, "r", encoding="utf-8") as f:
merged.update(toml.load(f))
merged.update(_canonicalize_lora_fp32_compute_layer(toml.load(f)))
if os.path.exists(overlay_path):
with open(overlay_path, "r", encoding="utf-8") as f:
merged.update(toml.load(f)) # overlay wins on conflict
merged.update(
_canonicalize_lora_fp32_compute_layer(toml.load(f))
) # overlay wins on conflict
# source_path: absolute path for _flatten_toml's schema line lookup
# (prefers the overlay so line numbers point at the user-owned file).
# source_tag: display path for provenance / print-config output.
Expand Down Expand Up @@ -796,6 +824,7 @@ def load_method_preset(
for kind, path, tag, raw in _iter_method_preset_layers(
preset, configs_dir, methods_subdir, method, require_files=True
):
raw = _canonicalize_lora_fp32_compute_layer(raw)
# Preset sections are flat scalar tables, so wrap them as
# ``{preset: section}`` to mirror base/method top-level section tables —
# ``_flatten_toml`` then descends one level into the section contents.
Expand All @@ -806,6 +835,10 @@ def load_method_preset(
merged[k] = v
provenance[k] = tag

if str(merged.get(_LORA_FP32_COMPUTE_KEY, "")).strip().lower() == "auto":
merged.pop(_LORA_FP32_COMPUTE_KEY, None)
provenance.pop(_LORA_FP32_COMPUTE_KEY, None)

_drop_unset_config_values(merged, provenance)

if return_provenance:
Expand Down
173 changes: 173 additions & 0 deletions tests/test_config_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,179 @@ def test_sample_decode_inline_validation_rejects_unknown_mode():
]


def test_lora_fp32_compute_is_editable_tri_state_and_round_trips(
monkeypatch, tmp_path
):
configs = tmp_path / "configs"
methods = configs / "gui-methods"
overlays = configs / "custom" / "variants"
methods.mkdir(parents=True)
overlays.mkdir(parents=True)
(configs / "base.toml").write_text(
'network_module = "networks.lora_anima"\n', encoding="utf-8"
)
(configs / "model.toml").write_text("", encoding="utf-8")
(configs / "presets.toml").write_text("[default]\n", encoding="utf-8")
(methods / "lora.toml").write_text("network_dim = 16\n", encoding="utf-8")

monkeypatch.setattr(config_service, "CONFIGS_DIR", configs)
monkeypatch.setattr(config_service, "GUI_METHODS_DIR", methods)
monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml")
monkeypatch.setattr(config_service, "CUSTOM_VARIANTS_DIR", overlays)

result = config_service.build_merged_config("lora", "default", lang="en")
field = next(f for f in result["fields"] if f["key"] == "lora_fp32_compute")
assert field["value"] == "auto"
assert field["origin"] == "base"
assert field["field_type"] == "select"
assert field["group"] == "Performance"
assert field["read_only"] is False
assert field["options"] == ["auto", "true", "false"]
assert field["description"]

config_service.save_variant_config("lora", {"lora_fp32_compute": "true"})
saved = toml.loads((overlays / "lora.toml").read_text(encoding="utf-8"))
assert saved["lora_fp32_compute"] is True

merged, origin = config_service.merged_gui_variant_preset("lora", "default")
assert merged["lora_fp32_compute"] is True
assert origin["lora_fp32_compute"] == "method"

config_service.save_variant_config("lora", {"lora_fp32_compute": "false"})
saved = toml.loads((overlays / "lora.toml").read_text(encoding="utf-8"))
assert saved["lora_fp32_compute"] is False

config_service.save_variant_config("lora", {"lora_fp32_compute": "auto"})
saved = toml.loads((overlays / "lora.toml").read_text(encoding="utf-8"))
assert saved["lora_fp32_compute"] == "auto"

training = load_method_preset(
"lora", "default", configs_dir=str(configs), methods_subdir="gui-methods"
)
assert "lora_fp32_compute" not in training


def test_lora_fp32_auto_clears_preset_and_legacy_network_args(monkeypatch, tmp_path):
configs = tmp_path / "configs"
methods = configs / "gui-methods"
overlays = configs / "custom" / "variants"
custom_presets = configs / "custom" / "presets"
methods.mkdir(parents=True)
overlays.mkdir(parents=True)
custom_presets.mkdir(parents=True)
(configs / "base.toml").write_text(
'network_module = "networks.lora_anima"\n', encoding="utf-8"
)
(configs / "model.toml").write_text("", encoding="utf-8")
(configs / "presets.toml").write_text("[default]\n", encoding="utf-8")
(custom_presets / "forced.toml").write_text(
'network_args = ["lora_fp32_compute=true"]\n', encoding="utf-8"
)
(methods / "lora.toml").write_text(
'network_args = ["use_timestep_mask=true", "lora_fp32_compute=false"]\n',
encoding="utf-8",
)

monkeypatch.setattr(config_service, "CONFIGS_DIR", configs)
monkeypatch.setattr(config_service, "GUI_METHODS_DIR", methods)
monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml")
monkeypatch.setattr(config_service, "CUSTOM_PRESETS_DIR", custom_presets)
monkeypatch.setattr(config_service, "CUSTOM_VARIANTS_DIR", overlays)

result = config_service.build_merged_config("lora", "forced", lang="en")
field = next(f for f in result["fields"] if f["key"] == "lora_fp32_compute")
assert field["value"] == "false"
assert field["origin"] == "method"

config_service.save_variant_config("lora", {"lora_fp32_compute": "true"})
training = load_method_preset(
"lora", "forced", configs_dir=str(configs), methods_subdir="gui-methods"
)
assert str(training["lora_fp32_compute"]).lower() == "true"
assert training["network_args"] == ["use_timestep_mask=true"]

config_service.save_variant_config("lora", {"lora_fp32_compute": "auto"})
merged, origin = config_service.merged_gui_variant_preset("lora", "forced")
assert merged["lora_fp32_compute"] == "auto"
assert origin["lora_fp32_compute"] == "method"

training = load_method_preset(
"lora", "forced", configs_dir=str(configs), methods_subdir="gui-methods"
)
assert "lora_fp32_compute" not in training
assert training["network_args"] == ["use_timestep_mask=true"]


def test_lora_fp32_compute_not_shown_for_non_lora_network(monkeypatch, tmp_path):
configs = tmp_path / "configs"
methods = configs / "gui-methods"
custom_presets = configs / "custom" / "presets"
methods.mkdir(parents=True)
custom_presets.mkdir(parents=True)
(configs / "base.toml").write_text(
'network_module = "networks.lora_anima"\n', encoding="utf-8"
)
(configs / "model.toml").write_text("", encoding="utf-8")
(configs / "presets.toml").write_text("[default]\n", encoding="utf-8")
(custom_presets / "forced.toml").write_text(
"lora_fp32_compute = true\n", encoding="utf-8"
)
(methods / "easycontrol.toml").write_text(
'network_module = "networks.methods.easycontrol"\n', encoding="utf-8"
)

monkeypatch.setattr(config_service, "CONFIGS_DIR", configs)
monkeypatch.setattr(config_service, "GUI_METHODS_DIR", methods)
monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml")
monkeypatch.setattr(config_service, "CUSTOM_PRESETS_DIR", custom_presets)
monkeypatch.setattr(
config_service, "CUSTOM_VARIANTS_DIR", configs / "custom" / "variants"
)

result = config_service.build_merged_config(
"easycontrol", "forced", lang="en"
)
assert "lora_fp32_compute" not in {field["key"] for field in result["fields"]}


def test_lora_fp32_compute_validation_rejects_unknown_mode():
assert validate_config({"lora_fp32_compute": "sometimes"}) == [
"lora_fp32_compute must be auto, true, or false"
]


def test_lora_fp32_compute_help_documents_reliable_mode_override():
description = config_service._field_desc("lora_fp32_compute", "en")
assert description is not None
assert "reliable block-swap mode overrides false" in description


def test_create_custom_preset_normalizes_lora_fp32_compute(monkeypatch, tmp_path):
configs = tmp_path / "configs"
custom_presets = configs / "custom" / "presets"
custom_presets.mkdir(parents=True)
(configs / "presets.toml").write_text("[default]\n", encoding="utf-8")

monkeypatch.setattr(config_service, "PRESETS_FILE", configs / "presets.toml")
monkeypatch.setattr(config_service, "CUSTOM_PRESETS_DIR", custom_presets)

config_service.create_custom_preset(
"auto-protection", {"lora_fp32_compute": "auto"}
)
auto_saved = toml.loads(
(custom_presets / "auto-protection.toml").read_text(encoding="utf-8")
)
assert "lora_fp32_compute" not in auto_saved

config_service.create_custom_preset(
"forced-protection", {"lora_fp32_compute": "true"}
)
forced_saved = toml.loads(
(custom_presets / "forced-protection.toml").read_text(encoding="utf-8")
)
assert forced_saved["lora_fp32_compute"] is True


def test_create_custom_preset_strips_neutral_resume_defaults(monkeypatch, tmp_path):
"""A fresh WebUI preset must not turn an empty warm-start into a path."""
configs = tmp_path / "configs"
Expand Down
1 change: 1 addition & 0 deletions webui/explanations/guides/cn/_fields.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
"torch_compile": "为前向传播启用torch.compile。初始编译后训练速度加快。以原生形状编译每个DiT块 — 每个令牌数桶系列一张计算图,无静态填充。",
"compile_dynamic_seq": "在每个已编译DiT块内部仅将原生令牌序列轴标记为动态。free-fit形状必须启用,使同一分辨率层级复用一张图,而不是每个令牌数各编译一张。",
"v100_flash_stability": "V100 FlashAttention诊断模式。off使用完整Flash;hybrid将交叉注意力路由到Torch SDPA;safe保留Flash并检查注意力边界有限性。只有本机全部验证门禁通过后才生成使用off的生产预设;safe仅用于诊断。",
"lora_fp32_compute": "LoRA 系列的 FP32 保护模式。auto:仅在 V100/sm_70 + FP16 训练时自动开启;true:强制使用 FP32 适配器计算;false:通常关闭,但 V100 可靠块交换模式会为维持安全约束而覆盖 false。保护生效时,适配器参数、优化器输入和保存的检查点张量会保持 FP32,冻结的 DiT 权重仍使用模型精度。通常保持 auto。",
"debug_finite_checks": "当注意力输入/输出、块残差、loss或梯度出现NaN/Inf时立即失败。它只用于诊断,不修复或隐藏无效值。",
"cache_llm_adapter_outputs": "将LLM适配器层输出缓存到磁盘。避免每轮重新计算文本编码器投影。",
"masked_loss": "仅在未被遮罩的区域(如排除文字气泡)应用损失。需要post_image_dataset/masks/(或旧版masks/merged/)中的遮罩文件 — 运行`make mask`生成。",
Expand Down
1 change: 1 addition & 0 deletions webui/explanations/guides/en/_fields.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
"torch_compile": "Enable torch.compile for the forward pass. Faster training after initial compilation. Compiles each DiT block at native shapes — one graph per token-count bucket family, no static padding.",
"compile_dynamic_seq": "Mark only the native token-sequence axis dynamic inside each compiled DiT block. Required with free-fit shapes so a tier reuses one graph instead of compiling one graph per token count.",
"v100_flash_stability": "V100 FlashAttention diagnostic mode. off uses full Flash; hybrid routes cross-attention to Torch SDPA; safe keeps Flash and enables attention-boundary finite checks. A generated production preset uses off only after every local validation gate passes; safe is diagnostic only.",
"lora_fp32_compute": "FP32 protection for the LoRA family. auto enables it only for V100/sm_70 FP16 training; true forces FP32 adapter compute; false disables it except that V100 reliable block-swap mode overrides false to preserve its safety contract. When active, adapter parameters, optimizer inputs, and saved checkpoint tensors stay FP32 while frozen DiT weights keep the model dtype. Usually leave this on auto.",
"debug_finite_checks": "Fail immediately if attention inputs/outputs, block residuals, loss, or gradients contain NaN or Inf. Diagnostic guard only; it never repairs or hides invalid values.",
"cache_llm_adapter_outputs": "Cache the LLM adapter layer outputs to disk. Avoids recomputing text encoder projections each epoch.",
"masked_loss": "Apply loss only to non-masked regions (e.g., exclude text bubbles). Requires mask files in post_image_dataset/masks/ (or legacy masks/merged/) — run `make mask` to generate them.",
Expand Down
1 change: 1 addition & 0 deletions webui/explanations/guides/ja/_fields.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
"torch_compile": "フォワードパスに torch.compile を有効化します。初期コンパイル後は学習が高速化されます。各 DiT ブロックをネイティブ形状でコンパイルします — トークン数バケットファミリーごとに 1 グラフ、静的パディングなし。",
"compile_dynamic_seq": "コンパイル済みDiTブロック内でネイティブトークン列軸だけを動的にします。free-fit形状で、トークン数ごとの再コンパイルを避けるために必要です。",
"v100_flash_stability": "V100 FlashAttention診断モード。offは完全なFlash、hybridはクロスアテンションをTorch SDPAへ送り、safeはFlashのまま有限値検査を追加します。ローカル検証ゲートをすべて通過した場合のみoffの本番プリセットを生成し、safeは診断専用です。",
"lora_fp32_compute": "LoRA 系列の FP32 保護モード。auto は V100/sm_70 + FP16 学習時のみ自動的に有効化し、true は FP32 アダプター計算を強制、false は通常無効化しますが、V100 の reliable block-swap モードでは安全性契約を維持するため false を上書きします。有効時はアダプターパラメーター、オプティマイザー入力、保存チェックポイントを FP32 に保ち、凍結 DiT 重みはモデル精度のままです。通常は auto を使用してください。",
"debug_finite_checks": "アテンション入出力、ブロック残差、loss、勾配にNaN/Infが現れた時点で失敗します。無効値を修復または隠蔽しません。",
"cache_llm_adapter_outputs": "LLM アダプターレイヤーの出力をディスクにキャッシュします。エポックごとにテキストエンコーダープロジェクションを再計算するのを避けます。",
"masked_loss": "マスクされていない領域 (例: テキストバブルを除く) にのみロスを適用します。post_image_dataset/masks/ (またはレガシー masks/merged/) にマスクファイルが必要です — `make mask` で生成してください。",
Expand Down
1 change: 1 addition & 0 deletions webui/explanations/guides/ko/_fields.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
"torch_compile": "포워드 패스에 torch.compile 활성화. 초기 컴파일 후 학습 속도 향상. 각 DiT 블록을 네이티브 형상으로 컴파일 — 토큰 수 버킷 패밀리당 하나의 그래프, 정적 패딩 없음.",
"compile_dynamic_seq": "컴파일된 DiT 블록에서 네이티브 토큰 시퀀스 축만 동적으로 표시합니다. free-fit 형상에서 토큰 수마다 다시 컴파일하지 않고 티어당 하나의 그래프를 재사용하려면 필요합니다.",
"v100_flash_stability": "V100 FlashAttention 진단 모드. off는 전체 Flash, hybrid는 크로스 어텐션을 Torch SDPA로 보내고, safe는 Flash를 유지하며 유한성 검사를 추가합니다. 모든 로컬 검증 게이트를 통과한 경우에만 off 운영 프리셋을 생성하며 safe는 진단 전용입니다.",
"lora_fp32_compute": "LoRA 계열 FP32 보호 모드입니다. auto는 V100/sm_70 + FP16 학습에서만 자동 활성화하고, true는 FP32 어댑터 계산을 강제하며, false는 일반적으로 비활성화하지만 V100 reliable block-swap 모드에서는 안전 계약을 유지하기 위해 false를 덮어씁니다. 활성화되면 어댑터 파라미터, 옵티마이저 입력, 저장 체크포인트 텐서는 FP32를 유지하고 동결된 DiT 가중치는 모델 정밀도를 유지합니다. 일반적으로 auto를 사용하세요.",
"debug_finite_checks": "어텐션 입출력, 블록 잔차, loss 또는 그래디언트에 NaN/Inf가 나타나면 즉시 실패합니다. 잘못된 값을 수정하거나 숨기지 않습니다.",
"cache_llm_adapter_outputs": "LLM 어댑터 레이어 출력을 디스크에 캐싱. 매 에폭 텍스트 인코더 투영 재계산 회피.",
"masked_loss": "마스크되지 않은 영역에만 손실 적용 (예: 말풍선 제외). post_image_dataset/masks/ (또는 레거시 masks/merged/) 에 마스크 파일 필요 — `make mask` 로 생성.",
Expand Down
Loading