diff --git a/.env.example b/.env.example index 581ba94..abad4b6 100644 --- a/.env.example +++ b/.env.example @@ -43,6 +43,8 @@ HF_TOKEN= # LDS_QWEN21_MODEL=qwen_image_2.1_int8_convrot.safetensors # LDS_QWEN21_TEXT_ENCODER=qwen3vl_8b_int8_convrot.safetensors # LDS_QWEN21_VAE=qwen_image_2.1_vae_bf16.safetensors +# LDS_QWEN21_MAX_REFS=1 # references per local shot (1-16) +# LDS_QWEN21_RESOLUTION=1536 # reference/output size, about N x N pixels # LDS_UPSCALE_MODEL=4xNomosWebPhoto_RealPLKSR.safetensors # LDS_DEJPG_MODEL=1xDeJPG_OmniSR.pth # LDS_SAM3_CHECKPOINT=sam3.1_multiplex_fp16.safetensors diff --git a/AGENTS.md b/AGENTS.md index bdea4cc..5cb8270 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,10 +8,10 @@ Turn a character, style, or concept into a ready-to-train LoRA dataset. One refe _Last verified: 2026-10-08_ -- **Status:** in active development; main is v0.18.1 (git tag `v0.18.1`). **Every version bump must ship a GitHub Release** or the in-app update check never fires. The project was renamed twice (lora-dataset-studio → lora-distillery → dataset-deviser); older references under the previous names are stale. -- **Works:** all five stages end to end (preprocess → generate & curate → caption → export → train config); the three dataset types (character, style, concept) with their own shot plans and caption framing; local and cloud paths for every stage; gallery curation with selection carried forward between stages, including shift-click range select; advisory dedupe, quality flags and caption lint; trainer configs for ai-toolkit (incl. SDXL), kohya sd-scripts, musubi-tuner and Fizgig; opt-in private Hugging Face publish. Since 0.15.0: per-image fault isolation in ① (one bad source never ends the batch), a cooperative ⏹ Stop across ①/②/③ with documented resume paths, translated ComfyUI failures (missing node, bad input, unreachable server) and an in-app 🩺 setup check. Since 0.15.1: a ② **shot style** (default = keep the reference's own medium; eight presets + custom text) threaded into both prompt builders, the CLI, ④ metadata and ⑤'s sample prompt, a final-prompt preview, and a prev/next/save-and-next caption editor with the image on screen. Since 0.16.0: **one name box and one trigger box in the header** that ②/③/④/⑤ all read (they used to be per-tab and went stale), ④ stating the pair it stamped, an opt-in ④ **hand-off to [Idiot LoRa Builder](https://github.com/Fablestarexpanse/Idiot-Lora-Builder)** (writes its `.lora-studio/ratings.json` so its grid opens pre-triaged — nothing is launched), a ⑤ advisory when the dataset can't fill the training resolution, every bundled ComfyUI model filename overridable from `.env`, and a `cli build` preflight for the local engine. Since 0.17.0: the local ② engine is **Qwen-Image 2.1** (plain-English prompts shared with Gemini, ~26 s/shot, `LDS_QWEN21_*` filenames) instead of Qwen-Image-Edit 2511 + the Multiple-Angles LoRA, whose graph stays in `studio/comfy_workflows/legacy/`; a dead ComfyUI mid-batch now fails one shot, not the run. Since 0.17.1: the UI port is chosen by `_pick_port` in `app.py` — 7861 if bindable, else an OS-assigned free port (Windows reserves a ~500-port block around 7861), or `LDS_PORT` to pin one. Since 0.17.2: `comfy_api.upload_image` downscales a source over 90 MB before upload (ComfyUI answers 413 above its 100 MB `--max-upload-size`). Since 0.17.3: the cloud image model defaults to **`auto`** (Google shut down the `-preview` ids on 2026-06-25; `resolve_image_model` picks from the cached live list, Nano Banana 2.1 first), Groq captions use `qwen/qwen3.8-27b` (key `groq-qwen`, old key aliased), every API image goes through `config.api_image` (true mime, size cap), musubi commands include the cache steps and the per-architecture network module, ai-toolkit gains Qwen-Image 2.1 and a correct Krea 2 (Raw) preset and keeps every checkpoint, ① applies EXIF rotation, no longer 4x-upscales large JPEGs, and can jump a busy ComfyUI queue like ② (`--front` on `preprocess`/`build`). Since 0.18.0: ⑤ counts **epochs** (one checkpoint and one sample set each) with auto repeats sized to a step target and a live exposure line; every config ships a `validation/` pack (8 fixed prompts at seed 42, `validation.md`, a per-epoch score sheet); **Fizgig** is a ⑤ target (Krea 2 / Qwen Image 2.1 / Klein); ① stamps a `dd_source` provenance PNG chunk so ⑤ names upscaled sources and ④ knows real photos; ④ can hold out N real photos in a `-heldout` sibling as a likeness test. Since 0.18.1: a ⑤ optimizer dropdown, AdamW8bit by default, Prodigy (lr 1.0, its diffusion args) for ai-toolkit/musubi/kohya; Fizgig stays AdamW8bit. -- **In progress:** a phased LoRA best-practice update. 0.19.0 is next: training copies keep their background, identity policy (outfits vary by default), framing tiers, a coverage panel, a full-body warning, near-duplicate/burst flags, an anchor shot, local multi-reference. Then 0.20.0 (a Quick build tab). Nothing from those is half-built on main. The attention-only toggle planned for 0.18.0 was deferred (ARCHITECTURE.md → Future ideas). -- **Known gaps / next steps:** Auto's model order (`IMAGE_MODEL_PREFERENCE`, Nano Banana 2.1 then Pro) rests on one A/B subject — see ARCHITECTURE.md; local ② still sends only the first reference. Otherwise pick up from `docs/ARCHITECTURE.md` → "Future ideas" / "Deferred", and read its *Maintainer principles* before changing anything; `gradio` is pinned `<6` because of a real stuck-loading-overlay regression — unpinning needs that verified fixed upstream; training is never launched for you and is not planned to be; the default export `output_root` writes into the repo root, so a smoke-test export leaves an untracked dataset folder to delete before committing. +- **Status:** in active development; main is v0.19.0 (git tag `v0.19.0`). **Every version bump must ship a GitHub Release** or the in-app update check never fires. The project was renamed twice (lora-dataset-studio → lora-distillery → dataset-deviser); older references under the previous names are stale. +- **Works:** all five stages end to end (preprocess → generate & curate → caption → export → train config); the three dataset types (character, style, concept) with their own shot plans and caption framing; local and cloud paths for every stage; gallery curation with selection carried forward between stages, including shift-click range select; advisory dedupe, quality flags and caption lint; trainer configs for ai-toolkit (incl. SDXL), kohya sd-scripts, musubi-tuner and Fizgig; opt-in private Hugging Face publish. Since 0.15.0: per-image fault isolation in ① (one bad source never ends the batch), a cooperative ⏹ Stop across ①/②/③ with documented resume paths, translated ComfyUI failures (missing node, bad input, unreachable server) and an in-app 🩺 setup check. Since 0.15.1: a ② **shot style** (default = keep the reference's own medium; eight presets + custom text) threaded into both prompt builders, the CLI, ④ metadata and ⑤'s sample prompt, a final-prompt preview, and a prev/next/save-and-next caption editor with the image on screen. Since 0.16.0: **one name box and one trigger box in the header** that ②/③/④/⑤ all read (they used to be per-tab and went stale), ④ stating the pair it stamped, an opt-in ④ **hand-off to [Idiot LoRa Builder](https://github.com/Fablestarexpanse/Idiot-Lora-Builder)** (writes its `.lora-studio/ratings.json` so its grid opens pre-triaged — nothing is launched), a ⑤ advisory when the dataset can't fill the training resolution, every bundled ComfyUI model filename overridable from `.env`, and a `cli build` preflight for the local engine. Since 0.17.0: the local ② engine is **Qwen-Image 2.1** (plain-English prompts shared with Gemini, ~26 s/shot, `LDS_QWEN21_*` filenames) instead of Qwen-Image-Edit 2511 + the Multiple-Angles LoRA, whose graph stays in `studio/comfy_workflows/legacy/`; a dead ComfyUI mid-batch now fails one shot, not the run. Since 0.17.1: the UI port is chosen by `_pick_port` in `app.py` — 7861 if bindable, else an OS-assigned free port (Windows reserves a ~500-port block around 7861), or `LDS_PORT` to pin one. Since 0.17.2: `comfy_api.upload_image` downscales a source over 90 MB before upload (ComfyUI answers 413 above its 100 MB `--max-upload-size`). Since 0.17.3: the cloud image model defaults to **`auto`** (Google shut down the `-preview` ids on 2026-06-25; `resolve_image_model` picks from the cached live list, Nano Banana 2.1 first), Groq captions use `qwen/qwen3.8-27b` (key `groq-qwen`, old key aliased), every API image goes through `config.api_image` (true mime, size cap), musubi commands include the cache steps and the per-architecture network module, ai-toolkit gains Qwen-Image 2.1 and a correct Krea 2 (Raw) preset and keeps every checkpoint, ① applies EXIF rotation, no longer 4x-upscales large JPEGs, and can jump a busy ComfyUI queue like ② (`--front` on `preprocess`/`build`). Since 0.18.0: ⑤ counts **epochs** (one checkpoint and one sample set each) with auto repeats sized to a step target and a live exposure line; every config ships a `validation/` pack (8 fixed prompts at seed 42, `validation.md`, a per-epoch score sheet); **Fizgig** is a ⑤ target (Krea 2 / Qwen Image 2.1 / Klein); ① stamps a `dd_source` provenance PNG chunk so ⑤ names upscaled sources and ④ knows real photos; ④ can hold out N real photos in a `-heldout` sibling as a likeness test. Since 0.18.1: a ⑤ optimizer dropdown, AdamW8bit by default, Prodigy (lr 1.0, its diffusion args) for ai-toolkit/musubi/kohya; Fizgig stays AdamW8bit. Since 0.19.0: ① training copies **keep their background** and the isolated copy goes to `prepped/refs/` for ② only (angle re-isolation is its own ② option, off); a header **identity policy** for characters (Identity only varies outfits and captions clothing; Signature costume keeps the outfit out of captions); four framing tiers in the shot plan and a ② coverage line with a dominance warning; a cut-off-reference warning; ② near-duplicate finder and primary-reference preview, ④ burst grouping; local ② sends several references (`LDS_QWEN21_MAX_REFS`, default 1) at `LDS_QWEN21_RESOLUTION` (1536); Gemini gets a per-shot aspect and a reference cap; an opt-in **anchor shot**; SAM3 load errors name a stale `HF_TOKEN`. +- **In progress:** a phased LoRA best-practice update. 0.20.0 is next: a ⚡ Quick build tab and `cli build` on shared `pipeline.build_start`/`build_finish`. Nothing from it is half-built on main. The attention-only toggle planned for 0.18.0 was deferred (ARCHITECTURE.md → Future ideas). +- **Known gaps / next steps:** Auto's model order (`IMAGE_MODEL_PREFERENCE`, Nano Banana 2.1 then Pro) rests on one A/B subject — see ARCHITECTURE.md; the local defaults (one reference, 1536, anchor off) likewise rest on one A/B subject. Otherwise pick up from `docs/ARCHITECTURE.md` → "Future ideas" / "Deferred", and read its *Maintainer principles* before changing anything; `gradio` is pinned `<6` because of a real stuck-loading-overlay regression — unpinning needs that verified fixed upstream; training is never launched for you and is not planned to be; the default export `output_root` writes into the repo root, so a smoke-test export leaves an untracked dataset folder to delete before committing. - **Deep docs:** `docs/ARCHITECTURE.md` (module map, stage and backend detail, gotchas, backlog — the deep reference), `docs/comfyui-setup.md`. Worklogs under `docs/` are gitignored and local-only by design. ## Architecture in 60 seconds diff --git a/README.md b/README.md index a4848be..e823d59 100644 --- a/README.md +++ b/README.md @@ -222,13 +222,22 @@ CLI: `--shot-style anime`, or `--shot-style custom --shot-style-text "a woodcut ## Two things that ruin a character LoRA -Both have a switch in ②, because both bit us: +Both are handled by default, because both bit us: -- **Same clothes in every image** → the LoRA learns the outfit as the character. Hit - **🎲 Randomize outfits** to dress each angle/pose shot differently. +- **Same clothes in every image** → the LoRA learns the outfit as the character. The header's + **Identity policy** is *Identity only* by default: every angle/pose shot gets a different + outfit and the captions name the clothing. Pick *Signature costume* when the outfit **is** the + character (a superhero suit): outfits stay as drawn and captions leave them out. - **Props copied into every shot** (a backpack in the reference → the LoRA thinks the backpack - *is* the character). Best fix: **isolate the source in ①** — SAM3 keeps bags and held objects - out of the subject mask. The "Exclude props" toggle in ② helps too, especially on Gemini. + *is* the character). Best fix: **isolate the source in ①** and name anything held (a cup, a + plate) in **Objects to remove** — SAM3 counts a held object as part of the person. The + "Exclude props" toggle in ② helps too, on Gemini only. + +① keeps your photos' own backgrounds for training: the white cut-out it makes goes to +`prepped/refs/` and is only ②'s reference. ② shows the coverage of the shots you keep (framing, +turnaround, expressions, outfits) and can flag near-duplicates. If your only photos are cut off +at the waist, ① says so: full-body shots will invent the legs, so add a full-body photo if you +have one, or try ②'s **Anchor shot**, which builds every shot from one generated front view. ## Picking the best checkpoint @@ -244,9 +253,10 @@ Every stage is a standalone subcommand; `--help` shows all options. ```bash python cli.py preprocess ./sources --out ./prepped -python cli.py generate ./prepped --name "Sy Snootles" --engine comfyui --randomize-outfits -python cli.py generate ./prepped --name "Sy Snootles" --shot-style anime # keep an illustrated look -python cli.py generate ./prepped --name "brass compass" --dataset-type concept # 18-shot object set +python cli.py generate ./prepped/refs --name "Sy Snootles" --engine comfyui +python cli.py generate ./prepped/refs --name "Sy Snootles" --identity costume # outfit is the character +python cli.py generate ./prepped/refs --name "Sy Snootles" --shot-style anime # keep an illustrated look +python cli.py generate ./prepped/refs --name "brass compass" --dataset-type concept # 18-shot object set python cli.py caption ./folder --trigger sysnootles # prose sidecars python cli.py caption ./folder --trigger sysnootles --caption-style tags # Danbooru tags python cli.py caption ./folder --trigger mystyle --dataset-type style # style-framed @@ -283,9 +293,9 @@ Set or change any of them with `python cli.py keys` — they live only in a giti generation, built-in SAM3, local captioning and export all work without it. **No custom nodes required.** See [docs/comfyui-setup.md](docs/comfyui-setup.md). - **Sources are never modified**; every stage writes copies. -- **Alpha cutout, on request.** ① Preprocess can export the isolated subject on a transparent - background instead of white (builtin SAM3 backend only) — for your own compositing workflows. - Leave it off (default) if you're continuing to ② Generate, which expects a white reference. +- **Alpha cutout, on request.** ① Preprocess can make the `refs/` cut-out transparent instead + of white (builtin SAM3 backend only) — for your own compositing workflows. ② expects a white + reference, so ① then leaves ②'s source folder for you to fill. - The UI binds to `127.0.0.1` with **no authentication** — don't expose it. So galleries can preview images from any folder you point a tab at, the local file server can read any file the app process can while it's running; that's safe only behind the localhost bind, so never forward diff --git a/app.py b/app.py index 6276a5d..147f678 100644 --- a/app.py +++ b/app.py @@ -54,7 +54,13 @@ resolve_image_model, ) from studio.jobs import JobControl -from studio.shotplan import Shot, apply_prop_exclusion, apply_wardrobe, plan_for_type +from studio.shotplan import ( + Shot, + apply_prop_exclusion, + apply_wardrobe, + coverage, + plan_for_type, +) from studio.trainer_configs import TRAINER_MODELS, TRAINERS, optimizer_choices TRAINER_CHOICES = [(label, key) for key, label in TRAINERS.items()] @@ -140,7 +146,8 @@ def _cloud_model_default(pinned: str, choices: list[tuple[str, str]]) -> str: def on_dataset_type_change(dataset_type: str, name: str = "", - style_key: str = shot_style.MATCH, style_text: str = ""): + style_key: str = shot_style.MATCH, style_text: str = "", + identity: str = "identity"): """Retune every type-dependent control across the tabs, and remember the choice for the next launch. @@ -164,7 +171,7 @@ def on_dataset_type_change(dataset_type: str, name: str = "", visible=bool(_TYPE_GUIDANCE.get(dataset_type, ""))), gr.Button(value=f"Rebuild default plan with {label.lower()}", interactive=not is_style), # refresh plan - _plan_table(dataset_type, name, style_key, style_text), # shot plan table + _plan_table(dataset_type, name, style_key, style_text, identity), # shot plan gr.Button(interactive=not is_style), # generate gr.Button(interactive=not is_style), # regenerate gr.Button(visible=not (is_style or is_concept)), # randomize outfits @@ -264,20 +271,40 @@ def _allowed_media_paths() -> list[str]: # Human-editable columns lead; the long prompt cells trail. Column ORDER and # WIDTHS must be set explicitly: pydantic field order otherwise puts the two # ~200-char prompts in the middle, squeezing `outfit` to an unreadable sliver. -PLAN_COLUMNS = ["id", "kind", "emotion", "setting", "outfit", +PLAN_COLUMNS = ["id", "kind", "framing", "emotion", "setting", "outfit", "local_prompt", "cloud_prompt", "chain_from"] -PLAN_COLUMN_WIDTHS = ["110px", "70px", "110px", "200px", "220px", +PLAN_COLUMN_WIDTHS = ["110px", "70px", "90px", "110px", "200px", "220px", "260px", "260px", "100px"] def _plan_table(dataset_type: str, name: str = "", style_key: str = shot_style.MATCH, - style_text: str = "") -> pd.DataFrame: - """The ② table for a dataset type (empty for Style, which never generates).""" - return _shots_to_df(plan_for_type(dataset_type, name, style_key, style_text)) + style_text: str = "", identity: str = "identity") -> pd.DataFrame: + """The ② table for a dataset type (empty for Style, which never generates). + + A character under the "identity" policy starts dressed in varied outfits, so + the trigger learns the person rather than their clothes. + """ + shots = plan_for_type(dataset_type, name, style_key, style_text) + if dataset_type == "character" and identity == "identity": + from studio.wardrobe import dress + + shots = dress(shots) + return _shots_to_df(shots) + + +def _identity_visible(dataset_type: str): + return gr.update(visible=dataset_type == "character") + + +def on_identity_change(df: pd.DataFrame, identity: str, dataset_type: str): + """Dress or undress the current plan to match the identity policy.""" + if dataset_type != "character": + return gr.skip(), gr.skip() + return randomize_outfits(df) if identity == "identity" else clear_outfits(df) def rebuild_plan_for_style(dataset_type: str, name: str, style_key: str, - style_text: str): + style_text: str, identity: str = "identity"): """Rebuild the ② table when the shot style changes, and remember the choice. The style is baked into the prompt cells at build time (so the table shows @@ -298,7 +325,7 @@ def rebuild_plan_for_style(dataset_type: str, name: str, style_key: str, note = ("⚠️ Custom style selected but no description typed — falling back to " "**matching the reference image**. Type a style below and the plan " "rebuilds.") - return _plan_table(dataset_type, name, style_key, style_text), note + return _plan_table(dataset_type, name, style_key, style_text, identity), note def _toggle_style_text(style_key: str): @@ -505,7 +532,7 @@ def val(row, k): cols = ( "id", "kind", "local_prompt", "cloud_prompt", - "chain_from", "emotion", "setting", "outfit", + "chain_from", "emotion", "setting", "outfit", "framing", ) return [Shot(**{k: val(row, k) for k in cols}) for _, row in df.iterrows() if val(row, "id").strip()] @@ -578,12 +605,22 @@ def _preprocess_note(reports, out_dir: Path, alpha_cutout: bool) -> str: if not ok: head = (f"❌ **No image could be preprocessed** — nothing was written to " f"{out_dir}.") - elif alpha_cutout: - head = (f"✅ {len(ok)} image(s) preprocessed (transparent cutout) into " - f"{out_dir} — not auto-filled into ②/③, which expect a " - f"white-background reference.") else: head = f"✅ {len(ok)} image(s) preprocessed into {out_dir}" + refs = next((r.reference.parent for r in ok if r.reference), None) + if refs and alpha_cutout: + head += (f" — transparent cutouts in {refs}, not auto-filled into ②, " + f"which expects a white-background reference.") + elif refs: + head += f" — isolated references for ② in {refs}." + cut = [r.source.name for r in ok if r.cut_off] + if cut and len(cut) == len(ok): + head += (f"\n\n⚠️ **Every reference is cut off at the bottom** " + f"({', '.join(cut)}) — full-body shots will invent the legs. Add a " + f"full-body photo as another reference if you have one.") + elif cut: + head += (f"\n\nCut off at the bottom: {', '.join(cut)} — the other " + f"reference(s) show more of the body.") if not failed: return head lines = "\n".join(f"- `{r.source.name}` — {r.error} \n → {_failure_hint(r.error)}" @@ -647,15 +684,15 @@ def report(msg: str): # auto-filled folders all have to survive a partial failure. gr.Warning(f"Preprocessed {len(ok)} of {len(reports)} — " f"{len(failed)} skipped, see the result note.") - if alpha_cutout: - # Alpha-cutout output isn't a drop-in reference for ②/③ (see the checkbox's - # info text) — leave whatever those fields already had alone instead of - # silently pointing them at an image most consumers will read as un-isolated. - gen_src, cap = gr.update(), gr.update() + # Auto-fill downstream tabs (they can still be pointed anywhere else). ③ gets + # the training copies; ② gets the isolated references when there are any. + # Alpha cutouts aren't a drop-in reference for ② (see the checkbox's info). + refs = [r.reference for r in ok if r.reference] + if alpha_cutout and refs: + gen_src = gr.update() else: - # Auto-fill downstream tabs (they can still be pointed anywhere else) - gen_src, cap = str(out_dir), str(out_dir) - return gallery, note, "\n".join(log), gen_src, cap + gen_src = str(refs[0].parent if refs else out_dir) + return gallery, note, "\n".join(log), gen_src, str(out_dir) # ---------- ② generate & curate ---------- @@ -663,7 +700,7 @@ def report(msg: str): def do_generate(files: list[str], folder: str, plan_df: pd.DataFrame, engine: str, cloud_model: str, exclude_props: bool, isolate_angles: bool, isolation_backend: str, subject_prompt: str, exclude_prompt: str, - front: bool, gen_dir_prev: str, results_state, + front: bool, anchor: bool, gen_dir_prev: str, results_state, progress=gr.Progress()): sources = _inputs(files, folder) out_dir = _validate_out_dir(gen_dir_prev) if gen_dir_prev.strip() else _stamped("generated") @@ -680,7 +717,7 @@ def report(msg: str): sources, shots, engine, out_dir, cloud_model=cloud_model, isolate_angles=isolate_angles, subject_prompt=subject_prompt or "character", exclude_prompt=exclude_prompt, isolation_backend=isolation_backend, - exclude_props=exclude_props, front=front, should_stop=JOB, progress=report) + exclude_props=exclude_props, front=front, anchor=anchor, should_stop=JOB, progress=report) except OSError as e: raise gr.Error(f"Couldn't write to '{out_dir}': {e}. Check the output folder " f"path (valid drive, no forbidden characters, writable).") from e @@ -696,7 +733,7 @@ def report(msg: str): def do_regenerate(files: list[str], folder: str, plan_df: pd.DataFrame, engine: str, cloud_model: str, exclude_props: bool, isolate_angles: bool, isolation_backend: str, subject_prompt: str, exclude_prompt: str, - front: bool, gen_dir: str, results_state, keep_ids: list[str], + front: bool, anchor: bool, gen_dir: str, results_state, keep_ids: list[str], progress=gr.Progress()): if not results_state: raise gr.Error("Nothing generated yet.") @@ -716,7 +753,7 @@ def do_regenerate(files: list[str], folder: str, plan_df: pd.DataFrame, engine: sources, _df_to_shots(plan_df), engine, Path(gen_dir), cloud_model=cloud_model, isolate_angles=isolate_angles, subject_prompt=subject_prompt or "character", exclude_prompt=exclude_prompt, isolation_backend=isolation_backend, - exclude_props=exclude_props, front=front, existing=results_state, + exclude_props=exclude_props, front=front, anchor=anchor, existing=results_state, only_ids=redo, should_stop=JOB, progress=log.append) except OSError as e: raise gr.Error(f"Couldn't write to '{gen_dir}': {e}. Check the output folder " @@ -742,6 +779,50 @@ def do_refresh_disk(results_state, gen_dir: str, keep_ids: list[str]): return results, rows, gallery, keep, note +def coverage_note(results_state, keep_ids: list[str], percent: float) -> str: + """② coverage line for the kept shots (see `shotplan.coverage`).""" + kept = set(keep_ids or []) + shots = [r.shot for r in results_state or [] if r.path and r.shot.id in kept] + return coverage(shots, (percent or 40) / 100) + + +def primary_reference(files: list[str], folder: str) -> str | None: + """The reference ② leads with (it sets the local engine's output shape).""" + try: + return str(_inputs(files, folder)[0]) + except gr.Error: + return None + + +def find_gen_duplicates(results_state, keep_ids: list[str], files: list[str], + folder: str, distance: float) -> str: + """Near-duplicate kept shots, and shots that just copied a reference. + + Uses ④'s sensitivity slider so both scans agree on what "near" means. + """ + from studio.dedupe import find_near_duplicate_groups + + kept = {r.path for r in results_state or [] + if r.path and r.path.exists() and r.shot.id in set(keep_ids or [])} + if not kept: + raise gr.Error("No kept shots to compare — generate first.") + refs = [] if primary_reference(files, folder) is None else _inputs(files, folder) + groups = find_near_duplicate_groups(refs + sorted(kept), max_distance=int(distance)) + copies = [g for g in groups if any(p in refs for p in g)] + twins = [g for g in groups if g not in copies] + if not groups: + return f"🔁 No near-duplicates among {len(kept)} kept shot(s) (sensitivity {distance:g})." + note = "" + if copies: + note += (f"🔁 **{len(copies)} shot group(s) look like a copy of a reference** — " + f"the generator ignored the prompt; regenerate them: {_groups_text(copies)}") + if twins: + note += ("\n\n" if note else "") + ( + f"🔁 **{len(twins)} near-duplicate group(s)** among kept shots — keep one of " + f"each: {_groups_text(twins)}") + return note + + def send_kept_to_caption(results_state, keep_ids: list[str], gen_dir: str): """Load ②'s output folder into ③ with only the kept shots preselected. @@ -831,7 +912,8 @@ def do_test_caption(folder: str, selected: list[str], captioner_key: str, name: str, trigger: str, gemini_model: str, style: str, gen_thr: float, char_thr: float, prefix: str, suffix: str, blacklist: str, rating: bool, underscores: bool, - dataset_type: str = "character", sparse: bool = False): + dataset_type: str = "character", sparse: bool = False, + identity: str = "identity"): if not folder.strip() or not selected: raise gr.Error("Load a folder and select at least one image first.") path = Path(folder.strip()) / selected[0] @@ -844,7 +926,7 @@ def do_test_caption(folder: str, selected: list[str], captioner_key: str, cap = Captioner(captioner_key, model_override=model_override, spec_overrides=spec_overrides) try: raw = cap.caption(path, subject=name or "the character", style=style, - dataset_type=dataset_type, sparse=sparse) + dataset_type=dataset_type, sparse=sparse, identity=identity) except Exception as e: raise gr.Error(str(e)) from e finally: @@ -1007,7 +1089,8 @@ def do_caption(folder: str, selected: list[str], captioner_key: str, gen_thr: float, char_thr: float, prefix: str, suffix: str, blacklist: str, rating: bool, underscores: bool, skip_existing: bool, dataset_type: str, sparse: bool, - exp_folders_prev: str, carry_prev, progress=gr.Progress()): + exp_folders_prev: str, carry_prev, identity: str = "identity", + progress=gr.Progress()): if not folder.strip() or not selected: raise gr.Error("Load a folder and select the images to caption first.") base = Path(folder.strip()) @@ -1037,7 +1120,7 @@ def persist(img: Path, caption: str) -> None: style=style, prefix=prefix, suffix=suffix, skip_existing=skip_existing, blacklist=blacklist, dataset_type=dataset_type, sparse=sparse, on_item=persist, - should_stop=JOB) + should_stop=JOB, identity=identity) except Exception as e: if not written: raise gr.Error(f"Captioning failed: {friendly_api_error(e)}") from e @@ -1170,6 +1253,11 @@ def refresh_export_preview(folders_text: str, current_rows, current_selected): return rows, _picker_gallery(rows, values), gr.CheckboxGroup(choices=choices, value=values), note +def _groups_text(groups: list[list[Path]], limit: int = 5) -> str: + shown = "; ".join("=".join(f"{p.parent.name}/{p.name}" for p in g) for g in groups[:limit]) + return shown + (f" (+{len(groups) - limit} more)" if len(groups) > limit else "") + + def load_export_preview(folders_text: str, dup_distance: float = 5, carry=None): if not (folders_text or "").strip(): raise gr.Error("Enter at least one folder of captioned images (one per line).") @@ -1183,15 +1271,17 @@ def load_export_preview(folders_text: str, dup_distance: float = 5, carry=None): "Only checked images are exported; a checked image without a usable " "caption is skipped and called out in the result.") try: # advisory near-duplicate scan — never blocks the preview - from studio.dedupe import find_near_duplicate_groups + from studio.dedupe import find_bursts, find_near_duplicate_groups groups = find_near_duplicate_groups(images, max_distance=int(dup_distance)) if groups: - shown = "; ".join("=".join(f"{p.parent.name}/{p.name}" for p in g) - for g in groups[:5]) - more = f" (+{len(groups) - 5} more)" if len(groups) > 5 else "" note += (f"\n\n🔁 **{len(groups)} near-duplicate group(s)** — consider " - f"unchecking extras so one shot isn't over-weighted: {shown}{more}") + f"unchecking extras so one shot isn't over-weighted: " + f"{_groups_text(groups)}") + bursts = find_bursts(images) + if bursts: + note += (f"\n\n📸 **{len(bursts)} burst(s)** — photos taken seconds apart " + f"are one moment; keep the best of each: {_groups_text(bursts)}") except Exception: pass try: # advisory caption health + tag frequency — never blocks the preview @@ -1215,7 +1305,7 @@ def load_export_preview(folders_text: str, dup_distance: float = 5, carry=None): def do_export(selected: list[str], name: str, trigger: str, output_root: str, make_zip: bool = False, dataset_type: str = "character", style_key: str = shot_style.MATCH, style_text: str = "", - ilb_handoff: bool = False, holdout=0): + ilb_handoff: bool = False, holdout=0, identity: str = "identity"): if not selected: raise gr.Error("Click '📂 Load & preview', then keep at least one image checked.") from studio.package import package_dataset, resolve_export_items @@ -1236,6 +1326,8 @@ def do_export(selected: list[str], name: str, trigger: str, output_root: str, "source_folders": source_folders, "skipped_uncaptioned": res.missing, "skipped_empty_caption": res.empties} + if dataset_type == "character": + metadata["identity"] = identity out_root = _validate_out_dir(output_root) try: ds = package_dataset(res.items, out_root, name, trigger, metadata, @@ -1332,8 +1424,8 @@ def report(msg: str): def refresh_plan(name: str, dataset_type: str = "character", style_key: str = shot_style.MATCH, - style_text: str = "") -> pd.DataFrame: - return _plan_table(dataset_type, name, style_key, style_text) + style_text: str = "", identity: str = "identity") -> pd.DataFrame: + return _plan_table(dataset_type, name, style_key, style_text, identity) def do_save_plan(plan_df: pd.DataFrame, plan_name: str) -> str: @@ -1725,6 +1817,14 @@ def _check_for_update(): "Concept generate a shot set in ②; Style brings its own images and starts " "at ③ Caption. Tunes caption framing, the ② shot plan, the ① isolation " "default, and the ⑤ sample prompt.") + identity_policy = gr.Radio( + [("Identity only — outfits vary", "identity"), + ("Signature costume — the outfit is part of the character", "costume")], + value="identity", label="Identity policy", + info="Identity only dresses ②'s angle/pose shots in varied outfits and has ③ " + "describe the clothing, so the trigger learns the person. Signature " + "costume keeps the reference's outfit and leaves it out of captions, so " + "the trigger carries it.") # The ONE place that owns "who is this dataset about". ②, ③, ④ and ⑤ all # read these two boxes directly instead of each keeping its own copy — see # the note above `type_outputs` for why copies were removed rather than kept @@ -1789,9 +1889,11 @@ def _check_for_update(): info="Auto uses ComfyUI models if reachable, " "else basic Lanczos resize.") isolate = gr.Checkbox(value=True, - label="Isolate subject (cutout onto white background)", - info="Cuts the subject out onto white so background " - "and props aren't baked into the LoRA.") + label="Isolate subject for ② generation (training " + "copies keep their background)", + info="Also writes the subject cut out onto white to " + "refs/, as ②'s reference, so the old background " + "and props don't leak into generated shots.") isolation_backend = gr.Dropdown(ISOLATION_CHOICES, value=settings.isolation_backend, label="Isolation backend", @@ -1804,16 +1906,17 @@ def _check_for_update(): "for a Concept dataset ('radio', 'sword').") exclude_prompt = gr.Textbox( label="Objects to remove (props the subject holds/touches)", - placeholder="microphone, microphone stand", - info="Usually leave blank — SAM3 already excludes most props. Use only " - "for a prop fused into the subject.") + placeholder="cup, plate, microphone", + info="Name anything the subject holds. SAM3 keeps a held object as " + "part of the subject, and ② then redraws it in every shot. " + "Background clutter needs no entry.") pre_tighten = gr.Checkbox( - value=False, label="Tighten crop to subject (after isolation)", - info="Crop out the white padding around the isolated subject so framing " - "is consistent and less empty background is trained. Needs isolation on.") + value=False, label="Tighten crop to subject (refs/ copy)", + info="Crop out the white padding around the isolated reference so the " + "subject fills more of what ② sees. Needs isolation on.") pre_alpha_cutout = gr.Checkbox( value=False, label="Transparent cutout (alpha) instead of white", - info="Exports the isolated subject on a transparent background for your " + info="Writes the refs/ copy on a transparent background for your " "own compositing workflows. Builtin SAM3 backend only. Leave off " "(default) if you're continuing to ② Generate — it expects a white " "background reference. Needs isolation on.") @@ -1885,11 +1988,20 @@ def _check_for_update(): "Qwen draw it) — isolate the source in ① instead, the more " "reliable fix either way. Character-oriented wording — off " "by default for Concept datasets.") + gen_anchor = gr.Checkbox( + value=False, label="Anchor shot: build every shot from the front view", + info="Generates the front full-body view first, then leads every " + "other shot's references with it. Helps when your sources " + "are partial or poor (face in shadow, body cut off). It " + "copies whatever the front view gets wrong (a held object, " + "an outfit) into every shot, so check that view first.") gen_isolate = gr.Checkbox(value=False, label="Isolate generated angle shots (white background)", - info="Cut generated angle shots onto white too " - "(replaces each angle shot's setting " - "with a plain white background).") + info="Cut generated angle shots onto white too. " + "Off by default: a white void in many " + "training images trains into the LoRA. " + "Remove held props with ①'s exclude " + "prompt instead.") gen_iso_backend = gr.Dropdown(ISOLATION_CHOICES, value=settings.isolation_backend, label="Isolation backend", @@ -1924,9 +2036,10 @@ def _check_for_update(): "exact text a row will send.") wardrobe_note = gr.Markdown( "The **outfit** column varies wardrobe without breaking identity — " - "leave blank to keep the reference's clothing. If your source images " - "all show the same clothes, randomizing here stops the LoRA learning " - "the outfit as part of the character. Save/load plans as reusable " + "filled for you under **Identity only** (header), blank under " + "**Signature costume** to keep the reference's clothing. Varied " + "outfits stop the LoRA learning the clothes as part of the " + "character. Save/load plans as reusable " "prompt libraries under `shot_plans/`.") with gr.Row(): btn_outfits = gr.Button("🎲 Randomize outfits", scale=1) @@ -1953,18 +2066,30 @@ def _check_for_update(): gen_send_note = gr.Markdown() # allow_preview=False so a click TOGGLES the shot instead of opening a # lightbox; the Zoom checkbox flips it back when you want a closer look. - gen_gallery = gr.Gallery( - label="Generated shots — click a thumbnail to keep/reject it " - "(shift-click for a range)", - columns=6, height=420, allow_preview=False, elem_id="dd-gallery-gen") + with gr.Row(): + gen_gallery = gr.Gallery( + label="Generated shots — click a thumbnail to keep/reject it " + "(shift-click for a range)", + columns=6, height=420, allow_preview=False, elem_id="dd-gallery-gen", + scale=4) + gen_primary = gr.Image(label="Primary reference", height=420, + interactive=False, scale=1) with gr.Row(): btn_gen_all = gr.Button("Select all", size="sm") btn_gen_none = gr.Button("Select none", size="sm") + btn_gen_dupes = gr.Button("🔁 Find near-duplicates", size="sm") gen_zoom = gr.Checkbox(value=False, label="🔍 Zoom on click", elem_id="dd-zoom-gen", info="Clicks enlarge instead of selecting.") keep = gr.CheckboxGroup(label="✅ Kept shots — UNCHECK to reject", choices=[], elem_id="dd-picks-gen") + with gr.Row(): + gen_coverage = gr.Markdown() + gen_dominance = gr.Number( + value=40, minimum=10, maximum=100, step=5, scale=0, + label="Dominance warning (%)", + info="Warn when one view or expression is more than this share " + "of the kept shots.") with gr.Tab("③ Caption", id="caption"): gr.Markdown("Tag any folder of images with caption `.txt` sidecars — the folder " @@ -2322,15 +2447,21 @@ def _check_for_update(): cap_sparse, project_name, project_trigger] # The style controls are INPUTS only — adding them to type_outputs would # change the handler's return arity, which a test pins on purpose. - type_inputs = [dataset_type, project_name, gen_style, gen_style_text] + type_inputs = [dataset_type, project_name, gen_style, gen_style_text, identity_policy] dataset_type.change(on_dataset_type_change, type_inputs, type_outputs) demo.load(on_dataset_type_change, type_inputs, type_outputs) - - refresh.click(refresh_plan, [project_name, dataset_type, gen_style, gen_style_text], + # Wardrobe is a character-only idea. + for _event in (dataset_type.change, demo.load): + _event(_identity_visible, [dataset_type], [identity_policy]) + identity_policy.change(on_identity_change, [plan, identity_policy, dataset_type], + [plan, plan_note]) + + refresh.click(refresh_plan, + [project_name, dataset_type, gen_style, gen_style_text, identity_policy], [plan]) # Rebuild on pick. The custom textbox applies on Enter/blur rather than per # keystroke — rebuilding 24 prompts on every character typed is pure churn. - _style_inputs = [dataset_type, project_name, gen_style, gen_style_text] + _style_inputs = [dataset_type, project_name, gen_style, gen_style_text, identity_policy] gen_style.change(_toggle_style_text, [gen_style], [gen_style_text]) gen_style.change(rebuild_plan_for_style, _style_inputs, [plan, plan_note]) gen_style_text.submit(rebuild_plan_for_style, _style_inputs, [plan, plan_note]) @@ -2353,7 +2484,7 @@ def _force_refresh(current: str): gen_inputs = [gen_files, gen_src_folder, plan, engine, cloud_model, gen_exclude_props, gen_isolate, gen_iso_backend, gen_subject, - gen_exclude, gen_front] + gen_exclude, gen_front, gen_anchor] btn_gen.click(do_generate, gen_inputs + [gen_out_dir, results_state], [results_state, gen_rows, gen_gallery, keep, log_box, gen_out_dir, cap_folder]) @@ -2374,6 +2505,13 @@ def _force_refresh(current: str): (exp_gallery, exp_rows, exp_select, exp_zoom)): _boxes.change(_picker_mark, [_rows, _boxes], [_gallery]) _zoom.change(_set_zoom, [_zoom], [_gallery]) + for _event in (keep.change, gen_dominance.change): + _event(coverage_note, [results_state, keep, gen_dominance], [gen_coverage]) + for _event in (gen_files.change, gen_src_folder.change): + _event(primary_reference, [gen_files, gen_src_folder], [gen_primary]) + btn_gen_dupes.click(find_gen_duplicates, + [results_state, keep, gen_files, gen_src_folder, exp_dup_dist], + [gen_send_note]) btn_gen_all.click(_pick_all, [gen_rows], [keep]) btn_gen_none.click(_pick_none, [gen_rows], [keep]) btn_cap_all.click(_pick_all, [cap_rows], [cap_select]) @@ -2434,7 +2572,8 @@ def _cap_cost(key: str, model: str, selected: list[str]) -> str: [cap_folder, cap_select, captioner, project_name, project_trigger, cap_gemini_model, cap_style, cap_gen_thr, cap_char_thr, cap_prefix, cap_suffix, - cap_blacklist, cap_rating, cap_underscores, dataset_type, cap_sparse], + cap_blacklist, cap_rating, cap_underscores, dataset_type, cap_sparse, + identity_policy], [test_caption]) btn_caption.click( do_caption, @@ -2442,7 +2581,7 @@ def _cap_cost(key: str, model: str, selected: list[str]) -> str: cap_gemini_model, cap_style, cap_gen_thr, cap_char_thr, cap_prefix, cap_suffix, cap_blacklist, cap_rating, cap_underscores, cap_skip, dataset_type, cap_sparse, - exp_folders, cap_carry], + exp_folders, cap_carry, identity_policy], [cap_rows, cap_gallery, cap_select, cap_result, log_box, exp_folders, cap_analysis, cap_carry]) \ .then(_editor_choices, [cap_folder], [cap_edit_file, cap_edit_names]) @@ -2455,7 +2594,8 @@ def _cap_cost(key: str, model: str, selected: list[str]) -> str: [exp_rows, exp_gallery, exp_select, exp_preview_note]) btn_export.click(do_export, [exp_select, project_name, project_trigger, output_root, exp_zip, - dataset_type, gen_style, gen_style_text, exp_ilb, exp_holdout], + dataset_type, gen_style, gen_style_text, exp_ilb, exp_holdout, + identity_policy], [exp_result, tr_dataset, exp_ds_dir]) \ .then(inspect_dataset, [tr_dataset, dataset_type], [tr_stats]) btn_publish_hf.click(do_publish_hf, [exp_ds_dir, exp_hf_repo, exp_hf_private], diff --git a/cli.py b/cli.py index 7d7efa2..6e0919d 100644 --- a/cli.py +++ b/cli.py @@ -1,7 +1,7 @@ """Headless CLI. Every stage is its own subcommand and runs standalone: python cli.py preprocess ./sources --out ./prepped - python cli.py generate ./prepped --name "Sy Snootles" --engine comfyui + python cli.py generate ./prepped/refs --name "Sy Snootles" --engine comfyui python cli.py caption ./any/folder --trigger sysnootles # .txt sidecars python cli.py export ./prepped ./generated --name "Sy Snootles" python cli.py build img.png --name "Sy Snootles" --trigger sysnootles # all four @@ -49,7 +49,12 @@ def _report_outputs(reports: list) -> list[Path]: def _echo_preprocess_failures(reports: list) -> int: - """Print the skipped sources; return how many there were.""" + """Print the skipped sources (and a cropped-reference warning); return how + many were skipped.""" + ok = [r for r in reports if r.output] + if ok and all(r.cut_off for r in ok): + typer.echo("WARNING: every reference is cut off at the bottom - full-body " + "shots will invent the legs. Add a full-body photo if you have one.") failed = [r for r in reports if r.error] for r in failed: typer.echo(f" SKIPPED {r.source.name}: {r.error}") @@ -160,23 +165,42 @@ def _props_default(exclude_props: bool | None, dataset_type: str) -> bool: return exclude_props +_ANCHOR_HELP = ("Generate the front full-body view first and lead every other " + "shot's references with it (helps partial or poor sources; it " + "copies the front view's mistakes too, so check it first)") +_IDENTITY_HELP = ("Characters only: identity (default — angle/pose shots get varied " + "outfits and captions describe the clothing, so the trigger learns " + "the person) | costume (keep the reference's outfit, leave it out of " + "captions, so the trigger carries it)") + + +def _check_identity(identity: str) -> str: + from studio.config import IDENTITY_POLICIES + + if identity not in IDENTITY_POLICIES: + raise typer.BadParameter(f"--identity must be one of {list(IDENTITY_POLICIES)}") + return identity + + +def _wants_outfits(identity: str, randomize: bool, dataset_type: str) -> bool: + """--randomize-outfits is the pre-0.19 spelling of --identity identity.""" + return randomize or (_check_identity(identity) == "identity" + and dataset_type == "character") + + def _dress(shots: list, dataset_type: str = "character") -> list: """Fill angle/pose shots with random unisex outfits (close-ups stay blank). Wardrobe is a character-only idea — an object has no clothing — so it is a no-op (with a note) for any other dataset type. """ - from studio.wardrobe import OUTFIT_SHOT_KINDS, random_outfits + from studio.wardrobe import dress if dataset_type != "character": typer.echo(f"Skipping outfit randomization: not applicable to a " f"{dataset_type} dataset.") return shots - targets = [s for s in shots if s.kind in OUTFIT_SHOT_KINDS] - outfits = random_outfits(len(targets)) - dressed = dict(zip((s.id for s in targets), outfits, strict=True)) - return [s.model_copy(update={"outfit": dressed[s.id]}) if s.id in dressed else s - for s in shots] + return dress(shots) @app.command() @@ -188,16 +212,17 @@ def preprocess( help="Force restoration on/off (default: auto)"), restore_backend: str = typer.Option(settings.restore_backend, help="auto | comfyui | basic"), isolate: bool = typer.Option(True, "--isolate/--no-isolate", - help="Cut out subject, drop background/props"), + help="Also write the subject cut out onto white to refs/, " + "for generate; training copies keep their background"), isolation_backend: str = typer.Option(settings.isolation_backend, help="builtin | comfyui"), subject_prompt: str = typer.Option("character", help="SAM3 prompt for what to keep"), - exclude_prompt: str = typer.Option("", help="SAM3 prompt for held props to remove"), + exclude_prompt: str = typer.Option("", help="Objects the subject holds, to cut out (SAM3 keeps a held object otherwise)"), tighten: bool = typer.Option( False, "--tighten/--no-tighten", - help="Crop to the subject's bounding box after isolation (less white padding)"), + help="Crop the refs/ copy to the subject's bounding box"), alpha_cutout: bool = typer.Option( False, "--alpha-cutout/--no-alpha-cutout", - help="Export on a transparent background instead of white (builtin backend " + help="refs/ copy on a transparent background instead of white (builtin backend " "only; for your own compositing workflows — not meant to feed 'generate')"), front: bool = typer.Option(False, help="Jump ComfyUI's pending queue"), ): @@ -215,6 +240,9 @@ def preprocess( typer.echo("No image could be preprocessed.") raise typer.Exit(1) typer.echo(f"Done: {out}") + refs = next((r.reference.parent for r in reports if r.reference), None) + if refs: + typer.echo(f"Isolated references for generate: {refs}") @app.command() @@ -248,8 +276,10 @@ def generate( None, "--exclude-props/--keep-props", help="Ask the generator to omit bags/held objects from the reference " "(default: on for character, off for concept — the clause is character-worded)"), + identity: str = typer.Option("identity", "--identity", help=_IDENTITY_HELP), randomize_outfits: bool = typer.Option( - False, help="Dress angle/pose shots in random unisex outfits (character only)"), + False, help="Same as --identity identity (kept for old scripts)"), + anchor: bool = typer.Option(False, "--anchor/--no-anchor", help=_ANCHOR_HELP), front: bool = typer.Option(False, help="Jump ComfyUI's pending queue"), ): """Generate the shot set from reference image(s) (standalone).""" @@ -258,14 +288,14 @@ def generate( shots = _plan_for(dtype, name, _check_shot_style(shot_style), shot_style_text) if max_shots: shots = shots[:max_shots] - if randomize_outfits: + if _wants_outfits(identity, randomize_outfits, dtype): shots = _dress(shots, dtype) _echo_cloud_estimate(engine, cloud_model, len(shots)) results = pipeline.generate_shots( _expand(references), shots, engine, out, cloud_model=cloud_model, isolate_angles=isolate_angles, subject_prompt=subject_prompt, exclude_prompt=exclude_prompt, exclude_props=_props_default(exclude_props, dtype), - front=front, progress=typer.echo) + front=front, anchor=anchor, progress=typer.echo) if not any(r.path for r in results): raise typer.Exit(1) typer.echo(f"Done: {out}") @@ -304,6 +334,7 @@ def caption( skip_captioned: bool = typer.Option( False, "--skip-captioned", help="Leave images that already have a non-empty .txt caption untouched"), + identity: str = typer.Option("identity", "--identity", help=_IDENTITY_HELP), ): """Write .txt caption sidecars for every image in a folder (standalone). @@ -330,7 +361,8 @@ def caption( caption_folder(folder, captioner, name, trigger, progress=typer.echo, model_override=model_override, spec_overrides=spec_overrides, style=style, prefix=prefix, suffix=suffix, skip_existing=skip_captioned, - blacklist=drop_tags, dataset_type=dtype, sparse=sparse) + blacklist=drop_tags, dataset_type=dtype, sparse=sparse, + identity=_check_identity(identity)) @app.command() @@ -447,20 +479,25 @@ def build( max_shots: int = typer.Option(0, help="Limit number of shots (0 = full plan)"), isolate: bool = typer.Option( None, "--isolate/--no-isolate", - help="Cut out subject, drop background/props (default: on, except style — " - "a style is whole-image)"), + help="Cut the subject out onto white as ②'s reference; training copies keep " + "their background (default: on, except style — a style is whole-image)"), tighten: bool = typer.Option( False, "--tighten/--no-tighten", - help="Crop to the subject's bounding box after isolation (less white padding)"), + help="Crop the isolated reference to the subject's bounding box"), + isolate_angles: bool = typer.Option( + False, "--isolate-angles/--no-isolate-angles", + help="Also cut generated angle shots onto white (default off: a white " + "background trains in)"), subject_prompt: str = typer.Option("character", help="SAM3 prompt for what to keep"), - exclude_prompt: str = typer.Option("", help="SAM3 prompt for held props to remove"), + exclude_prompt: str = typer.Option("", help="Objects the subject holds, to cut out (SAM3 keeps a held object otherwise)"), cloud_model: str = typer.Option("", help=f"Cloud image model (default {settings.gemini_image_model})"), exclude_props: bool = typer.Option( None, "--exclude-props/--keep-props", help="Ask the generator to omit bags/held objects from the reference " "(default: on for character, off for concept)"), + identity: str = typer.Option("identity", "--identity", help=_IDENTITY_HELP), randomize_outfits: bool = typer.Option( - False, help="Dress angle/pose shots in random unisex outfits (character only)"), + False, help="Same as --identity identity (kept for old scripts)"), zip_: bool = typer.Option(False, "--zip", help="Also write a .zip of the dataset"), prefix: str = typer.Option( "", help="Fixed text added before every caption (e.g. Pony 'score_9, score_8_up')"), @@ -468,6 +505,7 @@ def build( drop_tags: str = typer.Option( "", "--drop-tags", help="Comma-separated tags to strip from tag captions (e.g. 'watermark, signature')"), + anchor: bool = typer.Option(False, "--anchor/--no-anchor", help=_ANCHOR_HELP), front: bool = typer.Option(False, help="Jump ComfyUI's pending queue"), ): """Full pipeline: preprocess -> generate -> caption -> export.""" @@ -476,6 +514,7 @@ def build( style = _check_caption_style(caption_style) dtype = _check_dataset_type(dataset_type) + identity = _check_identity(identity) # Check the generation backend before spending a preprocess pass on images # it would then refuse to use. Style never generates, so it never needs one. if dtype != "style": @@ -509,16 +548,18 @@ def build( shots = _plan_for(dtype, name, _check_shot_style(shot_style), shot_style_text) if max_shots: shots = shots[:max_shots] - if randomize_outfits: + if _wants_outfits(identity, randomize_outfits, dtype): shots = _dress(shots, dtype) _echo_cloud_estimate(engine, cloud_model, len(shots)) + # Generate from the isolated references; caption the training copies. + refs = [r.reference or r.output for r in reports if r.output] results = pipeline.generate_shots( - prepped, shots, engine, run_dir / "generated", - cloud_model=cloud_model, isolate_angles=do_isolate, subject_prompt=subject_prompt, + refs, shots, engine, run_dir / "generated", + cloud_model=cloud_model, isolate_angles=isolate_angles, subject_prompt=subject_prompt, exclude_prompt=exclude_prompt, exclude_props=_props_default(exclude_props, dtype), front=front, - progress=typer.echo) + anchor=anchor, progress=typer.echo) kept = [r.path for r in results if r.path] if not kept: typer.echo("No shots succeeded; aborting before captioning.") @@ -527,7 +568,7 @@ def build( all_images = prepped + kept items = caption_images(all_images, captioner, name, trigger, progress=typer.echo, style=style, prefix=prefix, suffix=suffix, blacklist=drop_tags, - dataset_type=dtype, sparse=sparse) + dataset_type=dtype, sparse=sparse, identity=identity) metadata = { "character_name": name, "trigger": trigger, @@ -538,6 +579,8 @@ def build( "caption_style": style, "sources": [str(s) for s in images], } + if dtype == "character": + metadata["identity"] = identity if results: # generation ran (character/concept) metadata["engine"] = engine metadata["shots"] = [{"id": r.shot.id, "seed": r.seed, "error": r.error} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a600c69..da1b620 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architecture -Version: 0.18.1 +Version: 0.19.0 ``` app.py Gradio UI — thin wiring over the stage functions (5 tabs). @@ -53,15 +53,18 @@ studio/ jobs.py Cooperative cancellation: JobControl (a threading.Event stop flag) + should_stop_now(). Checked BETWEEN items by the three batch loops; UI-free so the CLI and tests drive it directly - preprocess.py Restore (comfyui|basic|auto) + isolate + optional tighten-crop - + resize, stamping a `dd_source` provenance PNG chunk. - Atomic: any failure deletes the partial output. - failed_report() turns a failure into data, not an exception + preprocess.py Restore (comfyui|basic|auto) + resize, stamping a `dd_source` + provenance PNG chunk. The training copy keeps its background; + with isolation on, the cut-out (optionally tightened / alpha) + goes to `refs/` as ②'s reference only, and `cut_off` records a + subject touching the bottom edge. Atomic: any failure deletes + both outputs. failed_report() turns a failure into data isolate.py Subject isolation: builtin SAM3 (transformers) | comfyui. crop_to_content() tightens the isolated (subject-on-white, or subject-with-alpha) image to the subject's bounding box (opt-in, - both backends). alpha_cutout composites onto transparency - instead of white — builtin backend only, see the Gotcha + both backends); touches_bottom() is the cut-off check. alpha_cutout + composites onto transparency instead of white — builtin backend + only, see the Gotcha tagger.py ONNX booru-tagger backend: canonical tags straight from image features — WD (Danbooru) and Z3D (e621) via one code path, differing only in tag file + category SCHEMES. Pure @@ -71,8 +74,9 @@ studio/ (Danbooru cat 9), format_tag(keep_underscores=) keeps raw booru tokens dedupe.py Advisory near-duplicate detection (perceptual dHash, - numpy-only) surfaced in the ④ export preview; the Hamming - distance (default 5) is a ④ slider + numpy-only) surfaced in the ④ export preview and ②'s "Find + near-duplicates"; the Hamming distance (default 5) is a ④ + slider. find_bursts() groups photos taken seconds apart quality.py Advisory sharpness (blur) + exposure/contrast flags (dark/bright/low-contrast) for the curate/export views caption_lint.py Advisory caption analysis (pure string logic): health lint @@ -115,10 +119,13 @@ studio/ emotion, no outfit). Same Shot model for both, so the ② table, YAML plans and the pipeline need no special case. plan_for_type()/plan_subject() are the single selection seam - shared by UI + CLI (Style → empty plan). Plus apply_wardrobe() - outfit injection + apply_prop_exclusion() + shared by UI + CLI (Style → empty plan). Shot.framing (full / + waist-up / close-up / tight-face); coverage() is ②'s + requested-coverage panel. Plus apply_wardrobe() outfit + injection + apply_prop_exclusion() wardrobe.py Compositional unisex outfit pool for the outfit column - (colour x garment; one pool, no gender picker) + (colour x garment; one pool, no gender picker); dress() is the + identity policy's default dressing plan_io.py Save/load shot plans as YAML (user-editable prompt libraries) dataset_stats.py Inspect a dataset folder (count/sizes/aspects/① provenance) -> target steps, auto repeats + bucket ladder, so ⑤'s numbers are @@ -154,8 +161,11 @@ studio/ cached, force-refreshable image- AND caption-model lists (list_image_models / list_caption_models). Retries transient statuses with backoff (GENERATE_RETRIES / GENERATE_BACKOFF_S) - and reports failures through friendly_api_error() - comfyui.py Local engine (Qwen-Image 2.1, one reference per shot) + and reports failures through friendly_api_error(). + shot_aspect() sets 2:3 / 1:1 per shot; references are capped + at the model's character limit (CHARACTER_REFS) + comfyui.py Local engine (Qwen-Image 2.1): one LoadImage per reference up + to LDS_QWEN21_MAX_REFS, at LDS_QWEN21_RESOLUTION comfy_workflows/*.json API-format ComfyUI graphs (restore, isolate ×2, qwen21 edit) comfy_workflows/legacy/ The pre-0.17 Qwen-Image-Edit-2511 + Multiple-Angles graph, kept for anyone who wants it; nothing loads it (see its README) @@ -921,6 +931,57 @@ quietly gets someone else's value. them. Only photos with provenance qualify (a generated shot is no test of likeness) and at least one image always stays in training. The dataset naming loop skips a name whose `-heldout` sibling exists, so a stale held-out folder never pairs with a new dataset. +- **Training copies keep their background; only ②'s reference is isolated (0.19.0).** Real + photos used to be cut onto white for training too, and a LoRA learns a white void as part of + the subject. ① now writes the training copy with its own background and, with isolation on, + an isolated copy to `prepped/refs/` that ② generates from. `list_images` is non-recursive, so + ③, ④, the ILB hand-off and dataset stats never see `refs/` (a test pins this). ②'s "isolate + generated angle shots" is a separate option, default off, for the same reason. +- **Identity policy decides whether clothing is part of the subject (0.19.0).** "Identity only" + (default) dresses each character shot in a different random outfit and tells the captioner to + describe clothing, so the trigger learns the face and body, not one outfit. "Signature + costume" leaves outfits blank and tells the captioner not to describe the costume, so the + trigger carries it. The clause is appended in `CaptionerSpec.prompt_for` for characters only; + an empty policy keeps the tuned templates verbatim. Recorded in metadata.json. +- **Framing is a Shot field, and coverage counts what was requested (0.19.0).** Two emotions + became tight face crops and two poses waist-up, so a default plan spans four framing tiers. + ②'s coverage panel counts the kept shots by framing, turnaround view, expression and outfit + from the plan's own fields, labelled "requested, not verified" (imported photos count as + unknown). Dominance is checked on the turnaround views and expressions only, and only for a + group of 5 or more: on a coarse view split every character plan is front-heavy by design. +- **The full-body warning is a geometry check, not a model (0.19.0).** A subject mask touching + the bottom 2% of the isolated reference means the legs are off-frame; when every reference is + cut off, ① and the CLI warn that full-body shots will invent them. Without isolation there is + no mask, so no check. +- **Gemini gets a per-shot aspect and at most its character-reference limit (0.19.0).** Standing + turnaround views ask for 2:3 and close-ups for 1:1; poses keep the reference's aspect (a + crouch or a lying pose squeezed into 2:3 loses the body). `image_size` is left at the 1K + default. References stop at 4 (5 for Pro): every reference is the same person, and past the + model's character limit more copies add cost, not likeness. +- **The anchor shot is a chain root for the whole plan (0.19.0).** With "Anchor shot" on, the + front full-body view (`angle-front`) runs first from every reference, then leads each other + shot's references; a chained view still leads its own shot, followed by the anchor and the + originals. It reuses the chain mechanism; a plan without `angle-front` has no anchor. +- **Local ② defaults to one reference at 1536, anchor off (0.19.0 A/B).** RTX 5090, eight of + the hardest character shots (back, front, low and right views, two expressions, seated, + walking) from two poor phone photos: a 3000×4000 one with a spoon's shadow over one eye and a + plate in hand, and a 580 px JPEG cut at the hips. No held-object exclusion, same seeds. Median + ComfyUI time per shot: 11.6 s at 1024 with one reference, 27.2 s at 1536 with one, 48.5 s at + 1536 with two. + - A second reference cost 1.8x and made the set worse: the low-resolution photo pulled the + outfit around (a grey top on the back view) and the plate came back in two shots, not one. + An extra reference helps only when it adds information at comparable quality. + - 1024 is 2.3x faster but redrew the plate in two shots and changed the outfit on the front + view. 1536 follows the reference more closely, so it stays the default. + - The anchor gave the most consistent set (one outfit throughout, no plate anywhere) because + that run's front view came out clean. It copies whatever the front view gets, though: the + 1024 run's front view held the plate, and anchoring on it would have put the plate in every + shot. The eye shadow carried through with or without it. So it stays opt-in and its tooltip + says to check the front view first. + - Re-isolating angle shots would not have removed the leaked plate: SAM3 keeps a held object + as part of the subject. Naming it in ①'s exclude prompt is the fix. + - Neither input can supply what it never showed: everything below the hips and the shadowed + eye are invented or copied, whatever the settings. - **The optimizer defaults to AdamW8bit; Prodigy is the one alternative (0.18.1).** AdamW8bit is the default and the validated recipe in ai-toolkit's UI, musubi's examples, sd-scripts and Fizgig. Prodigy is offered because it removes the learning-rate guess: it runs at lr 1.0 with @@ -1014,16 +1075,16 @@ quietly gets someone else's value. here instead of re-discovering it. **Caveat for anyone changing the pin by hand:** installing Gradio 5 can pull `pydantic` down to a version older than `google-genai`'s floor; reinstall `pydantic>=2.12.5` afterward if `import google.genai` breaks. -- **Alpha cutout is builtin-backend-only and doesn't feed ② Generate.** `isolate_builtin`'s +- **Alpha cutout is builtin-backend-only and is never auto-filled into ②.** `isolate_builtin`'s `alpha_cutout` flag composites the subject onto transparency (straight alpha = the mask) instead of white; `isolate_subject` raises `IsolationError` if asked for alpha cutout on the `comfyui` backend rather than silently returning white or ignoring the flag — the bundled ComfyUI isolation workflows composite via core nodes onto a solid `EmptyImage` and haven't been extended for transparency. `crop_to_content` dispatches on `image.mode` (alpha>0 bounding box for RGBA, non-white heuristic otherwise) so tighten-crop still works - on a cutout. This is a terminal output for the user's own compositing workflow — ② - Generate is unchanged and still expects (and gets, by default) a white-background - reference; turning alpha cutout on and continuing to ② is not supported. + on a cutout. Since 0.19.0 the cutout is the `refs/` copy (the training copy stays RGB with + its background); it is for the user's own compositing, so ① leaves ②'s source folder alone + when alpha is on — ② expects a white-background reference. ## Idiot LoRa Builder hand-off (④, 0.16.0) diff --git a/studio/__init__.py b/studio/__init__.py index 22612cd..fc86838 100644 --- a/studio/__init__.py +++ b/studio/__init__.py @@ -1,3 +1,3 @@ """Dataset Deviser: turn a character, style, or concept into a ready-to-train LoRA dataset.""" -__version__ = "0.18.1" +__version__ = "0.19.0" diff --git a/studio/captioner.py b/studio/captioner.py index 08f149d..a8162b5 100644 --- a/studio/captioner.py +++ b/studio/captioner.py @@ -154,13 +154,14 @@ def __init__(self, key: str, model_override: str = "", def caption(self, image_path: Path, subject: str = "the character", style: str = "prose", dataset_type: str = "character", - sparse: bool = False) -> str: + sparse: bool = False, identity: str = "identity") -> str: if self.spec.backend == "wd_tagger": # A dedicated tagger emits canonical tags directly from image features; # the prose/tags/e621 selector, the subject, and the dataset type (a # framing hint for VLMs) don't apply to it. return ", ".join(self._load_tagger().tag(image_path)) - instruction = self.spec.prompt_for(style, dataset_type, sparse).format(subject=subject) + instruction = self.spec.prompt_for(style, dataset_type, sparse, + identity).format(subject=subject) if self.spec.backend == "openai": return _clean(self._caption_openai(image_path, instruction)) if self.spec.backend == "gemini": @@ -501,6 +502,7 @@ def caption_images( sparse: bool = False, on_item: Callable[[Path, str], None] | None = None, should_stop: Callable[[], bool] | None = None, + identity: str = "identity", ) -> list[tuple[Path, str]]: """Caption a list of images. Standalone — no run/pipeline state needed. @@ -513,6 +515,9 @@ def caption_images( non-empty .txt untouched. Returns (image_path, finalized_caption) pairs; the captioner model is loaded once and freed afterwards. + `identity` is the character identity policy ("identity" captions clothing, + "costume" leaves it to the trigger — see config.IDENTITY_POLICIES). + `on_item(image, caption)` fires as each caption is finalized, so a caller can persist it immediately. That matters for cloud captioners: a 503 on the last image of a batch used to discard every caption already paid for, because @@ -563,7 +568,7 @@ def caption_images( break progress(f"Captioning {i}/{len(images)}: {p.name}") raw = cap.caption(p, subject=subject, style=style, - dataset_type=dataset_type, sparse=sparse) + dataset_type=dataset_type, sparse=sparse, identity=identity) cap_text = finalize_caption(raw, trigger, character_name, SUBJECT_ALIASES, style=style, dataset_type=dataset_type) # Drop noisy tags before affixes so a fixed prefix/suffix survives. @@ -593,6 +598,7 @@ def caption_folder( blacklist: str = "", dataset_type: str = "character", sparse: bool = False, + identity: str = "identity", ) -> list[tuple[Path, str]]: """Caption images in `folder` (all, or the subset in `only`) and write .txt sidecars next to each image. The classic 'point at a folder and tag @@ -610,7 +616,7 @@ def caption_folder( model_override=model_override, spec_overrides=spec_overrides, style=style, prefix=prefix, suffix=suffix, skip_existing=skip_existing, blacklist=blacklist, - dataset_type=dataset_type, sparse=sparse) + dataset_type=dataset_type, sparse=sparse, identity=identity) for img, caption in items: img.with_suffix(".txt").write_text(caption, encoding="utf-8") progress(f"Wrote {len(items)} .txt sidecar(s) in {folder}") diff --git a/studio/config.py b/studio/config.py index 89d6989..c3198f2 100644 --- a/studio/config.py +++ b/studio/config.py @@ -154,6 +154,18 @@ "e621": "Output a single comma-separated list of concise lowercase e621-style tags " "(one attribute per tag), no sentences. Output only the tags.", } +# Character identity policy: what the trigger owns. "identity" = the person only, +# so the varied outfits must be captioned or the trigger absorbs them; "costume" +# = the outfit is part of the character, so it is left to the trigger. +IDENTITY_POLICIES = ("identity", "costume") +_IDENTITY_CLAUSE = { + ("identity", "prose"): " Describe the clothing and the facial expression.", + ("identity", "tags"): " Include tags for the clothing and the facial expression.", + ("costume", "prose"): " Do not describe the clothing or costume; it is part of the " + "subject's fixed look.", + ("costume", "tags"): " Do not tag the clothing or costume; it is part of the " + "subject's fixed look.", +} _PLAINNESS_DIRECTIVE = { "prose": " If the content is explicit, describe it plainly without euphemism.", "tags": " If the content is explicit, tag it plainly without euphemism.", @@ -200,7 +212,7 @@ class CaptionerSpec(BaseModel): cost_note: str = "free" def prompt_for(self, style: str, dataset_type: str = "character", - sparse: bool = False) -> str: + sparse: bool = False, identity: str = "") -> str: """The instruction template for the requested caption style + dataset type. `style="tags"` selects the Danbooru-tag template, `style="e621"` the @@ -214,14 +226,16 @@ def prompt_for(self, style: str, dataset_type: str = "character", format directive, so a Style/Concept dataset is captioned about its content/ context rather than its (trigger-learned) look. `sparse` is a Style-only minimal-caption variant; it is ignored for the other types. + + `identity` (character only) appends the identity-policy clause — see + IDENTITY_POLICIES; "" leaves the template untouched. """ if dataset_type in ("style", "concept"): return self._compose_prompt(style, dataset_type, sparse) - if style == "tags": - return self.tags_template - if style == "e621": - return self.e621_template - return self.prompt_template + template = {"tags": self.tags_template, + "e621": self.e621_template}.get(style, self.prompt_template) + fmt = "prose" if template is self.prompt_template else "tags" + return template + _IDENTITY_CLAUSE.get((identity, fmt), "") def _compose_prompt(self, style: str, dataset_type: str, sparse: bool) -> str: """Framing (per type) + format directive (per style), lightly per-model. @@ -520,6 +534,13 @@ def _current_captioner_key(cls, v: str) -> str: # pointing at the Qwen2.5-VL file would otherwise be fed to the 2.1 graph. qwen21_text_encoder: str = "qwen3vl_8b_int8_convrot.safetensors" qwen21_vae: str = "qwen_image_2.1_vae_bf16.safetensors" + # References per local shot (the node takes up to 16), in order: the chained + # view when there is one, then the primary, then the rest. 1 and 1536 won the + # 0.19.0 A/B (docs/ARCHITECTURE.md). + qwen21_max_refs: int = 1 + # Each reference is resized to about this many pixels squared; the output + # follows the first reference's aspect at about the same size. + qwen21_resolution: int = 1536 upscale_model: str = "4xNomosWebPhoto_RealPLKSR.safetensors" dejpg_model: str = "1xDeJPG_OmniSR.pth" sam3_checkpoint: str = "sam3.1_multiplex_fp16.safetensors" diff --git a/studio/dedupe.py b/studio/dedupe.py index be97eeb..cb765ba 100644 --- a/studio/dedupe.py +++ b/studio/dedupe.py @@ -32,6 +32,36 @@ def hamming(a: int, b: int) -> int: return bin(a ^ b).count("1") +def find_bursts(paths: list[Path], within_s: float = 3.0) -> list[list[Path]]: + """Group real photos whose capture times (① provenance) are `within_s` apart. + + A burst is one moment shot several times: near-identical pose and light that + a pixel hash can miss after a crop or exposure change. + """ + from datetime import datetime + + from PIL import Image + + from studio.dataset_stats import provenance + + stamped: list[tuple[datetime, Path]] = [] + for p in paths: + try: + with Image.open(p) as im: + stamped.append((datetime.strptime(provenance(im)["captured"], + "%Y:%m:%d %H:%M:%S"), p)) + except Exception: + continue # generated shot, no EXIF time, unreadable: not a burst member + stamped.sort() + groups: list[list[Path]] = [] + for i, (t, p) in enumerate(stamped): + if i and (t - stamped[i - 1][0]).total_seconds() <= within_s: + groups[-1].append(p) + else: + groups.append([p]) + return [g for g in groups if len(g) > 1] + + def find_near_duplicate_groups( paths: list[Path], max_distance: int = 5 ) -> list[list[Path]]: diff --git a/studio/engines/comfyui.py b/studio/engines/comfyui.py index b933669..7767668 100644 --- a/studio/engines/comfyui.py +++ b/studio/engines/comfyui.py @@ -7,20 +7,25 @@ import httpx from studio import comfy_api +from studio.config import settings from studio.engines.base import GenerationError from studio.shotplan import Shot +# TextEncodeQwenImage21's `images` socket grows to this many slots. +MAX_REFS = 16 + class ComfyUIEngine: name = "comfyui" - def __init__(self) -> None: + def __init__(self, front: bool = False) -> None: up, reason = comfy_api.server_status() if not up: raise GenerationError( f"ComfyUI is not reachable — {reason}. The local engine needs it " f"(see docs/comfyui-setup.md), or switch the engine to Cloud (Gemini)." ) + self.front = front self._uploaded: dict[Path, str] = {} def _source_name(self, source: Path) -> str: @@ -29,11 +34,17 @@ def _source_name(self, source: Path) -> str: return self._uploaded[source] def generate(self, sources: list[Path], shot: Shot, out_path: Path, seed: int) -> Path: - # One reference per shot; use the primary (first) source. Qwen-Image 2.1 - # accepts up to 10, but that is untested here (see ARCHITECTURE.md). + # The first reference sets the output's aspect, so callers order them: + # chained view, primary, extras. LDS_QWEN21_MAX_REFS caps how many go. + refs = list(dict.fromkeys(sources))[:max(1, min(settings.qwen21_max_refs, MAX_REFS))] graph = comfy_api.load_template("qwen21_edit") graph["5"]["inputs"]["prompt"] = shot.local_prompt + graph["5"]["inputs"]["resolution"] = settings.qwen21_resolution graph["6"]["inputs"]["seed"] = seed + for i in range(2, len(refs) + 1): + graph[str(100 + i)] = {"class_type": "LoadImage", "inputs": {}} + graph["5"]["inputs"][f"images.image_{i}"] = [str(100 + i), 0] + loaders = ["1"] + [str(100 + i) for i in range(2, len(refs) + 1)] last_err: Exception | None = None for _ in range(2): @@ -41,9 +52,10 @@ def generate(self, sources: list[Path], shot: Shot, out_path: Path, seed: int) - # Upload inside the try: ComfyUI dying mid-batch (OOM restart) # surfaces as a raw httpx error from here, from /history polling # or from /view — all of which must fail ONE shot, not the run. - graph["1"]["inputs"]["image"] = self._source_name(sources[0]) - refs = comfy_api.run_prompt(graph, timeout=600) - return comfy_api.fetch_image(refs[0], out_path) + for node, ref in zip(loaders, refs, strict=True): + graph[node]["inputs"]["image"] = self._source_name(ref) + images = comfy_api.run_prompt(graph, timeout=600, front=self.front) + return comfy_api.fetch_image(images[0], out_path) except (comfy_api.ComfyError, httpx.HTTPError) as e: last_err = e graph["6"]["inputs"]["seed"] = seed + 1 diff --git a/studio/engines/gemini.py b/studio/engines/gemini.py index 8527b71..a9a4a18 100644 --- a/studio/engines/gemini.py +++ b/studio/engines/gemini.py @@ -27,7 +27,10 @@ from studio.engines.base import GenerationError from studio.shotplan import Shot -MAX_REFERENCE_IMAGES = 14 +# Every reference shows the same character, and these models keep at most this +# many people consistent (Pro: 5, the rest: 4). More copies add cost, not likeness. +CHARACTER_REFS = {"gemini-3-pro-image": 5} +DEFAULT_CHARACTER_REFS = 4 # References are capped here before upload. ① already outputs 1024 px, but ② # also takes files straight from the user (a pasted 12 MP photo), and every # current Gemini image model works at 1-2K. @@ -247,6 +250,17 @@ def _plain(ids: list[str]) -> list[tuple[str, str]]: return _plain(_CAPTION_FALLBACK) +def shot_aspect(shot: Shot) -> str | None: + """Output aspect for a shot: portrait for a standing turnaround view, square + for a close-up. None (the model follows the reference) for anything else — + a lying or crouching pose would be squeezed by a fixed portrait frame.""" + if shot.framing in ("close-up", "tight-face"): + return "1:1" + if shot.kind == "angle" and shot.framing == "full": + return "2:3" + return None + + class GeminiEngine: name = "gemini" @@ -267,10 +281,13 @@ def generate(self, sources: list[Path], shot: Shot, out_path: Path, seed: int) - from google.genai import types parts: list = [] - for p in sources[:MAX_REFERENCE_IMAGES]: + cap = CHARACTER_REFS.get(self._model, DEFAULT_CHARACTER_REFS) + for p in list(dict.fromkeys(sources))[:cap]: data, mime = api_image(p, REFERENCE_MAX_SIDE) parts.append(types.Part.from_bytes(data=data, mime_type=mime)) parts.append(shot.cloud_prompt) + aspect = shot_aspect(shot) + image_config = types.ImageConfig(aspect_ratio=aspect) if aspect else None last_err: Exception | None = None for attempt in range(GENERATE_RETRIES): @@ -280,6 +297,7 @@ def generate(self, sources: list[Path], shot: Shot, out_path: Path, seed: int) - contents=parts, config=types.GenerateContentConfig( response_modalities=["TEXT", "IMAGE"], + image_config=image_config, ), ) for cand in resp.candidates or []: diff --git a/studio/isolate.py b/studio/isolate.py index 41bf92a..bd737f7 100644 --- a/studio/isolate.py +++ b/studio/isolate.py @@ -13,6 +13,7 @@ from __future__ import annotations +import os from collections.abc import Callable from pathlib import Path @@ -46,14 +47,23 @@ def _load_sam3(): # device rather than handed to accelerate's "auto" — that sharded it # across cuda:0/cuda:1 on multi-GPU machines and the forward pass # died with a two-device tensor mismatch. - _model = Sam3Model.from_pretrained(settings.sam3_hf_id).to(config.torch_device()) - _processor = Sam3Processor.from_pretrained(settings.sam3_hf_id) + # Both or neither: a processor that fails after the model loaded + # (a gated-repo 401 on one file) must not leave a half-cached pair + # that the next image calls as None. + model = Sam3Model.from_pretrained(settings.sam3_hf_id).to(config.torch_device()) + processor = Sam3Processor.from_pretrained(settings.sam3_hf_id) except Exception as e: + # An HF_TOKEN (from .env or the environment) wins over `hf auth login`, + # so a stale one 401s even when the saved login would work. + hint = (" An HF_TOKEN is set (in .env or the environment) and is used " + "instead of `hf auth login` — replace it (`python cli.py keys --set " + "HF_TOKEN`) or remove it." if os.environ.get("HF_TOKEN") else "") raise IsolationError( f"Could not load {settings.sam3_hf_id} — it is a gated model: accept the " f"license at https://huggingface.co/{settings.sam3_hf_id} and authenticate " - f"(`hf auth login` or set HF_TOKEN). Original error: {e}" + f"(`hf auth login` or set HF_TOKEN).{hint} Original error: {e}" ) from e + _model, _processor = model, processor return _model, _processor @@ -183,6 +193,22 @@ def isolate_builtin(image_path: Path, out_path: Path, subject_prompt: str = "cha return out_path +def _subject_mask(image: Image.Image, bg_tolerance: int = 8) -> np.ndarray: + """Subject pixels of an isolated image: alpha > 0, else anything darker than white.""" + if image.mode == "RGBA": + return np.asarray(image.getchannel("A")) > 0 + arr = np.asarray(image.convert("RGB")).astype(np.int16) + return np.any(255 - arr > bg_tolerance, axis=-1) + + +def touches_bottom(image: Image.Image, frac: float = 0.02) -> bool: + """True when the isolated subject reaches the bottom `frac` of the frame — + the photo cut it off, so a full-body shot has to invent what's missing.""" + mask = _subject_mask(image) + edge = max(1, round(image.height * frac)) + return bool(mask[-edge:].any()) + + def crop_to_content(image: Image.Image, bg_tolerance: int = 8, margin_frac: float = 0.02) -> Image.Image: """Crop an isolated (subject-on-white, or subject-with-alpha) image to the @@ -195,11 +221,7 @@ def crop_to_content(image: Image.Image, bg_tolerance: int = 8, A small margin (fraction of the long side) is left around the subject. If the image is effectively empty (nothing found), it is returned unchanged. """ - if image.mode == "RGBA": - nonbg = np.asarray(image.getchannel("A")) > 0 - else: - arr = np.asarray(image.convert("RGB")).astype(np.int16) - nonbg = np.any(255 - arr > bg_tolerance, axis=-1) # anything darker than white + nonbg = _subject_mask(image, bg_tolerance) if not nonbg.any(): return image rows = np.where(nonbg.any(axis=1))[0] diff --git a/studio/pipeline.py b/studio/pipeline.py index 1177448..7568ef1 100644 --- a/studio/pipeline.py +++ b/studio/pipeline.py @@ -17,10 +17,11 @@ from studio.engines.base import GenerationError from studio.jobs import ShouldStop, should_stop_now from studio.package import slugify -from studio.preprocess import PreprocessReport, failed_report, preprocess +from studio.preprocess import REFS_DIR, PreprocessReport, failed_report, preprocess from studio.shotplan import Shot, apply_prop_exclusion, apply_wardrobe ProgressFn = Callable[[str], None] +ANCHOR_SHOT = "angle-front" @dataclass @@ -38,11 +39,11 @@ def new_run_dir(name: str = "") -> Path: return run_dir -def make_engine(engine_key: str, cloud_model: str = ""): +def make_engine(engine_key: str, cloud_model: str = "", front: bool = False): if engine_key == "comfyui": from studio.engines.comfyui import ComfyUIEngine - return ComfyUIEngine() + return ComfyUIEngine(front=front) from studio.engines.gemini import GeminiEngine return GeminiEngine(model=cloud_model) @@ -88,7 +89,7 @@ def preprocess_sources( progress(f" SKIPPED {src.name}: {e}") reports.append(failed_report(src, str(e))) continue - extra = ", subject isolated" if rep.isolated else "" + extra = f", isolated reference in {REFS_DIR}/" if rep.reference else "" progress( f" {src.name}: {rep.original_size[0]}x{rep.original_size[1]} -> " f"{rep.final_size[0]}x{rep.final_size[1]} ({rep.reason}{extra})" @@ -115,6 +116,7 @@ def generate_shots( only_ids: set[str] | None = None, exclude_props: bool = True, front: bool = False, + anchor: bool = False, should_stop: ShouldStop | None = None, progress: ProgressFn = print, ) -> list[GenResult]: @@ -125,16 +127,23 @@ def generate_shots( `exclude_props` asks the generator to omit bags/held objects carried in the reference, so they don't end up baked into every dataset image. + + `anchor` generates the front full-body view first and leads every other + shot's references with it: partial or poor sources (a face in shadow, a + body cut at the hips) then give way to one clean, complete view to copy. """ if not sources: raise GenerationError("No reference images given — nothing to generate from.") - engine = make_engine(engine_key, cloud_model) + engine = make_engine(engine_key, cloud_model, front=front) out_dir.mkdir(parents=True, exist_ok=True) results = [r for r in (existing or []) if only_ids is None or r.shot.id not in only_ids] todo = [s for s in shots if only_ids is None or s.id in only_ids] - # Chained shots (e.g. back views built from a generated side view) run last - todo.sort(key=lambda s: bool(s.chain_from)) + # A plan without the front view (a concept, or the row deleted) has no anchor. + anchor_id = ANCHOR_SHOT if anchor else None + # The anchor runs first; chained shots (back views built from a generated + # side view) run last. + todo.sort(key=lambda s: (s.id != anchor_id, bool(s.chain_from))) done: dict[str, Path] = {r.shot.id: r.path for r in results if r.path} stopped_at = 0 @@ -154,10 +163,12 @@ def generate_shots( seed = random.randint(0, 2**48) out = out_dir / f"{shot.id}.png" shot_sources = sources + if anchor_id in done and shot.id != anchor_id: + shot_sources = [done[anchor_id], *sources] if shot.chain_from and shot.chain_from in done: # Lead with the chained view so single-reference engines rotate # stepwise instead of hallucinating the far side of the character. - shot_sources = [done[shot.chain_from], *sources] + shot_sources = [done[shot.chain_from], *shot_sources] progress(f"[{i}/{len(todo)}] {shot.id} ({engine_key})...") try: engine.generate(shot_sources, shot, out, seed) diff --git a/studio/preprocess.py b/studio/preprocess.py index e3c6423..bc3a05c 100644 --- a/studio/preprocess.py +++ b/studio/preprocess.py @@ -23,6 +23,8 @@ from studio.isolate import isolate_subject BLUR_THRESHOLD = 120.0 # Laplacian variance below this = soft/degraded image +# Subfolder of ①'s output holding the isolated copies ② uses as references. +REFS_DIR = "refs" @dataclass @@ -35,6 +37,10 @@ class PreprocessReport: restored: bool reason: str isolated: bool = False + # The isolated copy in `refs/` — ②'s reference, never a training image. + reference: Path | None = None + # The isolated subject runs off the bottom edge (needs isolation to know). + cut_off: bool = False # Empty on success. When set, this source was skipped and nothing was # written for it; the batch carried on with the remaining images. error: str = "" @@ -146,14 +152,15 @@ def preprocess( """Copy + clean one source image into `work_dir` at target resolution. force_restore: True = always restore, False = never, None = auto-decide. - isolate: cut out the subject so backgrounds and props don't leak into - generations or the dataset. - tighten_crop: after isolation, crop to the subject's bounding box (less white + isolate: also write the subject cut out onto white to `refs/`, as ②'s + reference, so backgrounds and props don't leak into generations. The + training copy in `work_dir` always keeps its background. + tighten_crop: crop the isolated copy to the subject's bounding box (less white padding, more consistent framing). No effect unless `isolate` is on. - alpha_cutout: export the isolated subject on a transparent background instead - of white (builtin SAM3 backend only — see `isolate_subject`). No effect - unless `isolate` is on. This is a terminal output for the caller's own - compositing workflow, not a new reference format for ② Generate. + alpha_cutout: the isolated copy goes on a transparent background instead of + white (builtin SAM3 backend only — see `isolate_subject`). No effect unless + `isolate` is on. A terminal output for the caller's own compositing + workflow, not a reference format for ② Generate. """ target = target or settings.target_long_side restore_backend = restore_backend or settings.restore_backend @@ -178,16 +185,19 @@ def preprocess( # so two sources sharing a stem (e.g. cat.jpg + cat.png, in one folder or # across merged inputs) would otherwise both map to `cat_prepped.png` and the # second would silently overwrite the first — quietly dropping an image. + refs_dir = work_dir / REFS_DIR out_path = work_dir / f"{source.stem}_prepped.png" n = 2 - while out_path.exists(): + while out_path.exists() or (refs_dir / out_path.name).exists(): out_path = work_dir / f"{source.stem}_prepped_{n}.png" n += 1 - # Every write below lands on `out_path`, and restoration writes it BEFORE - # isolation runs. A failure after that point used to leave the restored (not - # isolated, not resized) image behind, where `list_images` happily served it - # to ②/③ as a finished source — a silent half-processed file in the dataset. - # The stage is therefore atomic: complete output, or none at all. + ref_path = refs_dir / out_path.name if isolate else None + cut_off = False + # Restoration writes `out_path` BEFORE isolation runs. A failure after that + # point used to leave the restored (not isolated, not resized) image behind, + # where `list_images` happily served it to ②/③ as a finished source — a + # silent half-processed file in the dataset. The stage is therefore atomic: + # complete output, or none at all. # 2x the target is all a resize needs, but tighten-crop keeps only the # subject's box: shrinking first would upscale a small subject back up. staged = _stage_copy(oriented, rotated, @@ -208,29 +218,40 @@ def preprocess( # damage stays (note it so the user knows what they're getting). reason += " (basic Lanczos only — ComfyUI restore not used)" - if isolate: - isolate_subject(stage_path, out_path, subject_prompt, exclude_prompt, + def save(img: Image.Image, path: Path, isolated: bool) -> None: + info = PngInfo() + info.add_text(PROVENANCE_KEY, json.dumps({ + "name": source.name, "w": original_size[0], "h": original_size[1], + "captured": captured, "restored": restore, "isolated": isolated})) + img.save(path, "PNG", pnginfo=info) + + if ref_path: + # The isolated copy is ②'s reference only: it keeps props and the + # old background out of the generated shots. + refs_dir.mkdir(exist_ok=True) + isolate_subject(stage_path, ref_path, subject_prompt, exclude_prompt, backend=isolation_backend, progress=progress, alpha_cutout=alpha_cutout, label=source.name, front=front) - stage_path = out_path - - if isolate and alpha_cutout: - img = Image.open(stage_path) # keep RGBA — no forced flatten - else: - img = Image.open(stage_path).convert("RGB") - if isolate and tighten_crop: - # Crop the subject-on-white composite to its bounding box before resizing. - from studio.isolate import crop_to_content - - img = crop_to_content(img) - img = _resize_to_target(img, target) - info = PngInfo() - info.add_text(PROVENANCE_KEY, json.dumps({ - "name": source.name, "w": original_size[0], "h": original_size[1], - "captured": captured, "restored": restore, "isolated": isolate})) - img.save(out_path, "PNG", pnginfo=info) + with Image.open(ref_path) as im: + ref = im.copy() if alpha_cutout else im.convert("RGB") # keep RGBA + from studio.isolate import touches_bottom + + cut_off = touches_bottom(ref) + if tighten_crop: + # Crop the subject-on-white composite to its bounding box before resizing. + from studio.isolate import crop_to_content + + ref = crop_to_content(ref) + save(_resize_to_target(ref, target), ref_path, True) + # The training copy keeps its background: a dataset of subjects on white + # teaches the LoRA white backgrounds and cut-out edges. + with Image.open(stage_path) as im: + img = _resize_to_target(im.convert("RGB"), target) + save(img, out_path, False) except BaseException: out_path.unlink(missing_ok=True) + if ref_path: + ref_path.unlink(missing_ok=True) raise finally: if staged: @@ -243,4 +264,6 @@ def preprocess( restored=restore, reason=reason or "clean source, resize only", isolated=isolate, + reference=ref_path, + cut_off=cut_off, ) diff --git a/studio/shotplan.py b/studio/shotplan.py index cee51bb..4445c05 100644 --- a/studio/shotplan.py +++ b/studio/shotplan.py @@ -46,6 +46,16 @@ class Shot(BaseModel): # (no identity drift). When set, "wearing {outfit}" is injected into both # the local and cloud prompts so clothing can vary across the dataset. outfit: str = "" + # "full" | "waist-up" | "close-up" | "tight-face"; "" = unknown (concept + # plans, older saved plans). Requested, not verified — the generator decides. + framing: str = "" + + +# Framing per kind, with the rows that break the pattern. Tight face crops carry +# the most likeness signal; waist-up fills the gap between full body and close-up. +_KIND_FRAMING = {"angle": "full", "pose": "full", "emotion": "close-up"} +_FRAMING = {"pose-arms-raised": "waist-up", "pose-looking-back": "waist-up", + "emotion-smiling": "tight-face", "emotion-confident": "tight-face"} # Each tuple is: (id_suffix, kind, local phrase for Qwen-Image 2.1, plain-English @@ -57,6 +67,8 @@ class Shot(BaseModel): # - 8 poses: each pose is paired with a setting and emotion; lighting is part # of the setting rather than a separate repeated pose. # - 7 emotions: close-up expression shots with varied angles and settings. +# - Framing tiers: two poses are waist-up and two close-ups are tight face crops +# (see _FRAMING), so the set isn't only full body plus one close-up distance. # - Total: 24 shots (down from 28) while improving per-shot diversity. # # Settings are written as "natural" lighting/environment phrases so both the @@ -184,8 +196,8 @@ class Shot(BaseModel): ( "arms-raised", "pose", - "with both arms raised overhead", - "with both arms raised overhead", + "framed from the waist up, with both arms raised overhead", + "framed from the waist up, with both arms raised overhead", "", "triumphant", "lit by dramatic hard side lighting against a dark background", @@ -211,8 +223,8 @@ class Shot(BaseModel): ( "looking-back", "pose", - "standing and looking back over one shoulder", - "standing and looking back over one shoulder", + "framed from the waist up, looking back over one shoulder", + "framed from the waist up, looking back over one shoulder", "", "playful", "outdoors at golden hour with warm backlighting", @@ -221,8 +233,8 @@ class Shot(BaseModel): ( "smiling", "emotion", - "a close-up of the face and upper shoulders, smiling warmly", - "a close-up of the face and upper shoulders, smiling warmly", + "a tight close-up of the face filling the frame, smiling warmly", + "a tight close-up of the face filling the frame, smiling warmly", "", "smiling", "against a soft neutral studio background", @@ -266,8 +278,8 @@ class Shot(BaseModel): ( "confident", "emotion", - "a close-up of the face and upper shoulders, with a confident look and a slight smirk", - "a close-up of the face and upper shoulders, confident expression", + "a tight close-up of the face filling the frame, with a confident look and a slight smirk", + "a tight close-up of the face filling the frame, confident expression", "", "confident", "outdoors at golden hour with warm backlighting", @@ -606,11 +618,59 @@ def default_plan(subject: str = "the character", emotion=emotion, setting=setting, outfit="", + framing=_FRAMING.get(f"{kind}-{suffix}", _KIND_FRAMING[kind]), ) ) return shots +FRAMING_TIERS = ("full", "waist-up", "close-up", "tight-face") +# Turnaround direction of an angle shot, by id suffix. +_VIEW = {"front": "front", "front-left": "front", "front-right": "front", + "left": "side", "right": "side", "back": "back", "back-left": "back", + "back-right": "back", "low": "low"} + + +def coverage(shots: list[Shot], threshold: float = 0.4) -> str: + """Markdown summary of what a kept set covers — from the plan, not the pixels. + + Warns on an empty framing tier or turnaround direction, and on any view or + expression above `threshold` of its group: the LoRA leans toward whatever + dominates the set. + """ + if not shots: + return "" + from collections import Counter + + framing = Counter(s.framing or "unknown" for s in shots) + views = Counter(_VIEW.get(s.id.removeprefix("angle-"), "other") + for s in shots if s.kind == "angle") + moods = Counter(s.emotion or "neutral" for s in shots) + dressed = sum(1 for s in shots if s.outfit) + + def line(name: str, counts: Counter) -> str: + top = counts.most_common(4) + rest = sum(counts.values()) - sum(n for _, n in top) + more = f", {rest} across {len(counts) - 4} more" if rest else "" + return f"**{name}:** " + ", ".join(f"{k} {n}" for k, n in top) + more + + lines = [f"**Coverage of {len(shots)} kept shot(s)** — requested, not verified", + line("Framing", framing), line("Turnaround", views) if views else "", + line("Expression", moods), f"**Outfit:** {dressed} varied, " + f"{len(shots) - dressed} as in the reference"] + warn = [f"no {t} shots" for t in FRAMING_TIERS if any(s.framing for s in shots) + and not framing[t]] + if views: + warn += [f"no {v} views" for v in ("front", "side", "back") if not views[v]] + for name, counts in (("view", views), ("expression", moods)): + total = sum(counts.values()) + warn += [f"{name} '{k}' {n}/{total} = {n / total:.0%}" + for k, n in counts.items() if total >= 5 and n / total > threshold] + if warn: + lines.append("⚠️ " + "; ".join(warn)) + return " \n".join(x for x in lines if x) + + def plan_subject(name: str, dataset_type: str = "character") -> str: """The noun phrase woven into the ② prompts for a dataset type. diff --git a/studio/wardrobe.py b/studio/wardrobe.py index 750b652..38ab9f6 100644 --- a/studio/wardrobe.py +++ b/studio/wardrobe.py @@ -87,3 +87,11 @@ def random_outfits(n: int, seed: int | None = None) -> list[str]: # frame; describing a full outfit there invites the model to widen the shot and # defeat the close-up. Wardrobe variety is carried by the angle/pose shots. OUTFIT_SHOT_KINDS = ("angle", "pose") + + +def dress(shots: list) -> list: + """`shots` with distinct random outfits on the angle/pose rows.""" + targets = [s.id for s in shots if s.kind in OUTFIT_SHOT_KINDS] + outfits = dict(zip(targets, random_outfits(len(targets)), strict=True)) + return [s.model_copy(update={"outfit": outfits[s.id]}) if s.id in outfits else s + for s in shots] diff --git a/tests/test_caption_style.py b/tests/test_caption_style.py index 1158d68..a0ae7cb 100644 --- a/tests/test_caption_style.py +++ b/tests/test_caption_style.py @@ -142,7 +142,7 @@ def _stub_model(monkeypatch, text: str) -> None: monkeypatch.setattr( C.Captioner, "caption", lambda self, image_path, subject="the character", style="prose", - dataset_type="character", sparse=False: text) + dataset_type="character", sparse=False, identity="identity": text) monkeypatch.setattr(C.Captioner, "load", lambda self: None) monkeypatch.setattr(C.Captioner, "unload", lambda self: None) diff --git a/tests/test_cli_build_types.py b/tests/test_cli_build_types.py index 21183bc..4f88ce2 100644 --- a/tests/test_cli_build_types.py +++ b/tests/test_cli_build_types.py @@ -34,14 +34,17 @@ def fake_preprocess(sources, out_dir, **kw): out_dir.mkdir(parents=True, exist_ok=True) prepped = out_dir / "ref_prepped.png" prepped.write_bytes(b"x") + ref = out_dir / "refs" / prepped.name if kw.get("isolate") else None return [PreprocessReport(source=sources[0], output=prepped, original_size=(1, 1), final_size=(1, 1), - restored=False, reason="ok")] + restored=False, reason="ok", reference=ref)] def fake_generate(sources, shots, engine, out_dir, **kw): calls["generated"] += 1 + calls["sources"] = list(sources) calls["shots"] = shots calls["exclude_props"] = kw.get("exclude_props") + calls["isolate_angles"] = kw.get("isolate_angles") out_dir.mkdir(parents=True, exist_ok=True) results = [] for shot in shots: @@ -111,6 +114,16 @@ def test_character_build_is_unchanged(stubs: dict, tmp_path: Path) -> None: assert len(stubs["metadata"]["shots"]) == 24 +def test_build_generates_from_refs_and_captions_the_training_copy( + stubs: dict, tmp_path: Path) -> None: + _build(stubs, tmp_path) + assert [p.parent.name for p in stubs["sources"]] == ["refs"] + assert stubs["captioned"][0].parent.name == "prepped" + assert stubs["isolate_angles"] is False + _build(stubs, tmp_path, "--isolate-angles") + assert stubs["isolate_angles"] is True + + def test_explicit_flags_beat_the_type_defaults(stubs: dict, tmp_path: Path) -> None: _build(stubs, tmp_path, "--dataset-type", "concept", "--exclude-props", "--no-isolate") diff --git a/tests/test_comfy_engine.py b/tests/test_comfy_engine.py index beec759..d25e9c0 100644 --- a/tests/test_comfy_engine.py +++ b/tests/test_comfy_engine.py @@ -53,6 +53,37 @@ def run_prompt(graph: dict, timeout: float = 0, **_: object) -> list[dict]: assert seen["2"]["inputs"]["unet_name"].endswith(".safetensors") +def test_multi_reference_wires_one_loader_per_reference_in_order( + offline: None, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from studio.config import settings + + seen: dict = {} + monkeypatch.setattr(comfy_api, "upload_image", lambda path: f"up-{path.name}") + + def run_prompt(graph: dict, timeout: float = 0, front: bool = False) -> list[dict]: + seen.update(graph=graph, front=front) + return [{"filename": "o.png"}] + + monkeypatch.setattr(comfy_api, "run_prompt", run_prompt) + monkeypatch.setattr(comfy_api, "fetch_image", lambda ref, out: out) + monkeypatch.setattr(settings, "qwen21_max_refs", 3) + monkeypatch.setattr(settings, "qwen21_resolution", 1024) + refs = [] + for name in ("chain", "primary", "extra", "spare"): + refs.append(tmp_path / f"{name}.png") + refs[-1].write_bytes(b"x") + + ComfyUIEngine(front=True).generate(refs + [refs[1]], default_plan()[0], + tmp_path / "o.png", seed=1) + + g, enc = seen["graph"], seen["graph"]["5"]["inputs"] + loaded = [g[enc[f"images.image_{i}"][0]]["inputs"]["image"] for i in (1, 2, 3)] + assert loaded == ["up-chain.png", "up-primary.png", "up-extra.png"] + assert "images.image_4" not in enc and enc["resolution"] == 1024 + assert seen["front"] is True + + @pytest.mark.parametrize("stage", ["upload_image", "run_prompt", "fetch_image"]) def test_a_dead_comfyui_fails_the_shot_not_the_run( stage: str, offline: None, monkeypatch: pytest.MonkeyPatch, tmp_path: Path diff --git a/tests/test_coverage_0_19_0.py b/tests/test_coverage_0_19_0.py new file mode 100644 index 0000000..ab12699 --- /dev/null +++ b/tests/test_coverage_0_19_0.py @@ -0,0 +1,203 @@ +"""0.19.0: framing tiers, the ② coverage line, and the cut-off reference warning.""" + +from __future__ import annotations + +from collections import Counter +from pathlib import Path + +import pytest +from PIL import Image + +import app as A +from studio import pipeline +from studio.isolate import touches_bottom +from studio.preprocess import PreprocessReport +from studio.shotplan import concept_plan, coverage, default_plan + + +def test_the_character_plan_has_every_framing_tier() -> None: + plan = default_plan() + assert len(plan) == 24 + assert Counter(s.framing for s in plan) == { + "full": 15, "close-up": 5, "waist-up": 2, "tight-face": 2} + tight = next(s for s in plan if s.framing == "tight-face") + assert "tight close-up of the face" in tight.local_prompt + # Framing survives the ② table round trip. + assert [s.framing for s in A._df_to_shots(A._shots_to_df(plan))] == \ + [s.framing for s in plan] + assert not any(s.framing for s in concept_plan()) + + +def test_coverage_warns_on_gaps_and_dominance() -> None: + plan = default_plan() + assert "⚠️" not in coverage(plan) + culled = [s for s in plan if s.kind != "emotion" and "back" not in s.id] + note = coverage(culled) + assert "no close-up shots" in note and "no back views" in note + assert "view 'front' 3/6 = 50%" in note + assert "view 'front'" not in coverage(culled, threshold=0.6) + assert coverage([]) == "" + + +def test_coverage_note_counts_only_kept_shots_that_exist() -> None: + plan = default_plan() + results = [A.pipeline.GenResult(s, Path(f"{s.id}.png"), seed=1) for s in plan] + results[0] = A.pipeline.GenResult(plan[0], None, seed=1, error="boom") + note = A.coverage_note(results, [s.id for s in plan[:5]], 40) + assert note.startswith("**Coverage of 4 kept shot(s)**") + + +def _noise(path: Path, seed: int) -> Path: + import numpy as np + + rng = np.random.default_rng(seed) + Image.fromarray(rng.integers(0, 255, (64, 64, 3), dtype=np.uint8)).save(path) + return path + + +def test_find_gen_duplicates_names_reference_copies_and_twins(tmp_path: Path) -> None: + ref = _noise(tmp_path / "ref.png", 1) + plan = default_plan()[:4] + paths = [tmp_path / f"{s.id}.png" for s in plan] + for p, seed in zip(paths, (1, 2, 2, 3), strict=True): # copy, twin, twin, unique + _noise(p, seed) + results = [A.pipeline.GenResult(s, p, seed=1) for s, p in zip(plan, paths, strict=True)] + note = A.find_gen_duplicates(results, [s.id for s in plan], [str(ref)], "", 5) + assert "1 shot group(s) look like a copy of a reference" in note + assert f"{plan[0].id}.png" in note + assert "1 near-duplicate group(s)" in note + assert A.primary_reference([str(ref)], "") == str(ref) + assert A.primary_reference([], "") is None + + +def test_bursts_group_photos_taken_seconds_apart(tmp_path: Path) -> None: + import json + + from PIL.PngImagePlugin import PngInfo + + from studio.dataset_stats import PROVENANCE_KEY + from studio.dedupe import find_bursts + + def photo(name: str, captured: str) -> Path: + info = PngInfo() + info.add_text(PROVENANCE_KEY, json.dumps({"name": name, "captured": captured})) + Image.new("RGB", (8, 8)).save(tmp_path / name, pnginfo=info) + return tmp_path / name + + shots = [photo("a.png", "2026:05:01 10:00:00"), photo("b.png", "2026:05:01 10:00:02"), + photo("c.png", "2026:05:01 10:05:00"), photo("d.png", "")] + assert find_bursts(shots) == [shots[:2]] + + +def test_touches_bottom_on_white_and_alpha() -> None: + img = Image.new("RGB", (100, 100), "white") + img.paste((0, 0, 0), (40, 10, 60, 90)) + assert not touches_bottom(img) + img.paste((0, 0, 0), (40, 10, 60, 100)) + assert touches_bottom(img) + rgba = Image.new("RGBA", (100, 100), (0, 0, 0, 0)) + rgba.paste((9, 9, 9, 255), (40, 50, 60, 100)) + assert touches_bottom(rgba) + + +def test_preprocess_flags_a_cut_off_reference(tmp_path: Path, monkeypatch) -> None: + import studio.preprocess as pp + + src = tmp_path / "a.png" + Image.new("RGB", (64, 64), (90, 90, 90)).save(src) + + def fake_isolate(image_path, out_path, *a, **kw): + img = Image.new("RGB", (64, 64), "white") + img.paste((0, 0, 0), (20, 10, 40, 64)) # legs run off the bottom + img.save(out_path) + return out_path + + monkeypatch.setattr(pp, "isolate_subject", fake_isolate) + r = pp.preprocess(src, tmp_path / "out", target=64, force_restore=False, isolate=True, + tighten_crop=True) + assert r.cut_off + note = A._preprocess_note([r], tmp_path / "out", False) + assert "Every reference is cut off" in note + whole = PreprocessReport(source=Path("b.png"), output=Path("b.png"), original_size=(1, 1), + final_size=(1, 1), restored=False, reason="ok") + assert "other reference(s)" in A._preprocess_note([r, whole], tmp_path / "out", False) + + +def test_anchor_runs_first_and_leads_every_other_shots_references( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + + seen: dict[str, list[str]] = {} + + class FakeEngine: + def generate(self, sources, shot, out_path, seed): + seen[shot.id] = [p.name for p in sources] + out_path.write_bytes(b"x") + return out_path + + monkeypatch.setattr(pipeline, "make_engine", lambda *a, **kw: FakeEngine()) + src = tmp_path / "ref.png" + src.write_bytes(b"x") + plan = default_plan() + plan = plan[1:] + plan[:1] # the anchor runs first wherever the plan lists it + + pipeline.generate_shots([src], plan, "comfyui", tmp_path / "gen", anchor=True, + progress=lambda _m: None) + + assert next(iter(seen)) == "angle-front" and seen["angle-front"] == ["ref.png"] + assert seen["emotion-sad"] == ["angle-front.png", "ref.png"] + # A chained view still leads; the anchor and the original follow it. + assert seen["angle-back"] == ["angle-right.png", "angle-front.png", "ref.png"] + + +def test_without_anchor_shots_see_only_the_sources( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + + seen: dict[str, list[str]] = {} + + class FakeEngine: + def generate(self, sources, shot, out_path, seed): + seen[shot.id] = [p.name for p in sources] + out_path.write_bytes(b"x") + return out_path + + monkeypatch.setattr(pipeline, "make_engine", lambda *a, **kw: FakeEngine()) + src = tmp_path / "ref.png" + src.write_bytes(b"x") + pipeline.generate_shots([src], default_plan(), "comfyui", tmp_path / "gen", + progress=lambda _m: None) + assert seen["emotion-sad"] == ["ref.png"] + assert seen["angle-back"] == ["angle-right.png", "ref.png"] + + +def test_a_failed_processor_load_does_not_cache_a_half_loaded_sam3( + monkeypatch: pytest.MonkeyPatch) -> None: + """The model loaded, the processor 401'd: the next image must get the same + clear IsolationError, not "'NoneType' object is not callable".""" + import sys + import types + + from studio import isolate + + class Model: + @classmethod + def from_pretrained(cls, _id): + return cls() + + def to(self, _device): + return self + + class Processor: + @classmethod + def from_pretrained(cls, _id): + raise OSError("401 gated") + + fake = types.ModuleType("transformers") + fake.Sam3Model, fake.Sam3Processor = Model, Processor + monkeypatch.setitem(sys.modules, "transformers", fake) + monkeypatch.setattr(isolate, "_model", None) + monkeypatch.setattr(isolate, "_processor", None) + monkeypatch.setenv("HF_TOKEN", "stale") + + for _ in range(2): + with pytest.raises(isolate.IsolationError, match="HF_TOKEN is set"): + isolate._load_sam3() diff --git a/tests/test_gemini.py b/tests/test_gemini.py index bc03377..4a908e0 100644 --- a/tests/test_gemini.py +++ b/tests/test_gemini.py @@ -248,3 +248,62 @@ def test_api_image_keeps_16_bit_grey_and_palette_transparency(tmp_path: Path) -> assert mime == "image/png" with Image.open(io.BytesIO(data)) as im: assert im.mode == "RGBA" and im.getpixel((2, 2))[3] == 0 + + +def _engine_capturing(monkeypatch: pytest.MonkeyPatch, model: str) -> tuple[object, dict]: + """A GeminiEngine whose client records the last generate_content call.""" + from studio import config as config_mod + from studio.engines import gemini + + seen: dict = {} + + class Models: + def generate_content(self, **kw: object) -> object: + seen.update(kw) + part = type("P", (), {"inline_data": type("D", (), {"data": b"png"})()})() + content = type("Ct", (), {"parts": [part]})() + return type("R", (), {"candidates": [type("Cd", (), {"content": content})()]})() + + monkeypatch.setattr(config_mod.settings, "gemini_api_key", "k") + monkeypatch.setattr(gemini, "gemini_client", lambda key: type("C", (), {"models": Models()})()) + return gemini.GeminiEngine(model=model), seen + + +@pytest.mark.parametrize(("model", "cap"), [("gemini-3-pro-image", 5), ("gemini-nano-banana-2.1", 4)]) +def test_references_are_capped_at_the_models_character_limit( + model: str, cap: int, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from PIL import Image + + from studio.shotplan import default_plan + + engine, seen = _engine_capturing(monkeypatch, model) + refs = [] + for i in range(7): + refs.append(tmp_path / f"r{i}.png") + Image.new("RGB", (8, 8)).save(refs[-1]) + + engine.generate(refs + [refs[0]], default_plan()[0], tmp_path / "o.png", seed=1) + + assert len(seen["contents"]) == cap + 1 # references + the prompt + + +def test_aspect_follows_the_shots_framing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + from PIL import Image + + from studio.engines.gemini import shot_aspect + from studio.shotplan import default_plan + + by = {(s.kind, s.framing): shot_aspect(s) for s in default_plan()} + assert by[("angle", "full")] == "2:3" + assert by[("emotion", "close-up")] == by[("emotion", "tight-face")] == "1:1" + # A pose keeps the reference's aspect: a crouch squeezed into 2:3 loses the body. + assert by[("pose", "full")] is None and by[("pose", "waist-up")] is None + + engine, seen = _engine_capturing(monkeypatch, "gemini-nano-banana-2.1") + ref = tmp_path / "r.png" + Image.new("RGB", (8, 8)).save(ref) + angle = default_plan()[0] + engine.generate([ref], angle, tmp_path / "o.png", seed=1) + assert seen["config"].image_config.aspect_ratio == "2:3" + assert seen["config"].image_config.image_size is None # 1K stays the model default diff --git a/tests/test_identity_policy.py b/tests/test_identity_policy.py new file mode 100644 index 0000000..b09ead7 --- /dev/null +++ b/tests/test_identity_policy.py @@ -0,0 +1,70 @@ +"""0.19.0: the character identity policy — what the trigger owns. + +"identity" (default) varies outfits and captions the clothing, so the trigger +learns the person; "costume" keeps the reference's outfit and leaves it out of +captions, so the trigger carries it. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from PIL import Image +from typer.testing import CliRunner + +import app as A +import cli +from studio.config import CAPTIONERS_BY_KEY + +runner = CliRunner() + + +def test_caption_instruction_follows_the_policy() -> None: + spec = CAPTIONERS_BY_KEY["qwen3vl"] + assert spec.prompt_for("prose", identity="") == spec.prompt_template + assert spec.prompt_for("prose", identity="identity").endswith( + "Describe the clothing and the facial expression.") + assert "Do not tag the clothing" in spec.prompt_for("e621", identity="costume") + # Style/Concept captions never mention a costume policy. + assert "clothing" not in spec.prompt_for("prose", "concept", identity="identity") + + +def test_the_plan_is_dressed_unless_the_outfit_is_the_character() -> None: + dressed = A._plan_table("character") + worn = {k: [bool(o) for o in g] for k, g in dressed.groupby("kind")["outfit"]} + assert all(worn["angle"] + worn["pose"]) and not any(worn["emotion"]) + assert not any(A._plan_table("character", identity="costume")["outfit"]) + assert not any(A._plan_table("concept")["outfit"]) + plain, _note = A.on_identity_change(dressed, "costume", "character") + assert not any(plain["outfit"]) + + +def test_identity_policy_reaches_export_metadata(tmp_path: Path) -> None: + src = tmp_path / "src" + src.mkdir() + Image.new("RGB", (64, 64)).save(src / "a.png") + (src / "a.txt").write_text("a caption", encoding="utf-8") + _r, ds_dir, _hf = A.do_export([str(src / "a.png")], "Sy", "trg", str(tmp_path / "out"), + identity="costume") + meta = json.loads((Path(ds_dir) / "metadata.json").read_text(encoding="utf-8")) + assert meta["identity"] == "costume" + + +def test_cli_generate_dresses_by_default(monkeypatch, tmp_path: Path) -> None: + seen: dict = {} + + def fake_generate(sources, shots, engine, out_dir, **kw): + seen["outfits"] = [s.outfit for s in shots if s.kind == "angle"] + return [] + + monkeypatch.setattr(cli.pipeline, "generate_shots", fake_generate) + ref = tmp_path / "r.png" + Image.new("RGB", (8, 8)).save(ref) + runner.invoke(cli.app, ["generate", str(ref), "--out", str(tmp_path / "g")]) + assert all(seen["outfits"]) + runner.invoke(cli.app, ["generate", str(ref), "--out", str(tmp_path / "g"), + "--identity", "costume"]) + assert not any(seen["outfits"]) + bad = runner.invoke(cli.app, ["generate", str(ref), "--identity", "nope"]) + assert bad.exit_code != 0 diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py index 0374210..099c0a2 100644 --- a/tests/test_preprocess.py +++ b/tests/test_preprocess.py @@ -58,7 +58,34 @@ def fake_isolate(image_path, out_path, subject_prompt="character", exclude_promp monkeypatch.setattr(pp, "isolate_subject", fake_isolate) r = pp.preprocess(src, out, target=32, force_restore=False, isolate=True, alpha_cutout=True) - assert Image.open(r.output).mode == "RGBA" + assert Image.open(r.reference).mode == "RGBA" + assert Image.open(r.output).mode == "RGB" + + +def test_training_copy_keeps_its_background_and_refs_hold_the_cutout( + tmp_path: Path, monkeypatch) -> None: + """0.19.0: a dataset of subjects on white teaches the LoRA white backgrounds, + so isolation only feeds ②'s reference in `refs/`.""" + import studio.preprocess as pp + from studio.config import list_images + + src = tmp_path / "cat.png" + _img(src, color=(10, 120, 30)) + + def fake_isolate(image_path, out_path, *a, **kw): + Image.new("RGB", Image.open(image_path).size, "white").save(out_path, "PNG") + return out_path + + monkeypatch.setattr(pp, "isolate_subject", fake_isolate) + out = tmp_path / "out" + r = pp.preprocess(src, out, target=32, force_restore=False, isolate=True) + assert r.reference == out / pp.REFS_DIR / "cat_prepped.png" + assert Image.open(r.output).getpixel((0, 0)) == (10, 120, 30) + assert Image.open(r.reference).getpixel((0, 0)) == (255, 255, 255) + assert list_images(out) == [r.output] # ③ never sees the refs + # A second run must not reuse a name taken in refs/ either. + assert pp.preprocess(src, out, target=32, force_restore=False, + isolate=True).output.name == "cat_prepped_2.png" def test_preprocess_alpha_cutout_ignored_without_isolate(tmp_path: Path) -> None: diff --git a/tests/test_stage_resilience.py b/tests/test_stage_resilience.py index 85a856b..80626bd 100644 --- a/tests/test_stage_resilience.py +++ b/tests/test_stage_resilience.py @@ -69,9 +69,12 @@ def test_note_names_every_skipped_image_and_keeps_the_success_count() -> None: def test_note_for_alpha_cutout_still_explains_the_no_autofill_rule() -> None: - note = APP._preprocess_note([_ok("a.png")], Path("out"), True) + ok = _ok("a.png") + ok.reference = Path("out/refs/a.png") + note = APP._preprocess_note([ok], Path("out"), True) assert "transparent cutout" in note assert "not auto-filled" in note + assert "isolated references" in APP._preprocess_note([ok], Path("out"), False) def test_note_when_everything_failed_does_not_claim_a_success() -> None: diff --git a/tests/test_ui_wiring_0_16_0.py b/tests/test_ui_wiring_0_16_0.py index 23c4a5b..9cdc89f 100644 --- a/tests/test_ui_wiring_0_16_0.py +++ b/tests/test_ui_wiring_0_16_0.py @@ -44,9 +44,9 @@ def test_the_handoff_checkbox_is_actually_wired() -> None: def test_export_inputs_end_in_signature_order() -> None: - """`ilb_handoff, holdout` end do_export's signature, so they end the inputs.""" - assert _export_inputs()[-2:] == ["Prepare for Idiot LoRa Builder", - "Hold out N reference photos"] + """`ilb_handoff, holdout, identity` end do_export's signature, so they end the inputs.""" + assert _export_inputs()[-3:] == ["Prepare for Idiot LoRa Builder", + "Hold out N reference photos", "Identity policy"] def test_export_with_the_box_ticked_writes_the_sidecar(tmp_path: Path) -> None: