Skip to content
This repository was archived by the owner on Oct 25, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions library/config/cli_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,12 @@ def add_training_arguments(parser: argparse.ArgumentParser, support_dreambooth:
default=None,
help="save resumable checkpoint every N epochs (overwrites previous, auto-resumes on next run)",
)
parser.add_argument(
"--resume_state_every_n_steps",
type=int,
default=50,
help="atomically publish a rolling resume state every N optimizer steps (0 disables)",
)

parser.add_argument(
"--train_batch_size",
Expand Down
4 changes: 4 additions & 0 deletions library/io/output_layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ def state(self) -> Path:
def interrupted_state(self) -> Path:
return self.root / f"{self.name}-interrupted-state"

@property
def rolling_state(self) -> Path:
return self.root / f"{self.name}-rolling-state"

@property
def checkpoint_state(self) -> Path:
return self.root / f"{self.name}-checkpoint-state"
Expand Down
26 changes: 26 additions & 0 deletions library/training/checkpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
CHECKPOINT_STATE_NAME = "{}-checkpoint-state"
CHECKPOINT_FILE_NAME = "{}-checkpoint"
INTERRUPTED_STATE_NAME = "{}-interrupted-state"
ROLLING_STATE_NAME = "{}-rolling-state"


def default_if_none(value, default):
Expand Down Expand Up @@ -207,6 +208,11 @@ def get_interrupted_state_dir(args: argparse.Namespace) -> str:
return os.path.join(args.output_dir, INTERRUPTED_STATE_NAME.format(model_name))


def get_rolling_state_dir(args: argparse.Namespace) -> str:
model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME)
return os.path.join(args.output_dir, ROLLING_STATE_NAME.format(model_name))


def get_checkpoint_ckpt_name(args: argparse.Namespace, ext: str):
model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME)
return CHECKPOINT_FILE_NAME.format(model_name) + ext
Expand All @@ -222,6 +228,14 @@ def save_checkpoint_state(args: argparse.Namespace, accelerator):
_atomic_accelerator_save_state(accelerator, state_dir)


def save_rolling_state(args: argparse.Namespace, accelerator):
"""Publish the latest complete optimizer-boundary state atomically."""
state_dir = get_rolling_state_dir(args)
logger.info("saving rolling resume state to %s", state_dir)
os.makedirs(args.output_dir, exist_ok=True)
_atomic_accelerator_save_state(accelerator, state_dir)


def save_state_on_train_end(args: argparse.Namespace, accelerator):
model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME)

Expand Down Expand Up @@ -645,6 +659,10 @@ def auto_resume(self) -> None:
default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME)
)
]
rolling_suffix = ROLLING_STATE_NAME.format(
default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME)
)
suffixes.append(rolling_suffix)

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 High: Retire rolling state after a successful run

The rolling directory is always an auto-resume candidate, while normal completion only removes the epoch checkpoint state. For example, a successful 120-step run with the default cadence leaves a complete step-100 rolling state; a later fresh invocation with the same output name silently resumes from step 100 and skips most of the requested training.

Delete/retire the rolling state after success, or require an explicit continuation policy before considering it during a fresh invocation.

Confidence: 98/100 · Impact: 75/100

Code

if getattr(args, "checkpointing_epochs", None):
suffixes.append(
CHECKPOINT_STATE_NAME.format(
Expand Down Expand Up @@ -805,6 +823,14 @@ def maybe_save_resumable(
self.save(ckpt_name, network, global_step, epoch_no)
save_checkpoint_state(args, accelerator)

def maybe_save_rolling_state(self, global_step: int) -> None:
"""Save a rolling state after a committed optimizer/global step."""
cadence = int(getattr(self.args, "resume_state_every_n_steps", 50) or 0)
if cadence <= 0 or global_step <= 0 or global_step % cadence:
return
self.accelerator.wait_for_everyone()
save_rolling_state(self.args, self.accelerator)

def cleanup_resumable(self) -> None:
"""At training end, remove the resumable checkpoint state dir + ckpt
file. Main-process only; no-op when ``checkpointing_epochs`` is unset."""
Expand Down
3 changes: 3 additions & 0 deletions library/training/loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,9 @@ def _run_epoch_steps(trainer, state: LoopState, epoch: int) -> None:
)
return

# Publish recovery state only after the complete optimizer
# update/scheduler step in _run_step has committed.
state.saver.maybe_save_rolling_state(state.global_step)
_sample_at_step(trainer, state)
state.saver.maybe_save_step(state.network, state.global_step, epoch)
_log_checkpoint_artifact(trainer, args, state, epoch)
Expand Down
6 changes: 6 additions & 0 deletions library/training/progress.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ def run_start(
pid: int,
log_dir: Optional[str] = None,
sampling_enabled: bool = False,
config_signature: Optional[str] = None,
dataset_signature: Optional[str] = None,
) -> None:
"""Open the file fresh (truncating any stale stream) and write the
opening event.
Expand All @@ -211,6 +213,10 @@ def run_start(
self._fh = None
return
extra = {"log_dir": log_dir} if log_dir is not None else {}
if config_signature is not None:
extra["config_signature"] = config_signature
if dataset_signature is not None:
extra["dataset_signature"] = dataset_signature
self._emit(
"run_start",
run=self._run,
Expand Down
52 changes: 45 additions & 7 deletions scripts/daemon/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,23 @@ but hold the queue paused, omitted/`null` → leave the gate as-is.

Response: `201 {"job_id": "20260611-142233-a1b2c3", "state": "queued"}`.

### `GET /jobs` — list
### `GET /jobs` — list and history

Returns `[job, …]` (full records, submission order). Each job has `state` ∈
`queued | running | done | error | stopped`.
Returns historical and active records; terminal job directories are retained
until explicitly deleted. With no query string the response remains a list for
older clients. Add query parameters to receive `{jobs, total, offset, limit}`:

| param | meaning |
|-------|---------|
| `state` / `status` | one or more comma-separated states: `queued`, `running`, `done`, `error`, `stopped` |
| `resumable` | only jobs with a complete signature-matched recovery state (`true`/`false`) |
| `before` / `after` | submission-time Unix timestamp bounds |
| `offset` / `limit` | zero-based pagination; `limit` defaults to 100 |

Each job also exposes `recovery_state`, `recovery_step`, `target_steps`,
`target_epochs`, `stop_requested`, `forced_stop`, and `legacy`. A legacy job was
created before immutable config snapshots existed and needs manual config/data
verification before resuming.

### `GET /jobs/{id}` — status

Expand Down Expand Up @@ -134,10 +147,30 @@ curve at 1-in-50 resolution plus any warnings:
GET /jobs/{id}/progress?events=step,log&every_nth=50
```

### `POST /jobs/{id}/stop` — abort
### `POST /jobs/{id}/stop` — cancel

The endpoint is idempotent. Queued jobs become `stopped` immediately. Training
jobs first receive a cooperative stop request and save an interrupted state at a
complete optimizer boundary; only after `STOP_GRACE_SECONDS` does the daemon
tree-kill the process and persist `forced_stop=true`. The Python client's
`stop()` with no id resolves the active job from `/health`.

### `POST /jobs/{id}/resume` — continue training

Creates a **new** queued training job from the newest complete state among
`interrupted-state`, rolling state, checkpoint state, and compatible legacy
state directories. Selection is by committed `global_step`, then state type as
a tie-break. Config and dataset signatures must match when present. A missing
state, signature mismatch, or an already-reached `target_steps`/`target_epochs`
returns HTTP 409 and does not start a fresh run. The original job and its logs
remain intact.

### `DELETE /jobs/{id}` — delete history

Stops a running or queued job (tree-kills the process). Returns `{job_id, state}`.
The Python client's `stop()` with no id resolves the active job from `/health`.
Only terminal jobs may be deleted. This removes that job's `job.json`,
`stdout.log`, `progress.jsonl`, and `sample/` metadata. Model weights and all
training state directories under `output/ckpt` are outside the job directory and
are never removed.

### `POST /queue/pause` · `POST /queue/start`

Expand Down Expand Up @@ -166,7 +199,12 @@ false — has died (a bug worth a report).

### `POST /shutdown`

`{"kill_jobs": true}` → stop the daemon, optionally killing the running job.
Use `{"mode": "detach"}` to stop the daemon while leaving live and queued jobs
for a replacement daemon to adopt, `{"mode": "cooperative-stop"}` to request
checkpoint-capable stopping before exit, or `{"mode": "force"}` to explicitly
tree-kill the running process (`forced_stop=true`, latest state not guaranteed).
The old `kill_jobs` boolean is accepted as an alias (`true` = `force`, `false` =
`detach`).

## Python client (`scripts.daemon.client`)

Expand Down
36 changes: 32 additions & 4 deletions scripts/daemon/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Iterator, Optional
Expand Down Expand Up @@ -210,8 +211,15 @@ def pause_queue(self) -> Optional[dict]:
"""Hold the queue — queued jobs wait until ``start_queue``."""
return self._request("POST", "/queue/pause")

def list_jobs(self) -> list:
return self._request("GET", "/jobs") or []
def list_jobs(self, *, state: str | None = None, resumable: bool | None = None,
offset: int = 0, limit: int | None = None) -> list:
query = []
if state: query.append(f"state={urllib.parse.quote(state)}")
if resumable is not None: query.append(f"resumable={'true' if resumable else 'false'}")
if offset: query.append(f"offset={int(offset)}")
if limit is not None: query.append(f"limit={int(limit)}")
payload = self._request("GET", "/jobs" + ("?" + "&".join(query) if query else "")) or []
return payload.get("jobs", []) if isinstance(payload, dict) else payload

def get(self, job_id: str) -> dict:
return self._request("GET", f"/jobs/{job_id}")
Expand All @@ -226,9 +234,29 @@ def stop(self, job_id: Optional[str] = None) -> dict:
return {"error": "no active job"}
return self._request("POST", f"/jobs/{job_id}/stop")

def shutdown(self, *, kill_jobs: bool = True) -> Optional[dict]:
def resume(self, job_id: str) -> dict:
return self._request("POST", f"/jobs/{job_id}/resume")

def list_job_groups(self, *, state: str | None = None, offset: int = 0,
limit: int = 100) -> list:
query = [f"offset={int(offset)}", f"limit={int(limit)}"]
if state:
query.append(f"state={urllib.parse.quote(state)}")
payload = self._request("GET", "/job-groups?" + "&".join(query)) or {}
return list(payload.get("groups") or [])

def get_job_group(self, job_id: str) -> dict:
return self._request("GET", f"/job-groups/{job_id}")

def delete_job_group(self, job_id: str) -> dict:
return self._request("DELETE", f"/job-groups/{job_id}")

def delete(self, job_id: str) -> dict:
return self._request("DELETE", f"/jobs/{job_id}")

def shutdown(self, *, kill_jobs: bool = True, mode: str | None = None) -> Optional[dict]:
try:
return self._request("POST", "/shutdown", {"kill_jobs": kill_jobs})
return self._request("POST", "/shutdown", {"kill_jobs": kill_jobs, "mode": mode})
except (urllib.error.URLError, OSError, ValueError):
return None

Expand Down
24 changes: 23 additions & 1 deletion scripts/daemon/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,14 @@ class Job:
# guarding here only races the just-exited step's not-yet-released VRAM.
from_chain: bool = False

# One user-visible training may span several physical daemon processes.
# Each process keeps its own immutable job directory; these additive fields
# link resumed attempts into one logical task without breaking old records.
# Legacy job.json files omit them and are treated as independent roots.
root_job_id: Optional[str] = None
parent_job_id: Optional[str] = None
attempt_index: int = 0

state: str = STATE_QUEUED
submitted_at: float = field(default_factory=time.time)
started_at: Optional[float] = None
Expand Down Expand Up @@ -98,6 +106,18 @@ class Job:
stop_requested_at: Optional[float] = None
forced_stop: bool = False

# Immutable recovery contract captured at submission time. These fields
# are deliberately additive so older job.json records remain readable.
config_signature: Optional[str] = None
dataset_signature: Optional[str] = None
target_steps: Optional[int] = None
target_epochs: Optional[int] = None
data_manifest: Optional[str] = None
legacy: bool = False
recovery_state: Optional[str] = None
recovery_step: Optional[int] = None
terminal_reason: Optional[str] = None

@property
def dir(self) -> Path:
return config.job_dir(self.id)
Expand Down Expand Up @@ -132,7 +152,9 @@ def from_dict(cls, data: dict) -> "Job":
def public(self) -> dict:
"""The dict shape returned over HTTP (drops nothing sensitive — this is
localhost — but keeps the field order stable for clients)."""
return asdict(self)
payload = asdict(self)
payload["root_job_id"] = self.root_job_id or self.id
return payload


def load_all() -> dict[str, Job]:
Expand Down
Loading