From 548d800e431626237f517c6498c8ce0474dcb96a Mon Sep 17 00:00:00 2001 From: buxinzi2233 Date: Tue, 11 Aug 2026 21:39:28 +0800 Subject: [PATCH] feat: merge resumed training attempts into logical tasks --- library/config/cli_args.py | 6 + library/io/output_layout.py | 4 + library/training/checkpoints.py | 26 + library/training/loop.py | 3 + library/training/progress.py | 6 + scripts/daemon/README.md | 52 +- scripts/daemon/client.py | 36 +- scripts/daemon/jobs.py | 24 +- scripts/daemon/manager.py | 422 +++++++++++++- scripts/daemon/server.py | 197 ++++++- scripts/run_analyzer/analyze.py | 98 +++- scripts/run_analyzer/discovery.py | 232 +++++++- scripts/run_analyzer/server.py | 101 ++-- scripts/run_analyzer/static/css/base.css | 51 +- .../run_analyzer/static/css/view-detail.css | 41 ++ .../run_analyzer/static/css/view-monitor.css | 15 + scripts/run_analyzer/static/css/view-runs.css | 21 +- scripts/run_analyzer/static/index.html | 8 +- .../static/js/components/lightbox.js | 10 +- .../run_analyzer/static/js/views/detail.js | 3 + .../run_analyzer/static/js/views/monitor.js | 4 +- scripts/run_analyzer/static/js/views/runs.js | 2 +- .../run_analyzer/static/js/views/samples.js | 4 +- scripts/tasks/staged_resolution.py | 32 +- start-webui-linux.sh | 4 +- tests/test_daemon.py | 353 ++++++++++++ tests/test_preview.py | 48 ++ tests/test_progress_sink.py | 10 +- tests/test_run_analyzer.py | 138 ++++- tests/test_staged_resolution_plan.py | 92 +++ tests/test_task_service_recovery.py | 519 ++++++++++++++++- train.py | 2 + webui/api/preview.py | 101 +++- webui/api/tasks.py | 62 ++- .../frontend/src/components/SampleGallery.vue | 4 +- .../src/components/TaskAttemptHistory.vue | 90 +++ webui/frontend/src/components/TaskPanel.vue | 97 +++- .../src/composables/useTrainingStream.ts | 2 + webui/frontend/src/i18n/cn.ts | 32 ++ webui/frontend/src/i18n/en.ts | 32 ++ webui/frontend/src/i18n/ja.ts | 32 ++ webui/frontend/src/i18n/ko.ts | 32 ++ webui/frontend/src/stores/stagedResolution.ts | 2 +- webui/frontend/src/stores/task.ts | 153 ++++- webui/frontend/src/stores/training.ts | 15 +- webui/frontend/src/views/AdapterView.vue | 4 +- webui/frontend/src/views/DistillView.vue | 4 +- webui/frontend/src/views/MergeView.vue | 4 +- webui/frontend/src/views/PreprocessView.vue | 4 +- .../src/views/SuperResolutionView.vue | 2 +- webui/frontend/src/views/SystemView.vue | 4 +- webui/frontend/src/views/TaskMonitorView.vue | 263 +++++++-- webui/models/task.py | 10 +- webui/server.py | 12 +- webui/services/config_service.py | 16 +- webui/services/daemon_client.py | 103 +++- webui/services/task_service.py | 524 ++++++++++++++++-- webui/services/training_log_parser.py | 39 +- 58 files changed, 3895 insertions(+), 312 deletions(-) create mode 100644 webui/frontend/src/components/TaskAttemptHistory.vue diff --git a/library/config/cli_args.py b/library/config/cli_args.py index 83bd8f05d..fa7189e2b 100644 --- a/library/config/cli_args.py +++ b/library/config/cli_args.py @@ -233,6 +233,12 @@ def add_training_arguments(parser: argparse.ArgumentParser, support_dreambooth: default=None, help="save resumable checkpoint every N epochs (overwrites previous, auto-resumes on next run)", ) + parser.add_argument( + "--resume_state_every_n_steps", + type=int, + default=50, + help="atomically publish a rolling resume state every N optimizer steps (0 disables)", + ) parser.add_argument( "--train_batch_size", diff --git a/library/io/output_layout.py b/library/io/output_layout.py index ce000a390..69a43019d 100644 --- a/library/io/output_layout.py +++ b/library/io/output_layout.py @@ -94,6 +94,10 @@ def state(self) -> Path: def interrupted_state(self) -> Path: return self.root / f"{self.name}-interrupted-state" + @property + def rolling_state(self) -> Path: + return self.root / f"{self.name}-rolling-state" + @property def checkpoint_state(self) -> Path: return self.root / f"{self.name}-checkpoint-state" diff --git a/library/training/checkpoints.py b/library/training/checkpoints.py index 5f468cb6a..69a0c4d26 100644 --- a/library/training/checkpoints.py +++ b/library/training/checkpoints.py @@ -40,6 +40,7 @@ CHECKPOINT_STATE_NAME = "{}-checkpoint-state" CHECKPOINT_FILE_NAME = "{}-checkpoint" INTERRUPTED_STATE_NAME = "{}-interrupted-state" +ROLLING_STATE_NAME = "{}-rolling-state" def default_if_none(value, default): @@ -207,6 +208,11 @@ def get_interrupted_state_dir(args: argparse.Namespace) -> str: return os.path.join(args.output_dir, INTERRUPTED_STATE_NAME.format(model_name)) +def get_rolling_state_dir(args: argparse.Namespace) -> str: + model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME) + return os.path.join(args.output_dir, ROLLING_STATE_NAME.format(model_name)) + + def get_checkpoint_ckpt_name(args: argparse.Namespace, ext: str): model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME) return CHECKPOINT_FILE_NAME.format(model_name) + ext @@ -222,6 +228,14 @@ def save_checkpoint_state(args: argparse.Namespace, accelerator): _atomic_accelerator_save_state(accelerator, state_dir) +def save_rolling_state(args: argparse.Namespace, accelerator): + """Publish the latest complete optimizer-boundary state atomically.""" + state_dir = get_rolling_state_dir(args) + logger.info("saving rolling resume state to %s", state_dir) + os.makedirs(args.output_dir, exist_ok=True) + _atomic_accelerator_save_state(accelerator, state_dir) + + def save_state_on_train_end(args: argparse.Namespace, accelerator): model_name = default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME) @@ -645,6 +659,10 @@ def auto_resume(self) -> None: default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME) ) ] + rolling_suffix = ROLLING_STATE_NAME.format( + default_if_none(args.output_name, DEFAULT_LAST_OUTPUT_NAME) + ) + suffixes.append(rolling_suffix) if getattr(args, "checkpointing_epochs", None): suffixes.append( CHECKPOINT_STATE_NAME.format( @@ -805,6 +823,14 @@ def maybe_save_resumable( self.save(ckpt_name, network, global_step, epoch_no) save_checkpoint_state(args, accelerator) + def maybe_save_rolling_state(self, global_step: int) -> None: + """Save a rolling state after a committed optimizer/global step.""" + cadence = int(getattr(self.args, "resume_state_every_n_steps", 50) or 0) + if cadence <= 0 or global_step <= 0 or global_step % cadence: + return + self.accelerator.wait_for_everyone() + save_rolling_state(self.args, self.accelerator) + def cleanup_resumable(self) -> None: """At training end, remove the resumable checkpoint state dir + ckpt file. Main-process only; no-op when ``checkpointing_epochs`` is unset.""" diff --git a/library/training/loop.py b/library/training/loop.py index 7d174f5ac..68c88db6f 100644 --- a/library/training/loop.py +++ b/library/training/loop.py @@ -651,6 +651,9 @@ def _run_epoch_steps(trainer, state: LoopState, epoch: int) -> None: ) return + # Publish recovery state only after the complete optimizer + # update/scheduler step in _run_step has committed. + state.saver.maybe_save_rolling_state(state.global_step) _sample_at_step(trainer, state) state.saver.maybe_save_step(state.network, state.global_step, epoch) _log_checkpoint_artifact(trainer, args, state, epoch) diff --git a/library/training/progress.py b/library/training/progress.py index 05042cfff..2b8f221f8 100644 --- a/library/training/progress.py +++ b/library/training/progress.py @@ -187,6 +187,8 @@ def run_start( pid: int, log_dir: Optional[str] = None, sampling_enabled: bool = False, + config_signature: Optional[str] = None, + dataset_signature: Optional[str] = None, ) -> None: """Open the file fresh (truncating any stale stream) and write the opening event. @@ -211,6 +213,10 @@ def run_start( self._fh = None return extra = {"log_dir": log_dir} if log_dir is not None else {} + if config_signature is not None: + extra["config_signature"] = config_signature + if dataset_signature is not None: + extra["dataset_signature"] = dataset_signature self._emit( "run_start", run=self._run, diff --git a/scripts/daemon/README.md b/scripts/daemon/README.md index ef1000f63..710452036 100644 --- a/scripts/daemon/README.md +++ b/scripts/daemon/README.md @@ -102,10 +102,23 @@ but hold the queue paused, omitted/`null` → leave the gate as-is. Response: `201 {"job_id": "20260611-142233-a1b2c3", "state": "queued"}`. -### `GET /jobs` — list +### `GET /jobs` — list and history -Returns `[job, …]` (full records, submission order). Each job has `state` ∈ -`queued | running | done | error | stopped`. +Returns historical and active records; terminal job directories are retained +until explicitly deleted. With no query string the response remains a list for +older clients. Add query parameters to receive `{jobs, total, offset, limit}`: + +| param | meaning | +|-------|---------| +| `state` / `status` | one or more comma-separated states: `queued`, `running`, `done`, `error`, `stopped` | +| `resumable` | only jobs with a complete signature-matched recovery state (`true`/`false`) | +| `before` / `after` | submission-time Unix timestamp bounds | +| `offset` / `limit` | zero-based pagination; `limit` defaults to 100 | + +Each job also exposes `recovery_state`, `recovery_step`, `target_steps`, +`target_epochs`, `stop_requested`, `forced_stop`, and `legacy`. A legacy job was +created before immutable config snapshots existed and needs manual config/data +verification before resuming. ### `GET /jobs/{id}` — status @@ -134,10 +147,30 @@ curve at 1-in-50 resolution plus any warnings: GET /jobs/{id}/progress?events=step,log&every_nth=50 ``` -### `POST /jobs/{id}/stop` — abort +### `POST /jobs/{id}/stop` — cancel + +The endpoint is idempotent. Queued jobs become `stopped` immediately. Training +jobs first receive a cooperative stop request and save an interrupted state at a +complete optimizer boundary; only after `STOP_GRACE_SECONDS` does the daemon +tree-kill the process and persist `forced_stop=true`. The Python client's +`stop()` with no id resolves the active job from `/health`. + +### `POST /jobs/{id}/resume` — continue training + +Creates a **new** queued training job from the newest complete state among +`interrupted-state`, rolling state, checkpoint state, and compatible legacy +state directories. Selection is by committed `global_step`, then state type as +a tie-break. Config and dataset signatures must match when present. A missing +state, signature mismatch, or an already-reached `target_steps`/`target_epochs` +returns HTTP 409 and does not start a fresh run. The original job and its logs +remain intact. + +### `DELETE /jobs/{id}` — delete history -Stops a running or queued job (tree-kills the process). Returns `{job_id, state}`. -The Python client's `stop()` with no id resolves the active job from `/health`. +Only terminal jobs may be deleted. This removes that job's `job.json`, +`stdout.log`, `progress.jsonl`, and `sample/` metadata. Model weights and all +training state directories under `output/ckpt` are outside the job directory and +are never removed. ### `POST /queue/pause` · `POST /queue/start` @@ -166,7 +199,12 @@ false — has died (a bug worth a report). ### `POST /shutdown` -`{"kill_jobs": true}` → stop the daemon, optionally killing the running job. +Use `{"mode": "detach"}` to stop the daemon while leaving live and queued jobs +for a replacement daemon to adopt, `{"mode": "cooperative-stop"}` to request +checkpoint-capable stopping before exit, or `{"mode": "force"}` to explicitly +tree-kill the running process (`forced_stop=true`, latest state not guaranteed). +The old `kill_jobs` boolean is accepted as an alias (`true` = `force`, `false` = +`detach`). ## Python client (`scripts.daemon.client`) diff --git a/scripts/daemon/client.py b/scripts/daemon/client.py index 9266a1dd2..4bae428b0 100644 --- a/scripts/daemon/client.py +++ b/scripts/daemon/client.py @@ -13,6 +13,7 @@ import sys import time import urllib.error +import urllib.parse import urllib.request from pathlib import Path from typing import Iterator, Optional @@ -210,8 +211,15 @@ def pause_queue(self) -> Optional[dict]: """Hold the queue — queued jobs wait until ``start_queue``.""" return self._request("POST", "/queue/pause") - def list_jobs(self) -> list: - return self._request("GET", "/jobs") or [] + def list_jobs(self, *, state: str | None = None, resumable: bool | None = None, + offset: int = 0, limit: int | None = None) -> list: + query = [] + if state: query.append(f"state={urllib.parse.quote(state)}") + if resumable is not None: query.append(f"resumable={'true' if resumable else 'false'}") + if offset: query.append(f"offset={int(offset)}") + if limit is not None: query.append(f"limit={int(limit)}") + payload = self._request("GET", "/jobs" + ("?" + "&".join(query) if query else "")) or [] + return payload.get("jobs", []) if isinstance(payload, dict) else payload def get(self, job_id: str) -> dict: return self._request("GET", f"/jobs/{job_id}") @@ -226,9 +234,29 @@ def stop(self, job_id: Optional[str] = None) -> dict: return {"error": "no active job"} return self._request("POST", f"/jobs/{job_id}/stop") - def shutdown(self, *, kill_jobs: bool = True) -> Optional[dict]: + def resume(self, job_id: str) -> dict: + return self._request("POST", f"/jobs/{job_id}/resume") + + def list_job_groups(self, *, state: str | None = None, offset: int = 0, + limit: int = 100) -> list: + query = [f"offset={int(offset)}", f"limit={int(limit)}"] + if state: + query.append(f"state={urllib.parse.quote(state)}") + payload = self._request("GET", "/job-groups?" + "&".join(query)) or {} + return list(payload.get("groups") or []) + + def get_job_group(self, job_id: str) -> dict: + return self._request("GET", f"/job-groups/{job_id}") + + def delete_job_group(self, job_id: str) -> dict: + return self._request("DELETE", f"/job-groups/{job_id}") + + def delete(self, job_id: str) -> dict: + return self._request("DELETE", f"/jobs/{job_id}") + + def shutdown(self, *, kill_jobs: bool = True, mode: str | None = None) -> Optional[dict]: try: - return self._request("POST", "/shutdown", {"kill_jobs": kill_jobs}) + return self._request("POST", "/shutdown", {"kill_jobs": kill_jobs, "mode": mode}) except (urllib.error.URLError, OSError, ValueError): return None diff --git a/scripts/daemon/jobs.py b/scripts/daemon/jobs.py index 3261018a7..d987b1fa2 100644 --- a/scripts/daemon/jobs.py +++ b/scripts/daemon/jobs.py @@ -63,6 +63,14 @@ class Job: # guarding here only races the just-exited step's not-yet-released VRAM. from_chain: bool = False + # One user-visible training may span several physical daemon processes. + # Each process keeps its own immutable job directory; these additive fields + # link resumed attempts into one logical task without breaking old records. + # Legacy job.json files omit them and are treated as independent roots. + root_job_id: Optional[str] = None + parent_job_id: Optional[str] = None + attempt_index: int = 0 + state: str = STATE_QUEUED submitted_at: float = field(default_factory=time.time) started_at: Optional[float] = None @@ -98,6 +106,18 @@ class Job: stop_requested_at: Optional[float] = None forced_stop: bool = False + # Immutable recovery contract captured at submission time. These fields + # are deliberately additive so older job.json records remain readable. + config_signature: Optional[str] = None + dataset_signature: Optional[str] = None + target_steps: Optional[int] = None + target_epochs: Optional[int] = None + data_manifest: Optional[str] = None + legacy: bool = False + recovery_state: Optional[str] = None + recovery_step: Optional[int] = None + terminal_reason: Optional[str] = None + @property def dir(self) -> Path: return config.job_dir(self.id) @@ -132,7 +152,9 @@ def from_dict(cls, data: dict) -> "Job": def public(self) -> dict: """The dict shape returned over HTTP (drops nothing sensitive — this is localhost — but keeps the field order stable for clients).""" - return asdict(self) + payload = asdict(self) + payload["root_job_id"] = self.root_job_id or self.id + return payload def load_all() -> dict[str, Job]: diff --git a/scripts/daemon/manager.py b/scripts/daemon/manager.py index 3605f55a4..0b9affc33 100644 --- a/scripts/daemon/manager.py +++ b/scripts/daemon/manager.py @@ -20,6 +20,8 @@ import shutil import threading import time +import json +from pathlib import Path from typing import Optional import toml @@ -116,16 +118,26 @@ def start(self) -> None: self._reconcile() self._worker.start() - def shutdown(self, *, kill_jobs: bool) -> None: + def shutdown(self, *, kill_jobs: bool = False, mode: str | None = None) -> None: """Stop accepting work and unblock the worker. With ``kill_jobs`` the active job tree is torn down and the GPU freed before the daemon exits. """ + # ``mode`` is the durable public contract. Keep kill_jobs as a + # backwards-compatible alias used by older clients. + mode = mode or ("force" if kill_jobs else "detach") + if mode not in {"detach", "cooperative-stop", "force"}: + raise ValueError("shutdown mode must be detach, cooperative-stop, or force") with self._lock: self._stopping = True - self._kill_on_shutdown = kill_jobs + self._kill_on_shutdown = mode == "force" current = self._current_running_locked() - if kill_jobs and current is not None: + if current is not None and mode == "cooperative-stop": + self.stop(current.id) + elif current is not None and mode == "force": current.stop_requested = True + current.forced_stop = True + current.status_detail = "force shutdown requested; latest checkpoint not guaranteed" + current.persist() self._kill_job_tree(current) self._run_gate.set() # release a worker parked on a paused queue self._queue.put(_SENTINEL) # wake the worker so it can exit @@ -141,6 +153,9 @@ def submit( overrides: Optional[dict] = None, extra: Optional[list[str]] = None, from_chain: bool = False, + root_job_id: Optional[str] = None, + parent_job_id: Optional[str] = None, + attempt_index: int = 0, start: Optional[bool] = None, ) -> Job: job = Job( @@ -151,6 +166,9 @@ def submit( overrides=dict(overrides or {}), extra=list(extra or []), from_chain=from_chain, + root_job_id=root_job_id, + parent_job_id=parent_job_id, + attempt_index=attempt_index, ) self._attach_config_file( job, config_snapshot=config_snapshot, config_file=config_file @@ -166,6 +184,9 @@ def submit_command( chain_train: Optional[dict] = None, config_snapshot: Optional[dict] = None, config_file: Optional[str] = None, + root_job_id: Optional[str] = None, + parent_job_id: Optional[str] = None, + attempt_index: int = 0, start: Optional[bool] = None, ) -> Job: """Enqueue a plain ``python `` task (preprocess / mask). @@ -188,6 +209,9 @@ def submit_command( argv=list(argv or []), extra_env=dict(extra_env or {}), chain_train=dict(chain_train) if chain_train else None, + root_job_id=root_job_id, + parent_job_id=parent_job_id, + attempt_index=attempt_index, ) self._attach_config_file( job, config_snapshot=config_snapshot, config_file=config_file @@ -219,6 +243,21 @@ def _attach_config_file( if os.path.abspath(str(dst)) != src: shutil.copyfile(src, dst) job.config_file = str(dst) + try: + merged = toml.load(dst) + job.config_signature = merged.get("config_signature") or merged.get("_config_signature") + target = merged.get("max_train_steps") or merged.get("max_train_steps_target") + job.target_steps = int(target) if target is not None else None + epochs = merged.get("max_train_epochs") + job.target_epochs = int(epochs) if epochs is not None else None + manifest = ( + merged.get("preprocess_run") + or merged.get("dataset_manifest") + or merged.get("staged_profile_manifest") + ) + job.data_manifest = str(manifest) if manifest else None + except (OSError, ValueError, TypeError): + job.legacy = True def _register_and_queue(self, job: Job, *, start: Optional[bool] = None) -> Job: # ``start`` controls the run gate atomically with enqueue, so there's no @@ -232,6 +271,7 @@ def _register_and_queue(self, job: Job, *, start: Optional[bool] = None) -> Job: # stalled auto-advance the moment the running job finished. # True → enqueue, then resume (run now — flushes any held backlog); # None → leave the gate as-is (legacy: runs if not currently paused). + job.root_job_id = job.root_job_id or job.id if start is False: with self._lock: queue_idle = self._queue_is_idle_locked() @@ -244,6 +284,31 @@ def _register_and_queue(self, job: Job, *, start: Optional[bool] = None) -> Job: # locate the gallery even after a WebUI restart (it reads this back # over HTTP, not from the injected argv). job.sample_dir = str(d / "sample") + if self._job_runs_train(job) and not job.config_file: + job.legacy = True + tokens = list(job.extra if job.kind == "train" else job.argv) + for key, attr in (("--preprocess_run", "data_manifest"), ("--dataset_config", "data_manifest")): + if key in tokens and tokens.index(key) + 1 < len(tokens): + setattr(job, attr, str(tokens[tokens.index(key) + 1])) + break + target = (job.overrides or {}).get("max_train_steps") + if target is None and "--max_train_steps" in tokens: + index = tokens.index("--max_train_steps") + target = tokens[index + 1] if index + 1 < len(tokens) else None + if target is not None: + try: + job.target_steps = int(target) + except (TypeError, ValueError): + pass + epochs = (job.overrides or {}).get("max_train_epochs") + if epochs is None and "--max_train_epochs" in tokens: + index = tokens.index("--max_train_epochs") + epochs = tokens[index + 1] if index + 1 < len(tokens) else None + if epochs is not None: + try: + job.target_epochs = int(epochs) + except (TypeError, ValueError): + pass with self._lock: self._jobs[job.id] = job job.persist() @@ -273,6 +338,307 @@ def list_jobs(self) -> list[Job]: with self._lock: return sorted(self._jobs.values(), key=lambda j: j.submitted_at) + def delete(self, job_id: str) -> bool: + """Delete only a terminal daemon job record and its owned artifacts.""" + with self._lock: + job = self._jobs.get(job_id) + if job is None or job.state not in TERMINAL_STATES: + return False + path = job.dir + self._jobs.pop(job_id, None) + # The job directory contains metadata/logs/progress/sample only. State + # and model outputs live elsewhere and are intentionally untouched. + shutil.rmtree(path, ignore_errors=False) + self._broadcast({"ev": "deleted", "job_id": job_id}) + return True + + @staticmethod + def _attempt_sort_key(job: Job) -> tuple[int, float, str]: + return (int(job.attempt_index or 0), float(job.submitted_at or 0.0), job.id) + + def lineage(self, job_or_id: Job | str) -> list[Job]: + """Return one logical task's physical attempts in execution order.""" + with self._lock: + job = job_or_id if isinstance(job_or_id, Job) else self._jobs.get(job_or_id) + if job is None: + return [] + root_id = job.root_job_id or job.id + attempts = [ + candidate + for candidate in self._jobs.values() + if (candidate.root_job_id or candidate.id) == root_id + ] + return sorted(attempts, key=self._attempt_sort_key) + + @staticmethod + def _attempt_public(job: Job) -> dict: + payload = job.public() + payload["job_id"] = job.id + return payload + + def job_group(self, job_id: str) -> dict | None: + attempts = self.lineage(job_id) + if not attempts: + return None + first, latest = attempts[0], attempts[-1] + payload = latest.public() + root_id = first.root_job_id or first.id + payload.update( + { + "id": root_id, + "root_job_id": root_id, + "current_job_id": latest.id, + "attempt_count": len(attempts), + "attempts": [self._attempt_public(attempt) for attempt in attempts], + "submitted_at": first.submitted_at, + "started_at": first.started_at or latest.started_at, + "ended_at": latest.ended_at, + } + ) + return payload + + def list_job_groups_filtered( + self, + *, + state: str | None = None, + resumable: bool | None = None, + before: float | None = None, + after: float | None = None, + offset: int = 0, + limit: int | None = None, + newest_first: bool = False, + ) -> tuple[list[dict], int]: + with self._lock: + root_ids = { + job.root_job_id or job.id + for job in self._jobs.values() + } + groups = [self.job_group(root_id) for root_id in root_ids] + states = {part.strip() for part in (state or "").split(",") if part.strip()} + filtered: list[dict] = [] + for group in groups: + if group is None: + continue + if states and group.get("state") not in states: + continue + submitted_at = float(group.get("submitted_at") or 0.0) + if before is not None and submitted_at >= before: + continue + if after is not None and submitted_at < after: + continue + if resumable is not None and bool(group.get("recovery_state")) != resumable: + continue + filtered.append(group) + filtered.sort( + key=lambda group: (float(group.get("submitted_at") or 0.0), str(group.get("id") or "")), + reverse=newest_first, + ) + total = len(filtered) + start = max(0, int(offset)) + end = None if limit is None else start + max(0, int(limit)) + return filtered[start:end], total + + def delete_group(self, job_id: str) -> list[str] | None: + """Delete all terminal attempt records for one logical task.""" + attempts = self.lineage(job_id) + if not attempts or any(job.state not in TERMINAL_STATES for job in attempts): + return None + paths = [job.dir for job in attempts] + ids = [job.id for job in attempts] + with self._lock: + for attempt_id in ids: + self._jobs.pop(attempt_id, None) + try: + for path in paths: + shutil.rmtree(path, ignore_errors=False) + except OSError: + # Keep the in-memory table consistent with the surviving records. + with self._lock: + for job in attempts: + if job.dir.exists(): + self._jobs[job.id] = job + raise + self._broadcast({"ev": "deleted_group", "root_job_id": attempts[0].root_job_id or attempts[0].id, "job_ids": ids}) + return ids + + def list_jobs_filtered(self, *, state: str | None = None, resumable: bool | None = None, + before: float | None = None, after: float | None = None, + offset: int = 0, limit: int | None = None, + newest_first: bool = False) -> tuple[list[Job], int]: + jobs = self.list_jobs() + states = {part.strip() for part in (state or "").split(",") if part.strip()} + filtered = [] + for job in jobs: + if states and job.state not in states: + continue + if before is not None and job.submitted_at >= before: + continue + if after is not None and job.submitted_at < after: + continue + if resumable is not None and bool(job.recovery_state) != resumable: + continue + filtered.append(job) + filtered.sort(key=lambda job: (job.submitted_at, job.id), reverse=newest_first) + total = len(filtered) + start = max(0, int(offset)) + end = None if limit is None else start + max(0, int(limit)) + return filtered[start:end], total + + @staticmethod + def _read_state_sidecar(path: Path) -> dict | None: + try: + data = json.loads((path / "train_state.json").read_text(encoding="utf-8")) + if not isinstance(data, dict) or "global_step" not in data and "current_step" not in data: + return None + if int(data.get("schema_version", 1) or 1) >= 2 and not (path / "complete.marker").is_file(): + return None + data["global_step"] = int(data.get("global_step", data.get("current_step", 0)) or 0) + return data + except (OSError, ValueError, TypeError): + return None + + @staticmethod + def _state_candidates(job: Job) -> list[tuple[int, Path]]: + """Discover complete state dirs from a pinned config/output contract.""" + try: + from library.io.output_layout import resolve_output_layout + import toml + cfg = toml.load(job.config_file) if job.config_file and os.path.isfile(job.config_file) else {} + if not cfg: + method = job.method + preset = job.preset or (job.extra_env or {}).get("PRESET") or "default" + methods_subdir = job.methods_subdir or "methods" + if job.kind == "command" and job.argv: + if len(job.argv) >= 3 and str(job.argv[0]).endswith("tasks.py") and job.argv[1] == "lora-gui": + method, methods_subdir = job.argv[2], "gui-methods" + elif len(job.argv) >= 3 and str(job.argv[0]).endswith("tasks.py") and job.argv[1] == "staged-train": + try: + from library.training.staged_resolution_plan import ( + compile_runtime_config, + load_profile, + ) + profile = str(job.argv[2]) + cfg_path = compile_runtime_config( + profile, load_profile(profile), config.ROOT + ) + cfg = toml.load(cfg_path) + except Exception: + cfg = {} + elif len(job.argv) >= 2 and str(job.argv[0]).endswith("tasks.py"): + method = {"lora": "lora", "easycontrol": "easycontrol"}.get(job.argv[1], method) + try: + from library.config.io import load_method_preset + cfg = load_method_preset(method, preset, methods_subdir=methods_subdir) + except Exception: + cfg = {} + output_dir = cfg.get("output_dir", "output/ckpt") + output_name = cfg.get("output_name") or job.method or "last" + if job.target_steps is None and cfg.get("max_train_steps") is not None: + try: + job.target_steps = int(cfg["max_train_steps"]) + except (TypeError, ValueError): + pass + if job.target_epochs is None and cfg.get("max_train_epochs") is not None: + try: + job.target_epochs = int(cfg["max_train_epochs"]) + except (TypeError, ValueError): + pass + layout = resolve_output_layout(output_dir, output_name, cwd=config.ROOT) + suffixes = ("-interrupted-state", "-rolling-state", "-checkpoint-state", "-state") + raw_name = Path(str(output_name)).name + candidate_names = [layout.name] + if raw_name and raw_name not in candidate_names: + candidate_names.append(raw_name) + out = [] + for root in layout.legacy_candidates(): + paths = [ + root / f"{name}{suffix}" + for name in candidate_names + for suffix in suffixes + ] + for name in candidate_names: + paths.extend(sorted(root.glob(f"{name}-step*-state"))) + paths.extend(sorted(root.glob(f"{name}-[0-9]*-state"))) + for path in paths: + data = JobManager._read_state_sidecar(path) + if data is None: + continue + if job.config_signature and data.get("config_signature") != job.config_signature: + continue + if job.dataset_signature and data.get("dataset_signature") != job.dataset_signature: + continue + out.append((int(data.get("global_step", 0) or 0), path)) + def priority(path: Path) -> int: + name = path.name + if name.endswith("-interrupted-state"): return 0 + if name.endswith("-rolling-state"): return 1 + if name.endswith("-checkpoint-state"): return 2 + if name == f"{layout.name}-state": return 3 + return 4 + return sorted({(step, path) for step, path in out}, key=lambda item: (-item[0], priority(item[1]), str(item[1]))) + except Exception: + logger.exception("resume state discovery failed for job %s", job.id) + return [] + + def resume_job(self, job_id: str) -> Job | None: + source = self.get(job_id) + if source is None or source.state not in {STATE_STOPPED, STATE_ERROR} or source.kind != "train" and not self._command_runs_train(source.argv): + return None + attempts = self.lineage(source) + if not attempts: + return None + latest = attempts[-1] + if latest.id != source.id: + raise ValueError( + f"only the latest attempt can be resumed; latest_job_id={latest.id}" + ) + if any(attempt.state in {STATE_QUEUED, STATE_RUNNING} for attempt in attempts): + raise ValueError("this training already has an active attempt") + candidates = self._state_candidates(source) + if not candidates: + raise ValueError("no complete signature-matched training state found") + step, state_dir = candidates[0] + state_data = self._read_state_sidecar(state_dir) or {} + if source.target_steps is not None and step >= source.target_steps: + raise ValueError(f"training target already reached at global_step={step}") + if source.target_epochs is not None: + current_epoch = int(state_data.get("current_epoch", state_data.get("epoch", 0)) or 0) + if current_epoch >= source.target_epochs: + raise ValueError( + f"training target already reached at epoch={current_epoch}" + ) + if source.kind == "command": + argv = list(source.argv) + argv += ["--resume", str(state_dir)] + job = self.submit_command(label=source.method, argv=argv, extra_env=source.extra_env, + config_file=source.config_file, + root_job_id=source.root_job_id or source.id, + parent_job_id=source.id, + attempt_index=int(source.attempt_index or 0) + 1, + start=False) + else: + extra = list(source.extra) + extra += ["--resume", str(state_dir)] + job = self.submit(method=source.method, preset=source.preset, methods_subdir=source.methods_subdir, + config_file=source.config_file, overrides=source.overrides, extra=extra, + root_job_id=source.root_job_id or source.id, + parent_job_id=source.id, + attempt_index=int(source.attempt_index or 0) + 1, + start=False) + job.config_signature = source.config_signature + job.dataset_signature = source.dataset_signature + job.target_steps = source.target_steps + job.target_epochs = source.target_epochs + job.data_manifest = source.data_manifest + job.recovery_state = str(state_dir) + job.recovery_step = step + job.persist() + # Publish all recovery metadata before allowing the worker to launch; + # otherwise a fast queue can build the command before --config_file is + # injected for command-style training jobs. + self.resume() + return job + def get(self, job_id: str) -> Optional[Job]: with self._lock: return self._jobs.get(job_id) @@ -685,6 +1051,7 @@ def _finalize( # ``_finalize_from_daemon`` sees it atomically with the state flip. job.rc = rc job.ckpt_path = tail.last_ckpt_path(job.progress_path) + self._refresh_recovery_metadata(job) # Auto-chain: a done command job with a chain_train spec enqueues its # follow-on train job here (survives the GUI closing). chained_job_id # persists in the same write that flips us to `done` → atomic for a @@ -715,6 +1082,32 @@ def _finalize( job.persist() self._broadcast({"ev": "ended", "job_id": job.id, "state": state}) + def _refresh_recovery_metadata(self, job: Job) -> None: + """Persist the best available state/signature metadata after a run.""" + # Signatures are computed only after train.py has parsed the pinned + # config and materialized the dataset. Persist them in run_start so the + # daemon can distinguish this job's state from an older run that reused + # the same output_name, even after a daemon restart. + starts = tail.read_events(job.progress_path, events=["run_start"], last_n=1) + if starts: + started = starts[-1] + if started.get("config_signature"): + job.config_signature = started["config_signature"] + if started.get("dataset_signature"): + job.dataset_signature = started["dataset_signature"] + candidates = self._state_candidates(job) + if not candidates: + return + step, path = candidates[0] + job.recovery_state = str(path) + job.recovery_step = step + try: + data = self._read_state_sidecar(path) or {} + job.config_signature = job.config_signature or data.get("config_signature") + job.dataset_signature = job.dataset_signature or data.get("dataset_signature") + except Exception: + logger.debug("could not read recovery metadata for job %s", job.id) + def _gpu_guard( self, job: Job, @@ -1019,6 +1412,10 @@ def _build_cmd(self, job: Job) -> tuple[list[str], dict]: if self._command_runs_train(argv): env["TQDM_MININTERVAL"] = "0.5" env["TQDM_MINITERS"] = "1" + # A resumed command job uses its immutable submission snapshot; + # the original command remains unchanged for compatibility. + if job.recovery_state and job.config_file: + self._ensure_flag(argv, "--config_file", job.config_file) # Per-job progress + preview paths. The WebUI reads # job.progress_path / job.sample_dir back over HTTP (the argv # here is only the train process's copy, not visible to the @@ -1072,6 +1469,9 @@ def _build_cmd(self, job: Job) -> tuple[list[str], dict]: def _reconcile(self) -> None: self._jobs = load_all() for job in self._jobs.values(): + if self._job_runs_train(job) and not job.config_file and not job.legacy: + job.legacy = True + job.persist() if job.state == STATE_RUNNING: if proc.is_alive(job.pid, job.create_time): logger.info("reconcile: re-attaching live job %s", job.id) @@ -1084,6 +1484,17 @@ def _reconcile(self) -> None: self._start_stop_worker(job) else: logger.info("reconcile: job %s died while we were down", job.id) + # The progress stream is the only durable terminal proof + # available when the daemon did not own/reap the child. + # Prefer it over a synthetic orphan/error classification. + terminal = tail.last_event(job.progress_path) + if terminal and terminal.get("ev") == "run_end": + status = terminal.get("status") + mapped = {"ok": STATE_DONE, "stopped": STATE_STOPPED, "error": STATE_ERROR}.get(status) + if mapped: + self._finalize(job, mapped, error=terminal.get("error"), rc=None, + detail="reconciled from progress run_end") + continue if job.stop_requested: self._finalize( job, @@ -1103,6 +1514,11 @@ def _reconcile(self) -> None: ) elif job.state == STATE_QUEUED: self._queue.put(job.id) + elif job.state in TERMINAL_STATES: + previous = (job.recovery_state, job.recovery_step) + self._refresh_recovery_metadata(job) + if previous != (job.recovery_state, job.recovery_step): + job.persist() def _current_running_locked(self) -> Optional[Job]: for job in self._jobs.values(): diff --git a/scripts/daemon/server.py b/scripts/daemon/server.py index b6f4345ff..d8a282086 100644 --- a/scripts/daemon/server.py +++ b/scripts/daemon/server.py @@ -12,17 +12,18 @@ POST /jobs {method, preset, methods_subdir, overrides, extra} → {job_id} or {kind:"command", label, argv, extra_env, chain_train?} → {job_id} - GET /jobs → [job, …] + GET /jobs → [job, …] or {jobs, total, offset, limit} with query filters GET /jobs/{id} → job (+ latest progress event, stale_for) GET /jobs/{id}/progress → filtered progress.jsonl events ?events=step,val&since_step=N&every_nth=N&last_n=N - POST /jobs/{id}/stop → {job} + POST /jobs/{id}/stop → {job}; training stops cooperatively before timeout POST /queue/start → {ok, paused:false} (resume a paused queue) POST /queue/pause → {ok, paused:true} (hold queued jobs) GET /jobs/{id}/logs → SSE: tail of the job's stdout.log GET /events → SSE: daemon-level lifecycle events GET /health → {ok, pid, port, root, active_job, paused, worker_alive, worker_idle_for} - POST /shutdown {kill_jobs} → {ok} + DELETE /jobs/{id} → delete terminal job metadata only + POST /shutdown {mode: detach|cooperative-stop|force} → {ok} """ from __future__ import annotations @@ -39,13 +40,17 @@ from . import config, tail from .manager import JobManager +from .jobs import TERMINAL_STATES logger = logging.getLogger("anima.daemon") _JOB_RE = re.compile(r"^/jobs/(?P[^/]+)$") _JOB_STOP_RE = re.compile(r"^/jobs/(?P[^/]+)/stop$") +_JOB_RESUME_RE = re.compile(r"^/jobs/(?P[^/]+)/resume$") +_JOB_DELETE_RE = re.compile(r"^/jobs/(?P[^/]+)$") _JOB_LOGS_RE = re.compile(r"^/jobs/(?P[^/]+)/logs$") _JOB_PROGRESS_RE = re.compile(r"^/jobs/(?P[^/]+)/progress$") +_JOB_GROUP_RE = re.compile(r"^/job-groups/(?P[^/]+)$") _README = Path(__file__).resolve().parent / "README.md" @@ -157,10 +162,13 @@ }, { "name": "list_jobs", - "description": "List all jobs (full records, submission order). Each has state ∈ queued|running|done|error|stopped.", + "description": "List historical and active jobs. Supports state/status, resumable, before/after time filters and offset/limit pagination.", "method": "GET", "path": "/jobs", - "input_schema": {"type": "object", "properties": {}}, + "input_schema": {"type": "object", "properties": { + "state": {"type": "string"}, "resumable": {"type": "boolean"}, + "offset": {"type": "integer"}, "limit": {"type": "integer"} + }}, }, { "name": "get_job", @@ -217,7 +225,7 @@ }, { "name": "stop_job", - "description": "Abort a running or queued job (tree-kills the process). Returns {job_id, state}.", + "description": "Cancel a running or queued job. Training first saves cooperatively and is tree-killed only after the grace deadline. Returns {job_id, state}.", "method": "POST", "path": "/jobs/{id}/stop", "input_schema": { @@ -226,6 +234,20 @@ "properties": {"id": {"type": "string", "description": "Job id to stop."}}, }, }, + { + "name": "resume_job", + "description": "Create a new training job from the newest complete signature-matched state of a stopped/error/orphaned job.", + "method": "POST", + "path": "/jobs/{id}/resume", + "input_schema": {"type": "object", "required": ["id"], "properties": {"id": {"type": "string"}}}, + }, + { + "name": "delete_job_history", + "description": "Delete terminal job metadata/logs/progress/sample only. Model weights and training state directories are preserved.", + "method": "DELETE", + "path": "/jobs/{id}", + "input_schema": {"type": "object", "required": ["id"], "properties": {"id": {"type": "string"}}}, + }, { "name": "tail_logs", "description": ( @@ -274,7 +296,12 @@ "type": "boolean", "default": True, "description": "Kill the running job on the way down.", - } + }, + "mode": { + "type": "string", + "enum": ["detach", "cooperative-stop", "force"], + "description": "Detach preserves live/queued jobs for reattach; cooperative-stop waits for a checkpoint; force tree-kills explicitly." + }, }, }, }, @@ -347,7 +374,11 @@ def do_GET(self) -> None: # noqa: N802 elif path == "/health": self._handle_health() elif path == "/jobs": - self._handle_list() + self._handle_list(query) + elif path == "/job-groups": + self._handle_group_list(query) + elif m := _JOB_GROUP_RE.match(path): + self._handle_group_get(m.group("id")) elif path == "/events": self._handle_events() elif m := _JOB_LOGS_RE.match(path): @@ -371,11 +402,45 @@ def do_POST(self) -> None: # noqa: N802 self._send_json({"ok": True, "paused": True}) elif path == "/shutdown": self._handle_shutdown() + elif m := _JOB_RESUME_RE.match(path): + self._handle_resume(m.group("id")) elif m := _JOB_STOP_RE.match(path): self._handle_stop(m.group("id")) else: self._send_json({"error": "not found", "path": path}, 404) + def do_DELETE(self) -> None: # noqa: N802 + path = self.path.split("?", 1)[0] + if m := _JOB_GROUP_RE.match(path): + group = self.manager.job_group(m.group("id")) + if group is None: + self._send_json({"error": "no such job group", "root_job_id": m.group("id")}, 404) + elif any(attempt.get("state") not in TERMINAL_STATES for attempt in group["attempts"]): + self._send_json({"error": "active job groups cannot be deleted", "root_job_id": group["root_job_id"]}, 409) + else: + try: + deleted = self.manager.delete_group(m.group("id")) + except OSError as exc: + self._send_json({"error": f"could not delete job group: {exc}", "root_job_id": group["root_job_id"]}, 500) + else: + self._send_json({"ok": True, "root_job_id": group["root_job_id"], "job_ids": deleted or []}) + return + if m := _JOB_DELETE_RE.match(path): + job = self.manager.get(m.group("id")) + if job is None: + self._send_json({"error": "no such job", "job_id": m.group("id")}, 404) + elif job.state not in TERMINAL_STATES: + self._send_json({"error": "running jobs cannot be deleted", "job_id": job.id}, 409) + else: + try: + self.manager.delete(job.id) + except OSError as exc: + self._send_json({"error": f"could not delete job: {exc}", "job_id": job.id}, 500) + else: + self._send_json({"ok": True, "job_id": job.id}) + return + self._send_json({"error": "not found", "path": path}, 404) + def _handle_readme(self) -> None: try: text = _README.read_text(encoding="utf-8") @@ -443,8 +508,74 @@ def _handle_submit(self) -> None: 201, ) - def _handle_list(self) -> None: - self._send_json([j.public() for j in self.manager.list_jobs()]) + def _handle_list(self, query: str = "") -> None: + params = urllib.parse.parse_qs(query) + state = (params.get("state") or params.get("status") or [None])[0] + raw_resume = (params.get("resumable") or [None])[0] + resumable = None if raw_resume is None else raw_resume.lower() in {"1", "true", "yes"} + def integer(name, default): + try: return int((params.get(name) or [default])[0]) + except (TypeError, ValueError): return default + def number(name): + try: return float((params.get(name) or [None])[0]) + except (TypeError, ValueError): return None + order = (params.get("order") or ["asc"])[0].lower() + jobs, total = self.manager.list_jobs_filtered(state=state, resumable=resumable, + before=number("before") if number("before") is not None else number("to"), + after=number("after") if number("after") is not None else number("from"), + offset=integer("offset", 0), limit=integer("limit", 100), + newest_first=order in {"desc", "newest"}) + if not query: + self._send_json([j.public() for j in jobs]) + else: + self._send_json({"jobs": [j.public() for j in jobs], "total": total, + "offset": integer("offset", 0), "limit": integer("limit", 100)}) + + def _handle_group_list(self, query: str = "") -> None: + params = urllib.parse.parse_qs(query) + state = (params.get("state") or params.get("status") or [None])[0] + raw_resume = (params.get("resumable") or [None])[0] + resumable = None if raw_resume is None else raw_resume.lower() in {"1", "true", "yes"} + + def integer(name, default): + try: + return int((params.get(name) or [default])[0]) + except (TypeError, ValueError): + return default + + def number(name): + try: + return float((params.get(name) or [None])[0]) + except (TypeError, ValueError): + return None + + order = (params.get("order") or ["asc"])[0].lower() + before = number("before") + after = number("after") + groups, total = self.manager.list_job_groups_filtered( + state=state, + resumable=resumable, + before=before if before is not None else number("to"), + after=after if after is not None else number("from"), + offset=integer("offset", 0), + limit=integer("limit", 100), + newest_first=order in {"desc", "newest"}, + ) + self._send_json( + { + "groups": groups, + "total": total, + "offset": integer("offset", 0), + "limit": integer("limit", 100), + } + ) + + def _handle_group_get(self, job_id: str) -> None: + group = self.manager.job_group(job_id) + if group is None: + self._send_json({"error": "no such job group", "root_job_id": job_id}, 404) + return + self._send_json(group) def _handle_get(self, job_id: str) -> None: job = self.manager.get(job_id) @@ -500,15 +631,40 @@ def _handle_stop(self, job_id: str) -> None: return self._send_json({"job_id": job.id, "state": job.state}) + def _handle_resume(self, job_id: str) -> None: + try: + job = self.manager.resume_job(job_id) + except ValueError as exc: + self._send_json({"error": str(exc), "job_id": job_id}, 409) + return + if job is None: + self._send_json({"error": "job is not a resumable training terminal", "job_id": job_id}, 409) + return + self._send_json({"job_id": job.id, "source_job_id": job_id, "root_job_id": job.root_job_id or job.id, + "parent_job_id": job.parent_job_id, "attempt_index": job.attempt_index, + "state": job.state, "resume_state": job.recovery_state, + "resume_step": job.recovery_step}, 201) + def _handle_shutdown(self) -> None: body = self._read_json() - kill = bool(body.get("kill_jobs", True)) - self._send_json({"ok": True, "kill_jobs": kill}) + mode = body.get("mode") + if mode is None: + mode = "force" if bool(body.get("kill_jobs", True)) else "detach" + if mode not in {"detach", "cooperative-stop", "force"}: + self._send_json( + { + "error": "shutdown mode must be detach, cooperative-stop, or force", + "mode": mode, + }, + 400, + ) + return + self._send_json({"ok": True, "mode": mode}) # Trigger shutdown after the response is flushed, off the handler thread # (server.shutdown() must not run in a request thread). threading.Thread( target=self.server.request_shutdown, # type: ignore[attr-defined] - args=(kill,), + args=(mode,), daemon=True, ).start() @@ -570,8 +726,19 @@ def __init__(self, addr, manager: JobManager): super().__init__(addr, _Handler) self.manager = manager - def request_shutdown(self, kill_jobs: bool) -> None: - self.manager.shutdown(kill_jobs=kill_jobs) + def request_shutdown(self, mode: str | bool) -> None: + # Older in-process callers passed ``kill_jobs`` positionally. Keep that + # contract working while the HTTP/API surface uses explicit modes. + if isinstance(mode, bool): + mode = "force" if mode else "detach" + self.manager.shutdown(mode=mode) + if mode == "cooperative-stop": + deadline = time.time() + max(5.0, config.STOP_GRACE_SECONDS + 15.0) + while time.time() < deadline: + active = self.manager.active_job() + if active is None or active.state in TERMINAL_STATES: + break + time.sleep(0.25) self.shutdown() # unblocks serve_forever() diff --git a/scripts/run_analyzer/analyze.py b/scripts/run_analyzer/analyze.py index 53862fc84..d3a38ec21 100644 --- a/scripts/run_analyzer/analyze.py +++ b/scripts/run_analyzer/analyze.py @@ -25,6 +25,29 @@ def _resolve_path(run: Run, p: Optional[str]) -> Optional[str]: return os.path.normpath(os.path.join(MONADFORGE_ROOT, p)) +def _attempt_for(run: Run, attempt_id: Optional[str]) -> Optional[Run]: + if not attempt_id: + return None + return next( + (attempt for attempt in run.attempts if attempt.id == attempt_id), + None, + ) + + +def _resolve_sample_path( + run: Run, p: Optional[str], attempt_id: Optional[str] +) -> Optional[str]: + path = _resolve_path(run, p) + if path and os.path.isfile(path): + return path + attempt = _attempt_for(run, attempt_id) + if attempt is not None and attempt.sample_dir and p: + candidate = os.path.join(attempt.sample_dir, os.path.basename(p)) + if os.path.isfile(candidate): + return candidate + return path + + def sparkline(run: Run, n: int = 60) -> list: """loss/average 降采样为 n 点(用于索引行内迷你曲线)。 @@ -185,10 +208,12 @@ def sample_list(run: Run) -> list[dict]: if jl is not None: for s in jl.samples: p = s.get("path") - path = _resolve_path(run, p) + attempt_id = s.get("attempt_id") + path = _resolve_sample_path(run, p, attempt_id) size = _sample_size(path) items.append( { + "attempt_id": attempt_id, "global_step": s.get("global_step"), "epoch": s.get("epoch"), "path": path, @@ -198,20 +223,29 @@ def sample_list(run: Run) -> list[dict]: "h": size[1] if size else None, } ) - # PNG files in sample_dir not covered by events (defensive) - sdir = run.sample_dir - if sdir and os.path.isdir(sdir): - have = {os.path.basename(i["path"]) for i in items if i["path"]} + # Image files not covered by events (defensive), across every attempt. + scan_runs = run.attempts or [run] + have = { + (item.get("attempt_id"), os.path.basename(item["path"])) + for item in items + if item["path"] + } + for attempt in scan_runs: + sdir = attempt.sample_dir + if not sdir or not os.path.isdir(sdir): + continue + scan_attempt_id = attempt.id if run.kind == "daemon" else None for fn in sorted(os.listdir(sdir)): if not fn.lower().endswith((".png", ".jpg", ".jpeg", ".webp")): continue - if fn in have: + if (scan_attempt_id, fn) in have: continue m = re.match(r".*_e(\d{6})_(\d{2})_", fn) path = os.path.join(sdir, fn) size = _sample_size(path) items.append( { + "attempt_id": scan_attempt_id, "global_step": None, "epoch": int(m.group(1)) if m else None, "path": path, @@ -221,7 +255,36 @@ def sample_list(run: Run) -> list[dict]: "h": size[1] if size else None, } ) - return items + return sorted( + items, + key=lambda item: ( + float(item.get("ts") or 0.0), + int(item.get("global_step") or -1), + str(item.get("path") or ""), + ), + ) + + +def _attempt_payload(run: Run) -> list[dict]: + attempts = run.attempts or ([run] if run.kind == "daemon" else []) + out = [] + for position, attempt in enumerate(attempts): + job = attempt.job or {} + out.append( + { + "id": attempt.id, + "attempt_index": int(job.get("attempt_index") or position), + "parent_job_id": job.get("parent_job_id"), + "state": attempt.state, + "submitted_at": attempt.submitted_at, + "started_at": attempt.started_at, + "ended_at": attempt.ended_at, + "recovery_step": job.get("recovery_step"), + "rc": job.get("rc"), + "error": attempt.error, + } + ) + return out def overview_payload(run: Run) -> dict: @@ -229,6 +292,12 @@ def overview_payload(run: Run) -> dict: job = run.job or {} return { "id": run.id, + "root_job_id": run.id if run.kind == "daemon" else None, + "current_job_id": ( + run.attempts[-1].id if run.attempts else run.id + ) if run.kind == "daemon" else None, + "attempt_count": len(run.attempts) if run.attempts else 1, + "attempts": _attempt_payload(run), "kind": run.kind, "run_name": run.run_name or run.id, "method": run.method or "", @@ -260,6 +329,17 @@ def overview_payload(run: Run) -> dict: "sample_dir": run.sample_dir, "progress": run.jsonl_path, "stdout": os.path.join(run.dir, "stdout.log") if run.kind == "daemon" else None, + "attempts": [ + { + "id": attempt.id, + "dir": attempt.dir, + "log_dir": attempt.log_dir, + "sample_dir": attempt.sample_dir, + "progress": attempt.jsonl_path, + "stdout": os.path.join(attempt.dir, "stdout.log"), + } + for attempt in run.attempts + ], }, } @@ -315,6 +395,10 @@ def index_payload(runs: list[Run]) -> list[dict]: out.append( { "id": r.id, + "current_job_id": ( + r.attempts[-1].id if r.attempts else r.id + ) if r.kind == "daemon" else None, + "attempt_count": len(r.attempts) if r.attempts else 1, "kind": r.kind, "run_name": r.run_name or r.id, "method": r.method or "", diff --git a/scripts/run_analyzer/discovery.py b/scripts/run_analyzer/discovery.py index 23bbc9d2d..c5cf27985 100644 --- a/scripts/run_analyzer/discovery.py +++ b/scripts/run_analyzer/discovery.py @@ -68,6 +68,8 @@ class Run: error: Optional[str] = None stop_requested: bool = False sources: dict = field(default_factory=dict) + # Physical daemon attempts, oldest first. Inline runs leave this empty. + attempts: list["Run"] = field(default_factory=list) def _is_training_job(job: dict) -> bool: @@ -123,10 +125,199 @@ def apply_jsonl_metadata(run: Run, jl: JsonlRun) -> None: run.error = jl.run_end_error +def _link_log_sources(run: Run) -> None: + log_dir = run.log_dir + if log_dir is None or not os.path.isdir(log_dir): + run.sources = _source_flags(run) + return + if run.snapshot is None: + snaps = [f for f in os.listdir(log_dir) if f.endswith(".snapshot.toml")] + if snaps: + run.snapshot = parse_snapshot(os.path.join(log_dir, snaps[0])) + if run.tb is None: + run.tb = parse_tb(log_dir) + run.sources = _source_flags(run) + + +def _attempt_sort_key(run: Run) -> tuple[int, float, str]: + job = run.job or {} + try: + attempt_index = int(job.get("attempt_index") or 0) + except (TypeError, ValueError): + attempt_index = 0 + submitted = run.submitted_at + try: + submitted_value = float(submitted) if submitted is not None else 0.0 + except (TypeError, ValueError): + submitted_value = 0.0 + return attempt_index, submitted_value, run.id + + +def _shift_event(event: dict, attempt_id: str, offset: float) -> dict: + copied = dict(event) + copied["attempt_id"] = attempt_id + ts = copied.get("ts") + if isinstance(ts, (int, float)) and not isinstance(ts, bool): + copied["ts"] = float(ts) + offset + return copied + + +def _merge_jsonl(attempts: list[Run]) -> Optional[JsonlRun]: + available = [run for run in attempts if run.jsonl is not None] + if not available: + return None + + out = JsonlRun() + series_by_step: dict[str, dict[int, float]] = {} + step_times: list[float] = [] + starts = [float(run.started_at) for run in attempts if run.started_at is not None] + root_started_at = min(starts) if starts else None + + for run in attempts: + jl = run.jsonl + if jl is None: + continue + if jl.run: + out.run = jl.run + if jl.method: + out.method = jl.method + if jl.preset: + out.preset = jl.preset + if jl.total_steps is not None: + out.total_steps = jl.total_steps + if jl.total_epochs is not None: + out.total_epochs = jl.total_epochs + if jl.pid is not None: + out.pid = jl.pid + if jl.log_dir: + out.log_dir = jl.log_dir + out.sampling_enabled = out.sampling_enabled or jl.sampling_enabled + + offset = 0.0 + if root_started_at is not None and run.started_at is not None: + offset = max(0.0, float(run.started_at) - root_started_at) + for tag, points in jl.series.items(): + merged = series_by_step.setdefault(tag, {}) + for step, value in points: + merged[int(step)] = float(value) + out.gs_epoch.update({int(step): int(epoch) for step, epoch in jl.gs_epoch.items()}) + out.ckpts.extend(_shift_event(event, run.id, offset) for event in jl.ckpts) + out.samples.extend(_shift_event(event, run.id, offset) for event in jl.samples) + out.vals.extend(_shift_event(event, run.id, offset) for event in jl.vals) + out.logs.extend(_shift_event(event, run.id, offset) for event in jl.logs) + out.error_count += jl.error_count + for ts in (jl.first_step_ts, jl.last_step_ts): + if isinstance(ts, (int, float)) and not isinstance(ts, bool): + step_times.append(float(ts) + offset) + + out.series = { + tag: [[step, values[step]] for step in sorted(values)] + for tag, values in series_by_step.items() + } + all_steps = sorted({step for values in series_by_step.values() for step in values}) + if all_steps: + out.first_step = all_steps[0] + out.last_step = all_steps[-1] + if step_times: + out.first_step_ts = min(step_times) + out.last_step_ts = max(step_times) + latest_jsonl = attempts[-1].jsonl + if latest_jsonl is not None: + out.run_end_status = latest_jsonl.run_end_status + out.run_end_final_step = latest_jsonl.run_end_final_step + out.run_end_error = latest_jsonl.run_end_error + out.run_end_extra = dict(latest_jsonl.run_end_extra) + return out + + +def _merge_tensorboard(attempts: list[Run]) -> Optional[TbRun]: + series_by_step: dict[str, dict[int, float]] = {} + latest_file: Optional[str] = None + for run in attempts: + if run.tb is None: + continue + latest_file = run.tb.file or latest_file + for tag, points in run.tb.series.items(): + merged = series_by_step.setdefault(tag, {}) + for step, value in points: + merged[int(step)] = float(value) + if not series_by_step: + return None + series = { + tag: [[step, values[step]] for step in sorted(values)] + for tag, values in series_by_step.items() + } + steps = sorted({step for values in series_by_step.values() for step in values}) + return TbRun(series=series, steps=steps, file=latest_file) + + +def _merge_stdout(attempts: list[Run]) -> Optional[StdoutRun]: + if not any(run.stdout is not None for run in attempts): + return None + out = StdoutRun() + for position, run in enumerate(attempts, start=1): + stdout = run.stdout + if stdout is None: + continue + recovery_step = (run.job or {}).get("recovery_step") + detail = f" from step {recovery_step}" if recovery_step is not None else "" + out.lines.append(f"[attempt {position}/{len(attempts)} - {run.id}{detail}]") + out.lines.extend(stdout.lines) + out.warnings.extend(stdout.warnings) + out.errors.extend(stdout.errors) + out.markers.extend(stdout.markers) + return out + + +def merge_daemon_attempts(attempts: list[Run]) -> Run: + """Aggregate physical daemon attempts into one logical training run.""" + ordered = sorted(attempts, key=_attempt_sort_key) + current = ordered[-1] + current_job = current.job or {} + root_id = str(current_job.get("root_job_id") or ordered[0].id) + merged_jsonl = _merge_jsonl(ordered) + merged = Run( + id=root_id, + kind="daemon", + dir=current.dir, + job=current.job, + jsonl=merged_jsonl, + jsonl_path=current.jsonl_path, + tb=_merge_tensorboard(ordered), + snapshot=next((run.snapshot for run in reversed(ordered) if run.snapshot), None), + stdout=_merge_stdout(ordered), + log_dir=next((run.log_dir for run in reversed(ordered) if run.log_dir), None), + sample_dir=current.sample_dir, + run_name=(merged_jsonl.run if merged_jsonl else current.run_name), + method=(merged_jsonl.method if merged_jsonl else current.method), + preset=(merged_jsonl.preset if merged_jsonl else current.preset), + state=current.state, + submitted_at=min( + (run.submitted_at for run in ordered if run.submitted_at is not None), + default=current.submitted_at, + ), + started_at=min( + (run.started_at for run in ordered if run.started_at is not None), + default=current.started_at, + ), + ended_at=current.ended_at, + total_steps=(merged_jsonl.total_steps if merged_jsonl else current.total_steps), + total_epochs=(merged_jsonl.total_epochs if merged_jsonl else current.total_epochs), + ckpt_path=current.ckpt_path, + error=current.error, + stop_requested=current.stop_requested, + attempts=ordered, + ) + if merged_jsonl is not None: + apply_jsonl_metadata(merged, merged_jsonl) + merged.sources = _source_flags(merged) + return merged + + def _scan_daemon_jobs() -> list[Run]: - runs: list[Run] = [] + physical_runs: list[Run] = [] if not os.path.isdir(JOBS_DIR): - return runs + return [] for name in sorted(os.listdir(JOBS_DIR), reverse=True): job_dir = os.path.join(JOBS_DIR, name) if not os.path.isdir(job_dir): @@ -157,9 +348,19 @@ def _scan_daemon_jobs() -> list[Run]: jl = run.jsonl if jl is not None: apply_jsonl_metadata(run, jl) - run.sources = _source_flags(run) - runs.append(run) - return runs + _link_log_sources(run) + physical_runs.append(run) + + grouped: dict[str, list[Run]] = {} + for run in physical_runs: + root_id = str((run.job or {}).get("root_job_id") or run.id) + grouped.setdefault(root_id, []).append(run) + runs = [merge_daemon_attempts(group) for group in grouped.values()] + return sorted( + runs, + key=lambda run: float(run.submitted_at or 0.0), + reverse=True, + ) def _scan_inline_logs(known_log_dirs: set) -> list[Run]: @@ -226,21 +427,12 @@ def _source_flags(run: Run) -> dict: def discover(use_cache: bool = True) -> list[Run]: """Build the full run index (daemon jobs first, then inline runs).""" runs = _scan_daemon_jobs() - known = {r.log_dir for r in runs if r.log_dir} + known = { + attempt.log_dir + for run in runs + for attempt in (run.attempts or [run]) + if attempt.log_dir + } inline = _scan_inline_logs(known) runs.extend(inline) - # link missing snapshot/tb for daemon runs whose log_dir exists - for run in runs: - if run.kind != "daemon": - continue - ld = run.log_dir - if ld is None or not os.path.isdir(ld): - continue - if run.snapshot is None: - snaps = [f for f in os.listdir(ld) if f.endswith(".snapshot.toml")] - if snaps: - run.snapshot = parse_snapshot(os.path.join(ld, snaps[0])) - if run.tb is None: - run.tb = parse_tb(ld) - run.sources = _source_flags(run) return runs diff --git a/scripts/run_analyzer/server.py b/scripts/run_analyzer/server.py index ef78019b3..7e1fe7480 100644 --- a/scripts/run_analyzer/server.py +++ b/scripts/run_analyzer/server.py @@ -36,6 +36,7 @@ Run, apply_jsonl_metadata, discover, + merge_daemon_attempts, ) STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") @@ -107,6 +108,10 @@ def _refresh_index(force: bool = False) -> None: def _find(run_id: str) -> Run: + # A resumed attempt creates a new job directory while the browser may stay + # on the existing root run. Refresh first so that poll immediately adopts + # the new physical attempt into the same logical run. + _refresh_index() with _index_lock: runs = _index for r in runs: @@ -115,41 +120,53 @@ def _find(run_id: str) -> Run: raise HTTPException(status_code=404, detail=f"run {run_id} not found") +def _freshen_physical_attempt(run: Run) -> None: + """Reload one physical attempt from its durable files.""" + from scripts.run_analyzer.sources.jsonl import parse as parse_jsonl + from scripts.run_analyzer.sources.stdout_log import parse as parse_stdout + + jl = parse_jsonl(run.jsonl_path or os.path.join(run.dir, "progress.jsonl")) + if jl is not None: + apply_jsonl_metadata(run, jl) + job_path = os.path.join(run.dir, "job.json") + try: + with open(job_path, encoding="utf-8") as handle: + job = json.load(handle) + except (OSError, json.JSONDecodeError): + job = None + if job is not None: + run.job = job + run.state = job.get("state") or run.state + run.submitted_at = job.get("submitted_at") + run.started_at = job.get("started_at") + run.ended_at = job.get("ended_at") + run.ckpt_path = job.get("ckpt_path") + run.sample_dir = job.get("sample_dir") or run.sample_dir + run.error = job.get("error") + run.stop_requested = bool(job.get("stop_requested")) + if jl is not None: + apply_jsonl_metadata(run, jl) + stdout = parse_stdout(os.path.join(run.dir, "stdout.log")) + if stdout is not None: + run.stdout = stdout + + def _freshen_run(run: Run) -> None: - """按需重解析单个 run 的 jsonl/stdout(毫秒级),不触发全量 discover。 + """按需重解析逻辑 run 的所有 attempt,不触发全量 discover。 监控 2s 轮询调 /api/runs/{id}:index 里的 Run.jsonl 是上次 discover 的 快照,progress.jsonl 是内容追加写入(文件 mtime 变、目录 mtime 不变), 必须按请求重读才能拿到最新 step。index 列表仍由 _dir_mtime_key 控制。 """ try: - from scripts.run_analyzer.sources.jsonl import parse as parse_jsonl - from scripts.run_analyzer.sources.stdout_log import parse as parse_stdout - - jl = parse_jsonl(run.jsonl_path or os.path.join(run.dir, "progress.jsonl")) - if jl is not None: - apply_jsonl_metadata(run, jl) if run.kind == "daemon": - job_path = os.path.join(run.dir, "job.json") - try: - with open(job_path, encoding="utf-8") as handle: - job = json.load(handle) - except (OSError, json.JSONDecodeError): - job = None - if job is not None: - run.job = job - run.state = job.get("state") or run.state - run.submitted_at = job.get("submitted_at") - run.started_at = job.get("started_at") - run.ended_at = job.get("ended_at") - run.ckpt_path = job.get("ckpt_path") - run.error = job.get("error") - run.stop_requested = bool(job.get("stop_requested")) - if jl is not None: - apply_jsonl_metadata(run, jl) - so = parse_stdout(os.path.join(run.dir, "stdout.log")) - if so is not None: - run.stdout = so + if run.attempts: + for attempt in run.attempts: + _freshen_physical_attempt(attempt) + refreshed = merge_daemon_attempts(run.attempts) + run.__dict__.update(refreshed.__dict__) + else: + _freshen_physical_attempt(run) run.sources = { "jsonl": run.jsonl is not None, "tensorboard": run.tb is not None, @@ -176,6 +193,7 @@ def api_runs(): @app.get("/api/runs/{run_id}/overview") def api_run_overview(run_id: str): r = _find(run_id) + _freshen_run(r) return analyze.overview_payload(r) @@ -206,6 +224,7 @@ def api_live(run_id: str, since: float = 0.0): @app.get("/api/runs/{run_id}/stdout") def api_stdout(run_id: str, tail: int = Query(400, ge=1, le=20000), q: str = ""): r = _find(run_id) + _freshen_run(r) so = r.stdout if so is None: return {"lines": [], "warnings": [], "errors": []} @@ -233,15 +252,29 @@ def api_compare(ids: list[str] = Body(..., embed=True)): @app.get("/api/runs/{run_id}/samples/{fn}") -def api_sample_file(run_id: str, fn: str): +def api_sample_file(run_id: str, fn: str, attempt_id: Optional[str] = None): r = _find(run_id) - sdir = r.sample_dir - if not sdir or not os.path.isdir(sdir): - raise HTTPException(status_code=404, detail="no sample dir") - p = os.path.normpath(os.path.join(sdir, fn)) - if not p.startswith(os.path.normpath(sdir)) or not os.path.isfile(p): + if os.path.basename(fn) != fn or fn in {".", ".."}: raise HTTPException(status_code=404, detail="not found") - return FileResponse(p) + attempts = r.attempts or [r] + if attempt_id: + attempts = [attempt for attempt in attempts if attempt.id == attempt_id] + if not attempts: + raise HTTPException(status_code=404, detail="attempt not found") + for attempt in reversed(attempts): + sdir = attempt.sample_dir + if not sdir or not os.path.isdir(sdir): + continue + root = os.path.realpath(sdir) + path = os.path.realpath(os.path.join(root, fn)) + try: + if os.path.commonpath([root, path]) != root: + continue + except ValueError: + continue + if os.path.isfile(path): + return FileResponse(path) + raise HTTPException(status_code=404, detail="not found") app.mount("/", StaticFiles(directory=STATIC_DIR, html=True), name="static") diff --git a/scripts/run_analyzer/static/css/base.css b/scripts/run_analyzer/static/css/base.css index f183b0de0..0f9c68f5c 100644 --- a/scripts/run_analyzer/static/css/base.css +++ b/scripts/run_analyzer/static/css/base.css @@ -189,20 +189,65 @@ body.rail-open .main { margin-left: var(--rail-w-open); } .kpi-grid { grid-template-columns: repeat(2, 1fr); } .view { padding: 32px 24px 80px; } .rail { - inset: auto 0 auto 0; width: 100%; height: 56px; + inset: 0 0 auto 0; width: 100%; height: 56px; border-right: none; border-bottom: var(--line); } body.rail-open .rail { width: 100%; height: auto; } .rail-inner { width: 100%; height: auto; flex-direction: row; align-items: center; } .rail-brand { height: 56px; border-bottom: none; padding: 0 16px; align-items: center; } - .rail-nav { flex: 1; display: flex; padding: 0; } - .rail-item { height: 56px; border-left: none; border-bottom: 2px solid transparent; padding: 0 16px; } + .rail-nav { + flex: 1; min-width: 0; display: flex; padding: 0; + overflow-x: auto; overflow-y: hidden; scrollbar-width: none; + } + .rail-nav::-webkit-scrollbar { display: none; } + .rail-item { flex: none; height: 56px; border-left: none; border-bottom: 2px solid transparent; padding: 0 16px; } .rail-item.active { border-bottom-color: var(--ink); background: none; } .rail-foot { display: none; } .main { margin-left: 0; margin-top: 56px; } body.rail-open .main { margin-left: 0; } } +@media (max-width: 600px) { + .view { padding: 24px 16px 64px; } + + .rail-brand { flex: none; padding: 0 12px; } + .rb-mark { letter-spacing: 0.12em; } + .rb-mark::before { margin-right: 4px; } + .rail-item { gap: 8px; padding: 0 12px; } + + .masthead { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + gap: 10px 14px; + padding-bottom: 20px; + margin-bottom: 20px; + } + .mh-index { font-size: var(--ts-xl); } + .mh-title { + font-size: var(--ts-3xl); + letter-spacing: 0; + white-space: nowrap; + } + .mh-sub { font-size: var(--ts-sm); } + .mh-deco { display: none; } + + .index-tools { + align-items: stretch; + flex-direction: column; + gap: 12px; + } + .search-box { max-width: none; width: 100%; } + .chips { + max-width: 100%; + overflow-x: auto; + overflow-y: hidden; + scrollbar-width: thin; + } + .chip { flex: none; } + + .toast { width: calc(100% - 32px); max-width: 420px; } +} + /* ═══════════════ 减少动态偏好:关闭全部动画 ═══════════════ */ @media (prefers-reduced-motion: reduce) { *, *::before, *::after { diff --git a/scripts/run_analyzer/static/css/view-detail.css b/scripts/run_analyzer/static/css/view-detail.css index 264a48fb4..af352cba4 100644 --- a/scripts/run_analyzer/static/css/view-detail.css +++ b/scripts/run_analyzer/static/css/view-detail.css @@ -222,3 +222,44 @@ } .stdout-body .l-warn { color: var(--volt); } .stdout-body .l-err { color: var(--volt); font-weight: 700; } + +@media (max-width: 1024px) { + .sec-nav { top: 56px; } + .ov-grid, .ev-grid { grid-template-columns: minmax(0, 1fr); } + .kpi-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } +} + +@media (max-width: 600px) { + .detail-top { margin-bottom: 12px; } + .detail-head { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + gap: 10px 12px; + padding-bottom: 20px; + } + .dh-index { font-size: var(--ts-xl); } + .dh-title { font-size: var(--ts-2xl); overflow-wrap: anywhere; } + .dh-meta { font-size: var(--ts-micro); overflow-wrap: anywhere; } + .dh-badges { + grid-column: 1 / -1; + margin-left: 0; + align-items: flex-start; + } + .badge-row { justify-content: flex-start; } + + .sec-nav { margin-bottom: 28px; } + .sec-nav a { height: 46px; padding: 0 14px; } + .sec-block { margin-bottom: 48px; } + .sec-head { align-items: flex-start; flex-wrap: wrap; gap: 8px 12px; } + .sec-sub { width: 100%; margin-left: 0; } + + .epoch-table-wrap { max-width: 100%; overflow-x: auto; } + .epoch-table { min-width: 620px; } + .param-tools { align-items: stretch; flex-direction: column; } + .gallery { grid-template-columns: minmax(0, 1fr); } + .g-item { min-width: 0; } + .ev-row { align-items: flex-start; flex-wrap: wrap; } + .ev-msg { flex-basis: 100%; } + .stdout-head { align-items: stretch; flex-direction: column; } + .stdout-head .search-box { max-width: none; } +} diff --git a/scripts/run_analyzer/static/css/view-monitor.css b/scripts/run_analyzer/static/css/view-monitor.css index ab06590a4..5373bc5d3 100644 --- a/scripts/run_analyzer/static/css/view-monitor.css +++ b/scripts/run_analyzer/static/css/view-monitor.css @@ -201,3 +201,18 @@ .mon-grid { grid-template-columns: 1fr; } .mon-subrow { flex-direction: column; } } + +@media (max-width: 600px) { + .mon-bar { align-items: stretch; gap: 10px; } + .mon-dd, .mon-dd-trigger { width: 100%; max-width: 100%; } + .mon-dd-trigger .mdd-name { max-width: 160px; } + .mon-dd-panel { + min-width: 0; + width: calc(100vw - 32px); + max-width: calc(100vw - 32px); + } + .mon-bar-progress { flex-basis: 100%; min-width: 100%; } + .mon-kpi-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .mon-side { min-height: 0; } + .mon-feed-head { align-items: flex-start; flex-direction: column; gap: 4px; } +} diff --git a/scripts/run_analyzer/static/css/view-runs.css b/scripts/run_analyzer/static/css/view-runs.css index ce99c4e43..d06e8e701 100644 --- a/scripts/run_analyzer/static/css/view-runs.css +++ b/scripts/run_analyzer/static/css/view-runs.css @@ -1,7 +1,12 @@ /* view-runs.css — 档案索引页 */ -.table-wrap { border-top: var(--line); } -.runs-table { width: 100%; border-collapse: collapse; } +.table-wrap { + max-width: 100%; + border-top: var(--line); + overflow-x: auto; + overscroll-behavior-x: contain; +} +.runs-table { width: 100%; min-width: 980px; border-collapse: collapse; } .runs-table th { font-family: var(--font-mono); font-size: var(--ts-micro); @@ -71,3 +76,15 @@ .cmp-bar .cb-count b { color: var(--volt); font-weight: 500; } .cmp-bar .text-btn { color: var(--g5); } .cmp-bar .text-btn:hover { color: var(--paper); border-bottom-color: var(--paper); } + +@media (max-width: 600px) { + .runs-table { min-width: 920px; } + .runs-table th { padding: 10px 8px; } + .runs-table td { padding: 12px 8px; } + .ta-run { min-width: 230px; } + .cmp-bar { + width: calc(100% - 32px); + justify-content: space-between; + gap: 10px; + } +} diff --git a/scripts/run_analyzer/static/index.html b/scripts/run_analyzer/static/index.html index 949db9e9d..267bc9c52 100644 --- a/scripts/run_analyzer/static/index.html +++ b/scripts/run_analyzer/static/index.html @@ -6,11 +6,11 @@ run-analyzer — 训练档案 - + - - - + + + diff --git a/scripts/run_analyzer/static/js/components/lightbox.js b/scripts/run_analyzer/static/js/components/lightbox.js index eb86eb84f..1a673114c 100644 --- a/scripts/run_analyzer/static/js/components/lightbox.js +++ b/scripts/run_analyzer/static/js/components/lightbox.js @@ -10,12 +10,20 @@ function _metaBase() { return `${fn} · epoch ${(_cur && _cur.epoch) ?? '—'} · step ${(_cur && _cur.global_step) ?? '—'}`; } +export function sampleUrl(s, runId) { + const fn = s && s.path ? s.path.split('/').pop() : ''; + const params = new URLSearchParams(); + if (s && s.attempt_id) params.set('attempt_id', s.attempt_id); + const query = params.toString(); + return `/api/runs/${encodeURIComponent(runId)}/samples/${encodeURIComponent(fn)}${query ? `?${query}` : ''}`; +} + export function openLightbox(s, runId) { _cur = s; _curRunId = runId; const size = s && s.w && s.h ? ` · ${s.w}×${s.h}` : ''; $('#lb-meta').textContent = _metaBase() + size; - $('#lb-img').src = `/api/runs/${encodeURIComponent(runId)}/samples/${encodeURIComponent(s.path ? s.path.split('/').pop() : '')}`; + $('#lb-img').src = sampleUrl(s, runId); $('#lb-prompt').textContent = (s && s.prompt) || ''; $('#lightbox').classList.remove('hidden'); $('#lb-close').focus(); diff --git a/scripts/run_analyzer/static/js/views/detail.js b/scripts/run_analyzer/static/js/views/detail.js index 2a9d09fec..280e100fe 100644 --- a/scripts/run_analyzer/static/js/views/detail.js +++ b/scripts/run_analyzer/static/js/views/detail.js @@ -40,6 +40,7 @@ function renderDetailHead() { ${esc(detail.method || '—')} PRESET ${esc(detail.preset || '—')} ${st.label} + ${detail.attempt_count > 1 ? `已恢复 ${detail.attempt_count - 1} 次` : ''}
${['jsonl', 'tensorboard', 'snapshot', 'stdout'].filter(k => detail.sources[k]).map(k => `${k.toUpperCase()}`).join('')} @@ -103,6 +104,8 @@ function renderOverview() { ]; const meta = [ ['job id', detail.id], + ['current job', detail.current_job_id || detail.id], + ['attempts', String(detail.attempt_count || 1)], ['kind', detail.kind], ['state', detail.state + (detail.stop_requested ? ' (stop requested)' : '')], ['started', fmtTs(detail.started_at)], diff --git a/scripts/run_analyzer/static/js/views/monitor.js b/scripts/run_analyzer/static/js/views/monitor.js index d5d84d6dc..e7e9fd1ed 100644 --- a/scripts/run_analyzer/static/js/views/monitor.js +++ b/scripts/run_analyzer/static/js/views/monitor.js @@ -5,7 +5,7 @@ import { ST } from '../theme.js'; import { chartRegistry, mountChart, preserveZoom, zoomTo } from '../core/charts.js'; import { lossChartOption } from '../charts/loss.js'; import { lrChartOption } from '../charts/lr.js'; -import { openLightbox } from '../components/lightbox.js'; +import { openLightbox, sampleUrl } from '../components/lightbox.js'; import { createDropdown } from '../components/dropdown.js'; import { loadState, saveState } from '../core/state.js'; @@ -225,7 +225,7 @@ function renderMonFilm() { const ar = s.w && s.h ? ` style="--ar: ${s.w} / ${s.h}"` : ''; const size = s.w && s.h ? `${s.w}×${s.h}` : ''; return `
- sample + sample
${s.global_step != null ? `step ${s.global_step}` : ''}${size}
`; }).join('') || '
无采样
'; diff --git a/scripts/run_analyzer/static/js/views/runs.js b/scripts/run_analyzer/static/js/views/runs.js index c6b08d36d..fe11efcc9 100644 --- a/scripts/run_analyzer/static/js/views/runs.js +++ b/scripts/run_analyzer/static/js/views/runs.js @@ -109,7 +109,7 @@ export function renderRuns() { tr.innerHTML = ` ${String(i + 1).padStart(2, '0')} - ${esc(r.run_name)}${esc(r.id)} + ${esc(r.run_name)}${r.attempt_count > 1 ? ` · R+${r.attempt_count - 1}` : ''}${esc(r.id)} ${st.label} ${esc(r.method || '—')} · ${esc(r.preset || '—')} ${r.actual_epochs != null ? `${r.actual_epochs}/${r.total_epochs ?? '?'}` : '—'} diff --git a/scripts/run_analyzer/static/js/views/samples.js b/scripts/run_analyzer/static/js/views/samples.js index a7dccfb82..857be966e 100644 --- a/scripts/run_analyzer/static/js/views/samples.js +++ b/scripts/run_analyzer/static/js/views/samples.js @@ -1,7 +1,7 @@ /* views/samples.js — 采样画廊 */ import { $, $$, esc } from '../core/dom.js'; -import { openLightbox } from '../components/lightbox.js'; +import { openLightbox, sampleUrl } from '../components/lightbox.js'; import { emptyHtml } from '../components/states.js'; export function renderSamples(d) { @@ -13,7 +13,7 @@ export function renderSamples(d) { const ar = s.w && s.h ? ` style="--ar: ${s.w} / ${s.h}"` : ''; const size = s.w && s.h ? `${s.w}×${s.h}` : ''; return `
- sample ${fn} + sample ${fn}
${esc(fn)}${size}${s.global_step ? `step ${s.global_step}` : ''}
`; }).join('')}
` : emptyHtml('NO SAMPLES — 无采样记录')}`; diff --git a/scripts/tasks/staged_resolution.py b/scripts/tasks/staged_resolution.py index f3bf5d372..59e053570 100644 --- a/scripts/tasks/staged_resolution.py +++ b/scripts/tasks/staged_resolution.py @@ -32,6 +32,11 @@ def _profile_arg( forwarded: list[str] = [] allowed = {"--progress_jsonl", "--sample_dir"} + if allow_training_flags: + # Resume jobs receive an immutable config snapshot and explicit state + # path from the daemon. Keep these flags out of preprocessing, which + # has no training-state contract. + allowed |= {"--config_file", "--resume"} while args: flag = args.pop(0) if flag not in allowed or not args: @@ -145,8 +150,27 @@ def cmd_staged_preprocess(extra): def cmd_staged_train(extra): """Train from a validated, generated full config for a saved profile.""" name, forwarded = _profile_arg(extra, allow_training_flags=True) - plan = load_profile(name) - require_ready(name, plan) - config_path = compile_runtime_config(name, plan) + config_override = None + clean_forwarded: list[str] = [] + index = 0 + while index < len(forwarded): + token = forwarded[index] + if token == "--config_file" and index + 1 < len(forwarded): + config_override = forwarded[index + 1] + index += 2 + continue + clean_forwarded.append(token) + if token in {"--resume", "--progress_jsonl", "--sample_dir"} and index + 1 < len(forwarded): + clean_forwarded.append(forwarded[index + 1]) + index += 2 + continue + index += 1 + + if config_override: + config_path = config_override + else: + plan = load_profile(name) + require_ready(name, plan) + config_path = compile_runtime_config(name, plan) print(f"[staged-resolution] training profile={name} config={config_path}") - accelerate_launch("--config_file", str(config_path), *forwarded) + accelerate_launch("--config_file", str(config_path), *clean_forwarded) diff --git a/start-webui-linux.sh b/start-webui-linux.sh index 4ee158da0..176c15491 100755 --- a/start-webui-linux.sh +++ b/start-webui-linux.sh @@ -32,8 +32,8 @@ trap cleanup EXIT INT TERM sleep 3 if command -v xdg-open &>/dev/null; then - xdg-open "$WEBUI_URL" 2>/dev/null || true - xdg-open "$ANALYZER_URL" 2>/dev/null || true + setsid xdg-open "$WEBUI_URL" 2>/dev/null || true + setsid xdg-open "$ANALYZER_URL" 2>/dev/null || true elif command -v open &>/dev/null; then open "$WEBUI_URL" 2>/dev/null || true open "$ANALYZER_URL" 2>/dev/null || true diff --git a/tests/test_daemon.py b/tests/test_daemon.py index 64e1ec3b3..fa65a503d 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -83,6 +83,20 @@ def test_job_roundtrip(tmp_path, monkeypatch): assert loaded["j1"].overrides == {"network_dim": 16} +def test_filtered_history_can_page_newest_first(): + manager = JobManager.__new__(JobManager) + manager._lock = threading.RLock() + manager._jobs = { + "old": jobs.Job(id="old", method="old", preset="", submitted_at=1.0), + "new": jobs.Job(id="new", method="new", preset="", submitted_at=2.0), + } + + page, total = manager.list_jobs_filtered(offset=0, limit=1, newest_first=True) + + assert total == 2 + assert [job.id for job in page] == ["new"] + + def test_liveness_pid_create_time(): me = os.getpid() ct = proc.create_time(me) @@ -651,6 +665,293 @@ def test_reconcile_orphan_requeue_adopt(tmp_path, monkeypatch): assert "live" in mgr._adopt # re-attached for monitoring +def test_resume_selects_newest_complete_state_with_matching_run_signatures( + tmp_path, monkeypatch +): + from library.training.state import build_train_state, write_complete_marker + + monkeypatch.setattr(config, "STATE_DIR", tmp_path / "daemon") + monkeypatch.setattr(config, "JOBS_DIR", tmp_path / "daemon" / "jobs") + output_dir = tmp_path / "ckpt" + output_root = output_dir / "unit" + snapshot = tmp_path / "source.snapshot.toml" + snapshot.write_text( + f'output_dir = "{output_dir.as_posix()}"\n' + 'output_name = "unit"\n' + "max_train_steps = 100\n", + encoding="utf-8", + ) + + def write_state(name, step, config_signature, dataset_signature, *, complete=True): + path = output_root / name + path.mkdir(parents=True) + (path / "train_state.json").write_text( + json.dumps( + build_train_state( + global_step=step, + config_signature=config_signature, + dataset_signature=dataset_signature, + ) + ), + encoding="utf-8", + ) + if complete: + write_complete_marker(path) + return path + + matching = write_state("unit-rolling-state", 75, "cfg-a", "data-a") + write_state("unit-checkpoint-state", 90, "cfg-other", "data-a") + write_state("unit-interrupted-state", 95, "cfg-a", "data-a", complete=False) + + source = jobs.Job( + id="source", + method="lora", + preset="default", + state=jobs.STATE_ERROR, + config_file=str(snapshot), + progress_path=str(tmp_path / "source.progress.jsonl"), + target_steps=100, + ) + with open(source.progress_path, "w", encoding="utf-8") as stream: + stream.write( + json.dumps( + { + "ev": "run_start", + "config_signature": "cfg-a", + "dataset_signature": "data-a", + } + ) + + "\n" + ) + + manager = JobManager() + manager._jobs[source.id] = source + manager._refresh_recovery_metadata(source) + resumed = manager.resume_job(source.id) + + assert source.config_signature == "cfg-a" + assert source.dataset_signature == "data-a" + assert source.recovery_state == str(matching) + assert source.recovery_step == 75 + assert resumed is not None + assert resumed.recovery_state == str(matching) + assert resumed.recovery_step == 75 + assert resumed.extra[-2:] == ["--resume", str(matching)] + assert resumed.root_job_id == source.id + assert resumed.parent_job_id == source.id + assert resumed.attempt_index == 1 + group = manager.job_group(source.id) + assert group is not None + assert group["id"] == source.id + assert group["current_job_id"] == resumed.id + assert group["attempt_count"] == 2 + + with pytest.raises(ValueError, match="only the latest attempt"): + manager.resume_job(source.id) + + +def test_resume_discovers_raw_output_name_inside_safe_output_root(tmp_path, monkeypatch): + from library.training.state import build_train_state, write_complete_marker + + monkeypatch.setattr(config, "STATE_DIR", tmp_path / "daemon") + monkeypatch.setattr(config, "JOBS_DIR", tmp_path / "daemon" / "jobs") + output_dir = tmp_path / "ckpt" + output_root = output_dir / "Tlora-_artist" + state_dir = output_root / "Tlora-@artist-checkpoint-state" + state_dir.mkdir(parents=True) + (state_dir / "train_state.json").write_text( + json.dumps( + build_train_state( + global_step=840, + config_signature="cfg-special", + dataset_signature="data-special", + ) + ), + encoding="utf-8", + ) + write_complete_marker(state_dir) + snapshot = tmp_path / "special.snapshot.toml" + snapshot.write_text( + f'output_dir = "{output_dir.as_posix()}"\n' + 'output_name = "Tlora-@artist"\n' + "max_train_epochs = 12\n", + encoding="utf-8", + ) + job = jobs.Job( + id="special", + method="lora-gui", + preset="", + kind="command", + argv=["tasks.py", "lora-gui", "tlora"], + state=jobs.STATE_ERROR, + config_file=str(snapshot), + config_signature="cfg-special", + dataset_signature="data-special", + ) + + candidates = JobManager._state_candidates(job) + + assert candidates == [(840, state_dir)] + + +def test_job_groups_filter_latest_attempt_and_delete_whole_chain(tmp_path, monkeypatch): + monkeypatch.setattr(config, "STATE_DIR", tmp_path / "daemon") + monkeypatch.setattr(config, "JOBS_DIR", tmp_path / "daemon" / "jobs") + manager = JobManager() + root = jobs.Job( + id="root", + method="lora", + preset="default", + state=jobs.STATE_ERROR, + root_job_id="root", + submitted_at=1.0, + ) + child = jobs.Job( + id="child", + method="lora", + preset="default", + state=jobs.STATE_DONE, + root_job_id="root", + parent_job_id="root", + attempt_index=1, + submitted_at=2.0, + ) + for job in (root, child): + manager._jobs[job.id] = job + job.persist() + + groups, total = manager.list_job_groups_filtered( + state=jobs.STATE_DONE, newest_first=True + ) + + assert total == 1 + assert [group["id"] for group in groups] == ["root"] + assert groups[0]["current_job_id"] == "child" + assert [attempt["job_id"] for attempt in groups[0]["attempts"]] == [ + "root", + "child", + ] + assert manager.delete_group("child") == ["root", "child"] + assert not root.dir.exists() + assert not child.dir.exists() + + +def test_job_groups_http_list_get_and_delete_whole_chain(daemon): + cl, manager = daemon + root = jobs.Job( + id="http-root", + method="lora", + preset="default", + state=jobs.STATE_ERROR, + root_job_id="http-root", + attempt_index=0, + submitted_at=1.0, + ) + child = jobs.Job( + id="http-child", + method="lora", + preset="default", + state=jobs.STATE_DONE, + root_job_id="http-root", + parent_job_id="http-root", + attempt_index=1, + submitted_at=2.0, + ) + with manager._lock: + for job in (root, child): + manager._jobs[job.id] = job + job.persist() + + page = cl._request("GET", "/job-groups?state=done&offset=0&limit=1") + assert page["total"] == 1 + assert page["offset"] == 0 + assert page["limit"] == 1 + assert [group["id"] for group in page["groups"]] == ["http-root"] + + detail = cl.get_job_group("http-child") + assert detail["root_job_id"] == "http-root" + assert detail["current_job_id"] == "http-child" + assert [attempt["job_id"] for attempt in detail["attempts"]] == [ + "http-root", + "http-child", + ] + + deleted = cl.delete_job_group("http-child") + assert deleted == { + "ok": True, + "root_job_id": "http-root", + "job_ids": ["http-root", "http-child"], + } + assert manager.get("http-root") is None + assert manager.get("http-child") is None + assert not root.dir.exists() + assert not child.dir.exists() + + +def test_lineage_fields_survive_job_table_reload(tmp_path, monkeypatch): + monkeypatch.setattr(config, "STATE_DIR", tmp_path / "daemon") + monkeypatch.setattr(config, "JOBS_DIR", tmp_path / "daemon" / "jobs") + root = jobs.Job( + id="root", + method="lora", + preset="default", + state=jobs.STATE_ERROR, + root_job_id="root", + attempt_index=0, + ) + child = jobs.Job( + id="child", + method="lora", + preset="default", + state=jobs.STATE_DONE, + root_job_id="root", + parent_job_id="root", + attempt_index=1, + ) + root.persist() + child.persist() + + reloaded = jobs.load_all() + manager = JobManager() + manager._jobs = reloaded + group = manager.job_group("child") + + assert group is not None + assert group["id"] == "root" + assert group["current_job_id"] == "child" + assert [attempt["attempt_index"] for attempt in group["attempts"]] == [0, 1] + assert reloaded["child"].parent_job_id == "root" + + +def test_resume_rejects_lineage_with_active_attempt(tmp_path, monkeypatch): + monkeypatch.setattr(config, "STATE_DIR", tmp_path / "daemon") + monkeypatch.setattr(config, "JOBS_DIR", tmp_path / "daemon" / "jobs") + manager = JobManager() + root = jobs.Job( + id="root", + method="lora", + preset="default", + state=jobs.STATE_ERROR, + root_job_id="root", + attempt_index=0, + ) + active = jobs.Job( + id="active", + method="lora", + preset="default", + state=jobs.STATE_RUNNING, + root_job_id="root", + parent_job_id="root", + attempt_index=1, + ) + manager._jobs = {root.id: root, active.id: active} + + with pytest.raises(ValueError, match="only the latest attempt"): + manager.resume_job(root.id) + assert manager.resume_job(active.id) is None + assert set(manager._jobs) == {"root", "active"} + + def test_command_job_build_cmd(): """A `kind="command"` job builds a plain `python ` call (no accelerate launch) and merges its extra_env over the inherited env.""" @@ -676,6 +977,58 @@ def test_command_job_build_cmd(): assert env["TQDM_MININTERVAL"] == "10" +def test_resumed_training_command_uses_immutable_config_snapshot(): + job = jobs.Job( + id="resume-command", + method="lora-gui", + preset="", + kind="command", + argv=["tasks.py", "lora-gui", "lokr", "--resume", "/tmp/state"], + config_file="/tmp/job/config.snapshot.toml", + recovery_state="/tmp/state", + progress_path="/tmp/job/progress.jsonl", + sample_dir="/tmp/job/sample", + ) + mgr = JobManager.__new__(JobManager) + + cmd, _env = mgr._build_cmd(job) + + assert cmd.count("--config_file") == 1 + assert cmd[cmd.index("--config_file") + 1] == job.config_file + assert cmd.count("--resume") == 1 + + +def test_resumed_staged_training_forwards_snapshot_resume_and_daemon_paths(): + """Staged training must use the pinned runtime config on resume. + + The command wrapper accepts only the profile plus daemon-injected + ``--config_file``/``--resume``/telemetry paths. This guards the recovery + path against silently recompiling a profile that may have changed after the + original job was submitted. + """ + job = jobs.Job( + id="resume-staged", + method="staged-train", + preset="", + kind="command", + argv=["tasks.py", "staged-train", "default", "--resume", "/tmp/state"], + config_file="/tmp/job/config.snapshot.toml", + recovery_state="/tmp/state", + progress_path="/tmp/job/progress.jsonl", + sample_dir="/tmp/job/sample", + ) + mgr = JobManager.__new__(JobManager) + + cmd, _env = mgr._build_cmd(job) + + assert cmd.count("--config_file") == 1 + assert cmd[cmd.index("--config_file") + 1] == job.config_file + assert cmd.count("--resume") == 1 + assert cmd[cmd.index("--resume") + 1] == "/tmp/state" + assert cmd[cmd.index("--progress_jsonl") + 1] == job.progress_path + assert cmd[cmd.index("--sample_dir") + 1] == job.sample_dir + + def test_command_job_loads_with_train_default(): """A legacy job.json (written before `kind` existed) loads as a train job.""" job = jobs.Job.from_dict({"id": "old", "method": "lora", "preset": "default"}) diff --git a/tests/test_preview.py b/tests/test_preview.py index d8909714a..86ccb71c8 100644 --- a/tests/test_preview.py +++ b/tests/test_preview.py @@ -379,6 +379,54 @@ def test_sample_file_with_spaces_and_cjk_is_served(preview_env): assert resp.status_code == 200 +def test_logical_task_samples_select_exact_attempt(preview_env): + """Same-named samples remain independently addressable across attempts.""" + svc, tmp = preview_env + from fastapi import HTTPException + + from webui.api.preview import get_sample_file, list_task_samples + from webui.services.task_service import Task, TaskState + + root_dir = _mkdir(tmp / "output" / "daemon" / "jobs" / "root" / "sample") + child_dir = _mkdir(tmp / "output" / "daemon" / "jobs" / "child" / "sample") + filename = "same.png" + (root_dir / filename).write_bytes(b"root-attempt") + (child_dir / filename).write_bytes(b"child-attempt") + + task = Task(id="root", command="lora", args=[], state=TaskState.FAILED) + task.job_id = "child" + task.sample_dir = "output/daemon/jobs/child/sample" + task.attempts = [ + { + "id": "root", + "attempt_index": 0, + "sample_dir": "output/daemon/jobs/root/sample", + }, + { + "id": "child", + "attempt_index": 1, + "sample_dir": "output/daemon/jobs/child/sample", + }, + ] + svc._tasks[task.id] = task + + listing = list_task_samples("root", page=1, page_size=60) + assert listing.total == 2 + assert {(item.attempt_id, item.path) for item in listing.items} == { + ("root", filename), + ("child", filename), + } + + root_response = get_sample_file("root", path=filename, attempt_id="root") + child_response = get_sample_file("root", path=filename, attempt_id="child") + assert Path(root_response.path).read_bytes() == b"root-attempt" + assert Path(child_response.path).read_bytes() == b"child-attempt" + + with pytest.raises(HTTPException) as exc: + get_sample_file("root", path=filename, attempt_id="missing") + assert exc.value.status_code == 404 + + @pytest.mark.parametrize( "bad_path", [ diff --git a/tests/test_progress_sink.py b/tests/test_progress_sink.py index 52e4e0bd8..9cef87e4a 100644 --- a/tests/test_progress_sink.py +++ b/tests/test_progress_sink.py @@ -43,7 +43,13 @@ def test_resolve_path_explicit_override(tmp_path): def test_full_lifecycle_schema(tmp_path): path = str(tmp_path / "run.progress.jsonl") sink = ProgressSink(path, run="run", method="lora", preset="default", t0=0.0) - sink.run_start(total_steps=100, total_epochs=4, pid=4242) + sink.run_start( + total_steps=100, + total_epochs=4, + pid=4242, + config_signature="cfg-1", + dataset_signature="data-1", + ) sink.log({"loss": 0.5, "lr": 1e-4}, global_step=10, epoch=1) sink.log({"loss/val_average": 0.03, "loss/val_cmmd": 0.03}, global_step=10, epoch=1) sink.ckpt(global_step=10, path="output/ckpt/run-step10.safetensors") @@ -56,6 +62,8 @@ def test_full_lifecycle_schema(tmp_path): start = evs[0] assert start["run"] == "run" and start["method"] == "lora" assert start["total_steps"] == 100 and start["pid"] == 4242 + assert start["config_signature"] == "cfg-1" + assert start["dataset_signature"] == "data-1" step = evs[1] assert step["global_step"] == 10 and step["epoch"] == 1 and step["loss"] == 0.5 diff --git a/tests/test_run_analyzer.py b/tests/test_run_analyzer.py index a9fd54fde..ddc586999 100644 --- a/tests/test_run_analyzer.py +++ b/tests/test_run_analyzer.py @@ -6,8 +6,10 @@ from pathlib import Path import pytest +from fastapi import HTTPException -from scripts.run_analyzer import discovery, server +from scripts.run_analyzer import analyze, discovery, server +from scripts.run_analyzer.sources.tensorboard import TbRun def _write_jsonl(path: Path, *events: dict) -> None: @@ -109,6 +111,140 @@ def test_freshen_run_reloads_job_and_maps_run_end_to_terminal(tmp_path) -> None: assert run.jsonl is not None and run.jsonl.run_end_final_step == 4 +def test_daemon_resume_attempts_are_one_logical_run(tmp_path, monkeypatch) -> None: + jobs_dir = tmp_path / "jobs" + logs_dir = tmp_path / "logs" + root_id = "20260811-161531-98bd8a" + child_id = "20260811-180000-deadbe" + root_dir = jobs_dir / root_id + child_dir = jobs_dir / child_id + for path in (root_dir, child_dir): + (path / "sample").mkdir(parents=True) + logs_dir.mkdir() + + (root_dir / "job.json").write_text( + json.dumps( + { + "id": root_id, + "kind": "train", + "method": "lora", + "state": "error", + "root_job_id": root_id, + "parent_job_id": None, + "attempt_index": 0, + "submitted_at": 90.0, + "started_at": 100.0, + "ended_at": 120.0, + "rc": 1, + "sample_dir": str(root_dir / "sample"), + "error": "first attempt failed", + } + ), + encoding="utf-8", + ) + (child_dir / "job.json").write_text( + json.dumps( + { + "id": child_id, + "kind": "train", + "method": "lora", + "state": "done", + "root_job_id": root_id, + "parent_job_id": root_id, + "attempt_index": 1, + "submitted_at": 190.0, + "started_at": 200.0, + "ended_at": 220.0, + "rc": 0, + "recovery_step": 2, + "sample_dir": str(child_dir / "sample"), + } + ), + encoding="utf-8", + ) + _write_jsonl( + root_dir / "progress.jsonl", + {"ev": "run_start", "run": "lineage", "method": "lora", "total_steps": 3}, + {"ev": "step", "ts": 1.0, "global_step": 1, "epoch": 1, "loss/current": 0.9}, + {"ev": "step", "ts": 2.0, "global_step": 2, "epoch": 1, "loss/current": 0.8}, + {"ev": "sample", "ts": 10.0, "global_step": 2, "path": str(root_dir / "sample" / "same.png")}, + {"ev": "run_end", "ts": 20.0, "status": "error", "final_step": 2, "error": "nan"}, + ) + _write_jsonl( + child_dir / "progress.jsonl", + {"ev": "run_start", "run": "lineage", "method": "lora", "total_steps": 3}, + {"ev": "step", "ts": 1.0, "global_step": 2, "epoch": 1, "loss/current": 0.7}, + {"ev": "step", "ts": 2.0, "global_step": 3, "epoch": 1, "loss/current": 0.6}, + {"ev": "sample", "ts": 5.0, "global_step": 3, "path": str(child_dir / "sample" / "same.png")}, + {"ev": "run_end", "ts": 20.0, "status": "ok", "final_step": 3}, + ) + (root_dir / "stdout.log").write_text("root output\n", encoding="utf-8") + (child_dir / "stdout.log").write_text("child output\n", encoding="utf-8") + (root_dir / "sample" / "same.png").write_bytes(b"root") + (child_dir / "sample" / "same.png").write_bytes(b"child") + monkeypatch.setattr(discovery, "MONADFORGE_ROOT", str(tmp_path)) + monkeypatch.setattr(discovery, "JOBS_DIR", str(jobs_dir)) + monkeypatch.setattr(discovery, "LOGS_DIR", str(logs_dir)) + + runs = discovery.discover() + + assert len(runs) == 1 + run = runs[0] + assert run.id == root_id + assert [attempt.id for attempt in run.attempts] == [root_id, child_id] + assert run.state == "done" + assert run.jsonl is not None + assert run.jsonl.series["loss/current"] == [[1, 0.9], [2, 0.7], [3, 0.6]] + assert [sample["ts"] for sample in run.jsonl.samples] == [10.0, 105.0] + assert [sample["attempt_id"] for sample in run.jsonl.samples] == [root_id, child_id] + assert run.stdout is not None + assert run.stdout.lines == [ + f"[attempt 1/2 - {root_id}]", + "root output", + f"[attempt 2/2 - {child_id} from step 2]", + "child output", + ] + + payload = analyze.full_payload(run) + assert payload["id"] == root_id + assert payload["current_job_id"] == child_id + assert payload["attempt_count"] == 2 + assert [sample["attempt_id"] for sample in payload["samples"]] == [root_id, child_id] + + monkeypatch.setattr(server, "_index", [run]) + monkeypatch.setattr(server, "_refresh_index", lambda force=False: None) + root_response = server.api_sample_file(root_id, "same.png", attempt_id=root_id) + child_response = server.api_sample_file(root_id, "same.png", attempt_id=child_id) + latest_response = server.api_sample_file(root_id, "same.png") + assert Path(root_response.path).read_bytes() == b"root" + assert Path(child_response.path).read_bytes() == b"child" + assert Path(latest_response.path).read_bytes() == b"child" + with pytest.raises(HTTPException): + server.api_sample_file(root_id, "../same.png", attempt_id=root_id) + + +def test_tensorboard_overlap_prefers_newer_attempt() -> None: + root = discovery.Run( + id="root", + kind="daemon", + dir="/tmp/root", + job={"id": "root", "root_job_id": "root", "attempt_index": 0}, + tb=TbRun(series={"loss/current": [[1, 0.9], [2, 0.8]]}, steps=[1, 2]), + ) + child = discovery.Run( + id="child", + kind="daemon", + dir="/tmp/child", + job={"id": "child", "root_job_id": "root", "attempt_index": 1}, + tb=TbRun(series={"loss/current": [[2, 0.7], [3, 0.6]]}, steps=[2, 3]), + ) + + merged = discovery.merge_daemon_attempts([root, child]) + + assert merged.tb is not None + assert merged.tb.series["loss/current"] == [[1, 0.9], [2, 0.7], [3, 0.6]] + + def test_tensorboard_append_changes_index_cache_key(tmp_path, monkeypatch) -> None: jobs_dir = tmp_path / "jobs" logs_dir = tmp_path / "logs" diff --git a/tests/test_staged_resolution_plan.py b/tests/test_staged_resolution_plan.py index cd537c494..1abf8eb6f 100644 --- a/tests/test_staged_resolution_plan.py +++ b/tests/test_staged_resolution_plan.py @@ -282,6 +282,62 @@ def test_staged_task_argument_parser_rejects_browser_overrides(): _profile_arg(["default", "--network_dim", "128"], allow_training_flags=True) +def test_staged_resume_uses_pinned_config_without_recompiling_profile( + monkeypatch, tmp_path +): + from scripts.tasks import staged_resolution as task + + snapshot = tmp_path / "config.snapshot.toml" + snapshot.write_text('output_name = "pinned"\n', encoding="utf-8") + state = tmp_path / "rolling-state" + progress = tmp_path / "progress.jsonl" + samples = tmp_path / "sample" + launched: list[tuple[str, ...]] = [] + + monkeypatch.setattr( + task, + "load_profile", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("resume must not reload a mutable profile") + ), + ) + monkeypatch.setattr( + task, + "compile_runtime_config", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("resume must not recompile a mutable profile") + ), + ) + monkeypatch.setattr(task, "accelerate_launch", lambda *args: launched.append(args)) + + task.cmd_staged_train( + [ + "default", + "--config_file", + str(snapshot), + "--resume", + str(state), + "--progress_jsonl", + str(progress), + "--sample_dir", + str(samples), + ] + ) + + assert launched == [ + ( + "--config_file", + str(snapshot), + "--resume", + str(state), + "--progress_jsonl", + str(progress), + "--sample_dir", + str(samples), + ) + ] + + def test_staged_preprocess_uses_lowres_matching_and_caption_variant_flags( monkeypatch, tmp_path ): @@ -403,3 +459,39 @@ async def fake_start(_name: str, _command: str): ) assert allowed.status_code == 200 assert called == 1 + + +def test_same_origin_mutation_allows_runtime_webui_port(monkeypatch): + import library.config.io as config_io + from webui import server + from webui.api import staged_resolution as api + + called = 0 + + async def fake_start(_name: str, _command: str): + nonlocal called + called += 1 + return {"task_id": "test-task"} + + monkeypatch.setattr(config_io, "migrate_custom_configs", lambda: None) + monkeypatch.setattr(api, "_start_profile_task", fake_start) + client = TestClient(server.create_app()) + runtime_origin = "http://127.0.0.1:7460" + + allowed = client.post( + "/api/staged-resolution/profiles/default/preprocess", + headers={"Host": "127.0.0.1:7460", "Origin": runtime_origin}, + json={"version": 1}, + ) + + assert allowed.status_code == 200 + assert called == 1 + + rejected = client.post( + "/api/staged-resolution/profiles/default/preprocess", + headers={"Host": "127.0.0.1:7460", "Origin": "http://127.0.0.1:7459"}, + json={"version": 1}, + ) + + assert rejected.status_code == 403 + assert called == 1 diff --git a/tests/test_task_service_recovery.py b/tests/test_task_service_recovery.py index cacf9e66c..4649794e5 100644 --- a/tests/test_task_service_recovery.py +++ b/tests/test_task_service_recovery.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import json import pytest @@ -30,10 +31,10 @@ def test_reconcile_restores_running_daemon_job(monkeypatch): } monitor_calls: list[tuple[str, str | None]] = [] - async def list_jobs(): - return [job] + async def list_job_groups_page(**_kwargs): + return {"groups": [svc._physical_job_group(job)], "total": 1} - monkeypatch.setattr(task_module.daemon_client, "list_jobs", list_jobs) + monkeypatch.setattr(task_module.daemon_client, "list_job_groups_page", list_job_groups_page) monkeypatch.setattr( svc, "_ensure_monitors", @@ -54,6 +55,39 @@ async def list_jobs(): assert monitor_calls == [(job["id"], job["progress_path"])] +def test_reconcile_restores_stop_requested_job_as_stopping(monkeypatch): + import webui.services.task_service as task_module + from webui.services.task_service import TaskService, TaskState + + svc = TaskService() + job = { + "id": "20260714-120001-abcdef", + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lokr"], + "state": "running", + "stop_requested": True, + "pid": 4322, + "started_at": 1784001601.0, + "stdout_path": "output/daemon/jobs/job/stdout.log", + "progress_path": "output/daemon/jobs/job/progress.jsonl", + "sample_dir": "output/daemon/jobs/job/sample", + } + + async def list_job_groups_page(**_kwargs): + return {"groups": [svc._physical_job_group(job)], "total": 1} + + monkeypatch.setattr(task_module.daemon_client, "list_job_groups_page", list_job_groups_page) + monkeypatch.setattr(svc, "_ensure_monitors", lambda *_args, **_kwargs: None) + + _run(svc.reconcile_daemon_jobs()) + + restored = svc.get_task(job["id"]) + assert restored is not None + assert restored.state == TaskState.STOPPING + assert restored.info()["state"] == TaskState.STOPPING.value + + def test_ensure_monitors_is_idempotent(monkeypatch): from webui.services.task_service import Task, TaskService, TaskState @@ -106,10 +140,14 @@ def test_queue_poll_recovers_job_missed_at_startup(monkeypatch): async def list_jobs(): return [job] + async def list_job_groups_page(**_kwargs): + return {"groups": [svc._physical_job_group(job)], "total": 1} + async def health(): return {"paused": False} monkeypatch.setattr(task_module.daemon_client, "list_jobs", list_jobs) + monkeypatch.setattr(task_module.daemon_client, "list_job_groups_page", list_job_groups_page) monkeypatch.setattr(task_module.daemon_client, "health", health) monkeypatch.setattr( svc, @@ -165,6 +203,368 @@ async def no_sleep(_seconds): assert task.exit_code == 0 +def test_terminal_task_without_recovery_state_is_not_resumable(): + from webui.services.task_service import TaskService, TaskState + + task = TaskService()._task_from_daemon( + { + "id": "failed-without-state", + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "error", + "error": "process exited", + } + ) + + assert task.state == TaskState.FAILED + assert task.resumable is False + + +def test_logical_task_merges_attempt_history_and_overlapping_steps(tmp_path): + from webui.services.task_service import TaskService + + root_stdout = tmp_path / "root.stdout.log" + child_stdout = tmp_path / "child.stdout.log" + root_progress = tmp_path / "root.progress.jsonl" + child_progress = tmp_path / "child.progress.jsonl" + root_stdout.write_text("root output\n", encoding="utf-8") + child_stdout.write_text("child output\n", encoding="utf-8") + root_progress.write_text( + "\n".join( + json.dumps(event) + for event in ( + {"ev": "run_start", "total_steps": 3}, + {"ev": "step", "global_step": 1, "epoch": 1, "loss/average": 0.9}, + {"ev": "step", "global_step": 2, "epoch": 1, "loss/average": 0.8}, + {"ev": "run_end", "final_step": 2}, + ) + ), + encoding="utf-8", + ) + child_progress.write_text( + "\n".join( + json.dumps(event) + for event in ( + {"ev": "run_start", "total_steps": 3}, + {"ev": "step", "global_step": 2, "epoch": 1, "loss/average": 0.7}, + {"ev": "step", "global_step": 3, "epoch": 1, "loss/average": 0.6}, + {"ev": "run_end", "final_step": 3}, + ) + ), + encoding="utf-8", + ) + group = { + "id": "root", + "root_job_id": "root", + "current_job_id": "child", + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "done", + "attempts": [ + { + "id": "root", + "root_job_id": "root", + "attempt_index": 0, + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "error", + "stdout_path": str(root_stdout), + "progress_path": str(root_progress), + }, + { + "id": "child", + "root_job_id": "root", + "parent_job_id": "root", + "attempt_index": 1, + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "done", + "recovery_step": 2, + "rc": 0, + "stdout_path": str(child_stdout), + "progress_path": str(child_progress), + }, + ], + } + + task = TaskService()._task_from_daemon(group) + info = task.info() + + assert task.id == "root" + assert task.job_id == "child" + assert info["attempt_count"] == 2 + assert [attempt["job_id"] for attempt in info["attempts"]] == ["root", "child"] + assert task.lines == [ + "[attempt 1/2 · root]", + "root output", + "[attempt 2/2 · child from step 2]", + "child output", + ] + metrics = task.parser.metrics + assert metrics.step_history == [1, 2, 3] + assert metrics.loss_history == [0.9, 0.7, 0.6] + + +def test_resume_reuses_logical_task_and_delete_targets_group(monkeypatch): + import webui.services.task_service as task_module + from webui.services.task_service import TaskService + + svc = TaskService() + initial = { + "id": "root", + "root_job_id": "root", + "current_job_id": "root", + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "error", + "recovery_state": "/state/step-2", + "attempts": [ + { + "id": "root", + "root_job_id": "root", + "attempt_index": 0, + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "error", + "recovery_state": "/state/step-2", + } + ], + } + task = svc._task_from_daemon(initial) + svc._tasks[task.id] = task + resumed_group = { + **initial, + "current_job_id": "child", + "state": "queued", + "recovery_state": "/state/step-2", + "attempts": [ + initial["attempts"][0], + { + "id": "child", + "root_job_id": "root", + "parent_job_id": "root", + "attempt_index": 1, + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "queued", + "recovery_step": 2, + "recovery_state": "/state/step-2", + }, + ], + } + deleted: list[str] = [] + + async def resume(_job_id): + return {"root_job_id": "root", "job_id": "child"} + + async def get_job_group(_root_id): + return resumed_group + + async def delete_job_group(root_id): + deleted.append(root_id) + return {"ok": True} + + monkeypatch.setattr(task_module.daemon_client, "resume", resume) + monkeypatch.setattr(task_module.daemon_client, "get_job_group", get_job_group) + monkeypatch.setattr(task_module.daemon_client, "delete_job_group", delete_job_group) + monkeypatch.setattr(svc, "_ensure_monitors", lambda *_args, **_kwargs: None) + + resumed = _run(svc.resume_task("root")) + + assert resumed is task + assert list(svc._tasks) == ["root"] + assert task.job_id == "child" + assert task.info()["attempt_count"] == 2 + + task.state = task_module.TaskState.CANCELLED + assert _run(svc.delete_task("root")) is True + assert deleted == ["root"] + + +def test_gui_training_snapshot_pins_variant_and_manifest(monkeypatch): + import webui.services.config_service as config_service + from webui.services.task_service import TaskService + + monkeypatch.setattr( + config_service, + "merged_gui_variant_preset", + lambda variant, preset: ( + {"output_name": f"{variant}-{preset}", "max_train_steps": 100}, + {}, + ), + ) + + snapshot = TaskService._training_config_snapshot( + "lora-gui", + ["tasks.py", "lora-gui", "lokr", "--preprocess_run", "run/manifest.json"], + {"PRESET": "low_vram"}, + ) + + assert snapshot == { + "output_name": "lokr-low_vram", + "max_train_steps": 100, + "preprocess_run": "run/manifest.json", + } + + +def test_gui_epoch_only_snapshot_drops_inherited_step_default(monkeypatch): + import webui.services.config_service as config_service + from webui.services.task_service import TaskService + + monkeypatch.setattr( + config_service, + "merged_gui_variant_preset", + lambda variant, preset: ( + {"max_train_steps": 1600, "max_train_epochs": 4, "output_name": "demo"}, + {"max_train_steps": "base", "max_train_epochs": "method"}, + ), + ) + + snapshot = TaskService._training_config_snapshot( + "lora-gui", ["tasks.py", "lora-gui", "lora"], {} + ) + + assert snapshot is not None + assert "max_train_steps" not in snapshot + assert snapshot["max_train_epochs"] == 4 + + +def test_list_tasks_page_uses_daemon_total(monkeypatch): + import webui.services.task_service as task_module + from webui.services.task_service import TaskService, TaskState + + svc = TaskService() + jobs = [ + { + "id": "history-page-2", + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "error", + "rc": 1, + "status_detail": "process exited", + } + ] + + async def list_job_groups_page(**_kwargs): + return { + "groups": [svc._physical_job_group(job) for job in jobs], + "total": 37, + "offset": 25, + "limit": 25, + } + + monkeypatch.setattr(task_module.daemon_client, "list_job_groups_page", list_job_groups_page) + monkeypatch.setattr(svc, "_ensure_monitors", lambda *_args, **_kwargs: None) + + items, total = _run(svc.list_tasks_page(state="failed", limit=25, offset=25)) + + assert total == 37 + assert len(items) == 1 + assert items[0]["task_id"] == "history-page-2" + assert items[0]["state"] == TaskState.FAILED.value + + +def test_list_tasks_page_filters_legacy_unfiltered_daemon(monkeypatch): + """An older daemon may ignore the state query; keep the UI filter correct.""" + import webui.services.task_service as task_module + from webui.services.task_service import TaskService + + svc = TaskService() + unfiltered = [ + { + "id": "legacy-done", + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "done", + }, + { + "id": "legacy-running", + "kind": "command", + "method": "lora-gui", + "argv": ["tasks.py", "lora-gui", "lora"], + "state": "running", + }, + ] + calls: list[dict] = [] + + async def list_job_groups_page(**_kwargs): + from webui.services.daemon_client import DaemonError + + raise DaemonError("old daemon") + + async def list_jobs_page(**kwargs): + calls.append(kwargs) + if kwargs.get("state"): + return {"jobs": unfiltered, "total": len(unfiltered)} + return {"jobs": unfiltered, "total": len(unfiltered)} + + monkeypatch.setattr(task_module.daemon_client, "list_job_groups_page", list_job_groups_page) + monkeypatch.setattr(task_module.daemon_client, "list_jobs_page", list_jobs_page) + monkeypatch.setattr(svc, "_ensure_monitors", lambda *_args, **_kwargs: None) + + items, total = _run(svc.list_tasks_page(state="active", limit=25, offset=0)) + + assert total == 1 + assert [item["task_id"] for item in items] == ["legacy-running"] + assert calls == [ + {"state": "queued,running", "offset": 0, "limit": 25, "newest_first": True}, + ] + + +def test_direct_history_access_hydrates_missing_task(monkeypatch): + import webui.services.task_service as task_module + from webui.services.task_service import TaskService + + svc = TaskService() + + def get_job_group_sync(_task_id): + job = { + "id": "direct-history", + "kind": "command", + "method": "preprocess", + "argv": ["tasks.py", "preprocess"], + "state": "done", + "rc": 0, + "stdout_path": "/path/that/does/not/exist", + } + return svc._physical_job_group(job) + + monkeypatch.setattr(task_module.daemon_client, "get_job_group_sync", get_job_group_sync) + + info = svc.get_task_info("direct-history") + + assert info is not None + assert info["state"] == "success" + + +def test_daemon_snapshot_persists_epoch_target_without_step_default(monkeypatch, tmp_path): + from scripts.daemon import config as daemon_config + from scripts.daemon.manager import JobManager + + monkeypatch.setattr(daemon_config, "STATE_DIR", tmp_path) + monkeypatch.setattr(daemon_config, "JOBS_DIR", tmp_path / "jobs") + + manager = JobManager() + job = manager.submit_command( + label="lora-gui", + argv=["tasks.py", "lora-gui", "lora"], + config_snapshot={"max_train_epochs": 4, "output_name": "demo"}, + start=False, + ) + + assert job.target_steps is None + assert job.target_epochs == 4 + + def test_submit_failure_does_not_leave_fake_failed_task(monkeypatch): import webui.services.task_service as task_module from webui.services.daemon_client import DaemonError @@ -183,6 +583,119 @@ async def fail_submit(*_args, **_kwargs): assert svc.list_tasks() == [] +def test_cancel_keeps_daemon_poller_alive_until_stopped(monkeypatch): + """A successful stop request is not terminal until daemon teardown ends.""" + import webui.services.task_service as task_module + from webui.services.task_service import Task, TaskService, TaskState + + svc = TaskService() + task = Task( + id="stop-later", + command="lora", + args=[], + state=TaskState.RUNNING, + job_id="stop-later", + ) + svc._tasks[task.id] = task + replies = iter( + [ + {"state": "running", "pid": 123, "stdout_path": "ignored.log"}, + {"state": "stopped", "rc": 0, "stdout_path": "ignored.log"}, + ] + ) + + async def stop(_job_id): + return {"job_id": task.id, "state": "running"} + + async def get_job(_job_id): + return next(replies) + + async def no_sleep(_seconds): + return None + + async def no_drain(_task): + return None + + monkeypatch.setattr(task_module.daemon_client, "stop", stop) + monkeypatch.setattr(task_module.daemon_client, "get_job", get_job) + monkeypatch.setattr(task_module.asyncio, "sleep", no_sleep) + monkeypatch.setattr(svc, "_safe_drain_stdout", no_drain) + + async def exercise(): + assert await svc.cancel_task(task.id) is True + assert task.state == TaskState.STOPPING + await svc._poll_daemon_job(task) + + _run(exercise()) + assert task.state == TaskState.CANCELLED + assert task.exit_code == 0 + + +def test_cancel_queued_task_reaches_terminal_state(monkeypatch): + """Queued cancellation is visible immediately without a stale poller.""" + import webui.services.task_service as task_module + from webui.services.task_service import Task, TaskService, TaskState + + svc = TaskService() + task = Task( + id="stop-queued", + command="preprocess", + args=[], + state=TaskState.PENDING, + job_id="stop-queued", + ) + svc._tasks[task.id] = task + + async def stop(_job_id): + return {"job_id": task.id, "state": "stopped"} + + async def get_job(_job_id): + return {"state": "stopped", "rc": None, "stdout_path": "ignored.log"} + + async def no_drain(_task): + return None + + monkeypatch.setattr(task_module.daemon_client, "stop", stop) + monkeypatch.setattr(task_module.daemon_client, "get_job", get_job) + monkeypatch.setattr(svc, "_safe_drain_stdout", no_drain) + + async def exercise(): + assert await svc.cancel_task(task.id) is True + assert task.state == TaskState.STOPPING + await svc._poll_daemon_job(task) + + _run(exercise()) + assert task.state == TaskState.CANCELLED + + +def test_cancel_does_not_resurrect_terminal_task(monkeypatch): + """A poller completion racing the stop response remains authoritative.""" + import webui.services.task_service as task_module + from webui.services.task_service import Task, TaskService, TaskState + + svc = TaskService() + task = Task( + id="already-done", + command="lora", + args=[], + state=TaskState.RUNNING, + job_id="already-done", + ) + svc._tasks[task.id] = task + + async def stop(_job_id): + task.state = TaskState.SUCCESS + return {"job_id": task.id, "state": "done"} + + monkeypatch.setattr(task_module.daemon_client, "stop", stop) + + async def exercise(): + assert await svc.cancel_task(task.id) is True + + _run(exercise()) + assert task.state == TaskState.SUCCESS + + def test_webui_command_catalog_only_contains_real_cli_commands(): from tasks import COMMANDS from webui.services.task_catalog import COMMAND_CATALOG diff --git a/train.py b/train.py index 4923a6f83..94ae2a4f0 100644 --- a/train.py +++ b/train.py @@ -3175,6 +3175,8 @@ def _te_complete(group): pid=os.getpid(), log_dir=resolve_run_log_dir(args), sampling_enabled=sampling_enabled, + config_signature=getattr(args, "config_signature", None), + dataset_signature=getattr(args, "dataset_signature", None), ) # Mirror WARNING+ records into the stream so a reader debugging # the run gets them structured instead of buried in tqdm stdout. diff --git a/webui/api/preview.py b/webui/api/preview.py index 3928b2bc8..eaf8f6cc4 100644 --- a/webui/api/preview.py +++ b/webui/api/preview.py @@ -40,6 +40,7 @@ class SampleImage(BaseModel): + attempt_id: str path: str # relative to the task's sample dir filename: str stem: str @@ -55,7 +56,7 @@ class SampleListResponse(BaseModel): total: int -def _resolve_task_sample_dir(task_id: str) -> Path | None: +def _resolve_task_sample_dir(task_id: str, attempt_id: str | None = None) -> Path | None: """Resolve the on-disk sample directory for *task_id*. Resolution order (mirrors how ``--progress_jsonl`` is located): @@ -76,7 +77,29 @@ def _resolve_task_sample_dir(task_id: str) -> Path | None: task = task_service.get_task(task_id) sample_dir_raw: str | None = None if task is not None: - sample_dir_raw = task.sample_dir or task_service._arg_value( + if attempt_id: + if task.attempts: + attempt = next( + ( + item + for item in task.attempts + if str(item.get("id") or item.get("job_id")) == attempt_id + ), + None, + ) + if attempt is None: + return None + sample_dir_raw = attempt.get("sample_dir") + if not sample_dir_raw: + return None + else: + current_attempt_id = str(task.job_id or task.id) + if attempt_id != current_attempt_id: + return None + sample_dir_raw = task.sample_dir + else: + sample_dir_raw = task.sample_dir + sample_dir_raw = sample_dir_raw or task_service._arg_value( task.args, "--sample_dir" ) if not sample_dir_raw: @@ -87,7 +110,7 @@ def _resolve_task_sample_dir(task_id: str) -> Path | None: else: # Session-only task list lost (WebUI restarted). The daemon persists # job records (including sample_dir) to job.json — recover from there. - sample_dir_raw = _sample_dir_from_daemon_job(task_id) + sample_dir_raw = _sample_dir_from_daemon_job(task_id, attempt_id) if not sample_dir_raw: return None @@ -103,7 +126,7 @@ def _resolve_task_sample_dir(task_id: str) -> Path | None: return resolved if resolved.is_dir() else None -def _sample_dir_from_daemon_job(job_id: str) -> str | None: +def _sample_dir_from_daemon_job(job_id: str, attempt_id: str | None = None) -> str | None: """Recover ``sample_dir`` from the daemon's persisted job record. Used when the WebUI's in-memory task list no longer knows about *job_id* @@ -111,12 +134,45 @@ def _sample_dir_from_daemon_job(job_id: str) -> str | None: rather than propagating, so preview simply reports "no samples". """ try: - job = _daemon_client.get_job_sync(job_id) + group = _daemon_client.get_job_group_sync(job_id) except (DaemonError, OSError): + try: + group = _daemon_client.get_job_sync(job_id) + except (DaemonError, OSError): + return None + if not isinstance(group, dict): return None - if not isinstance(job, dict): - return None - return job.get("sample_dir") + attempts = list(group.get("attempts") or [group]) + target_id = attempt_id or str(group.get("current_job_id") or job_id) + attempt = next( + ( + item + for item in attempts + if str(item.get("id") or item.get("job_id")) == target_id + ), + None, + ) + if attempt is None and attempt_id is None and attempts: + attempt = attempts[-1] + return attempt.get("sample_dir") if attempt else None + + +def _resolve_task_sample_dirs(task_id: str) -> list[tuple[str, Path]]: + task = task_service.get_task(task_id) + if task is None: + return [] + attempts = task.attempts or [ + {"id": task.job_id or task.id, "sample_dir": task.sample_dir} + ] + out: list[tuple[str, Path]] = [] + for attempt in attempts: + attempt_id = str(attempt.get("id") or attempt.get("job_id") or "") + if not attempt_id: + continue + sample_dir = _resolve_task_sample_dir(task_id, attempt_id) + if sample_dir is not None: + out.append((attempt_id, sample_dir)) + return out def _scan_sample_dir(sample_dir: Path) -> list[Path]: @@ -143,23 +199,29 @@ def list_task_samples( if task is None: raise HTTPException(status_code=404, detail=f"Task {task_id} not found") - sample_dir = _resolve_task_sample_dir(task_id) - if sample_dir is None: + sample_dirs = _resolve_task_sample_dirs(task_id) + if not sample_dirs: return SampleListResponse(task_id=task_id, sample_dir=None, items=[], total=0) - all_files = _scan_sample_dir(sample_dir) + all_files = [ + (attempt_id, sample_dir, path) + for attempt_id, sample_dir in sample_dirs + for path in _scan_sample_dir(sample_dir) + ] + all_files.sort(key=lambda item: item[2].stat().st_mtime, reverse=True) total = len(all_files) start = (page - 1) * page_size page_files = all_files[start : start + page_size] items: list[SampleImage] = [] - for p in page_files: + for attempt_id, _sample_dir, p in page_files: try: stat = p.stat() except OSError: continue items.append( SampleImage( + attempt_id=attempt_id, path=p.name, filename=p.name, stem=p.stem, @@ -173,7 +235,7 @@ def list_task_samples( return SampleListResponse( task_id=task_id, - sample_dir=str(sample_dir), + sample_dir=str(sample_dirs[-1][1]), items=items, total=total, ) @@ -183,6 +245,7 @@ def list_task_samples( def get_sample_file( task_id: str, path: str = Query(..., description="Filename within the task's sample dir"), + attempt_id: str | None = None, ): """Stream a preview PNG by filename. @@ -196,7 +259,17 @@ def get_sample_file( if _UNSAFE_PATH.search(path): raise HTTPException(status_code=400, detail="Invalid sample path") - sample_dir = _resolve_task_sample_dir(task_id) + sample_dir = _resolve_task_sample_dir(task_id, attempt_id) + if sample_dir is None and attempt_id is None: + # Backward compatibility for clients that only send a filename: pick + # the newest matching artifact across the logical task. + matches = [] + for _candidate_attempt, candidate_dir in _resolve_task_sample_dirs(task_id): + candidate = (candidate_dir / path).resolve() + if candidate.is_file(): + matches.append((candidate.stat().st_mtime, candidate_dir)) + if matches: + sample_dir = max(matches, key=lambda item: item[0])[1] if sample_dir is None: raise HTTPException( status_code=404, diff --git a/webui/api/tasks.py b/webui/api/tasks.py index e734b3655..f6ad0aeca 100644 --- a/webui/api/tasks.py +++ b/webui/api/tasks.py @@ -2,8 +2,9 @@ from __future__ import annotations +from typing import Optional -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Response from pydantic import BaseModel, Field from webui.services.daemon_client import DaemonError @@ -45,9 +46,26 @@ class TaskCommandListResponse(BaseModel): @router.get("", response_model=list[dict]) -def list_tasks(): - """List all active tasks.""" - return task_service.list_tasks() +async def list_tasks( + response: Response, + state: Optional[str] = None, + limit: int = 100, + offset: int = 0, +): + """List one durable task-history page. + + Keep the response body backward-compatible as a list. Pagination metadata + is exposed in headers so older consumers do not need a schema migration. + """ + page, total = await task_service.list_tasks_page( + state=state, + limit=min(max(limit, 1), 500), + offset=max(offset, 0), + ) + response.headers["X-Total-Count"] = str(total) + response.headers["X-Page-Offset"] = str(max(offset, 0)) + response.headers["X-Page-Limit"] = str(min(max(limit, 1), 500)) + return page @router.get("/commands", response_model=TaskCommandListResponse) @@ -91,6 +109,7 @@ async def queue_resume(): class DaemonShutdownRequest(BaseModel): kill_jobs: bool = True + mode: Optional[str] = None @router.post("/daemon/shutdown") @@ -101,8 +120,13 @@ async def daemon_shutdown(body: DaemonShutdownRequest): as a sidecar — its shutdown tree-kills this server too. The shutdown has still been triggered in that case. """ + if body.mode is not None and body.mode not in {"detach", "cooperative-stop", "force"}: + raise HTTPException( + status_code=400, + detail="shutdown mode must be detach, cooperative-stop, or force", + ) try: - return await task_service.shutdown_daemon(kill_jobs=body.kill_jobs) + return await task_service.shutdown_daemon(kill_jobs=body.kill_jobs, mode=body.mode) except DaemonError as exc: raise HTTPException(status_code=502, detail=f"daemon: {exc}") from exc @@ -168,10 +192,32 @@ def _reject_forbidden_args(args: list[str]) -> None: @router.delete("/{task_id}") async def stop_task(task_id: str): - """Cancel a running task.""" + """Request cancellation; completion is reported after daemon teardown.""" ok = await task_service.cancel_task(task_id) if not ok: raise HTTPException( - status_code=404, detail=f"Task {task_id} not found or not running" + status_code=404, detail=f"Task {task_id} not found or not cancellable" ) - return {"status": "cancelled", "task_id": task_id} + return {"status": "stopping", "task_id": task_id} + + +@router.post("/{task_id}/resume") +async def resume_task(task_id: str): + try: + task = await task_service.resume_task(task_id) + except DaemonError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + if task is None: + raise HTTPException(status_code=404, detail=f"Task {task_id} not found") + return task.info() + + +@router.delete("/{task_id}/history") +async def delete_history(task_id: str): + try: + ok = await task_service.delete_task(task_id) + except DaemonError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + if not ok: + raise HTTPException(status_code=409, detail="Only terminal tasks can be deleted") + return {"status": "deleted", "task_id": task_id} diff --git a/webui/frontend/src/components/SampleGallery.vue b/webui/frontend/src/components/SampleGallery.vue index b83d65562..41a7f6917 100644 --- a/webui/frontend/src/components/SampleGallery.vue +++ b/webui/frontend/src/components/SampleGallery.vue @@ -97,7 +97,9 @@ function open(i: number) { function fileUrl(s: SampleInfo): string { // filename is the basename the server uses for its whitelist check. - return `/api/preview/runs/${encodeURIComponent(props.taskId)}/samples/file?path=${encodeURIComponent(s.filename)}` + const params = new URLSearchParams({ path: s.filename }) + if (s.attempt_id) params.set('attempt_id', s.attempt_id) + return `/api/preview/runs/${encodeURIComponent(props.taskId)}/samples/file?${params.toString()}` } function labelFor(s: SampleInfo): string { diff --git a/webui/frontend/src/components/TaskAttemptHistory.vue b/webui/frontend/src/components/TaskAttemptHistory.vue new file mode 100644 index 000000000..3e1c35a26 --- /dev/null +++ b/webui/frontend/src/components/TaskAttemptHistory.vue @@ -0,0 +1,90 @@ + + + + + diff --git a/webui/frontend/src/components/TaskPanel.vue b/webui/frontend/src/components/TaskPanel.vue index 883286af9..625e5db2c 100644 --- a/webui/frontend/src/components/TaskPanel.vue +++ b/webui/frontend/src/components/TaskPanel.vue @@ -29,14 +29,27 @@ {{ task.command }} {{ task.state }} + + {{ t('taskLegacy') }} + - + {{ task.task_id.slice(0, 8) }} · PID {{ task.pid ?? '—' }} + · step {{ task.recovery_step }} + · {{ t('taskLastProgress') }} {{ task.last_progress.step }}/{{ task.last_progress.total_steps }} - - + + {{ t('taskTerminalReason') }}: {{ task.terminal_reason }} + + + {{ t('taskLegacyDescription') }} + + + +
- + + {{ t('taskResume') }} + {{ t('taskDeleteHistory') }} {{ task.state }} + + {{ t('taskLegacy') }} + - + {{ task.task_id.slice(0, 8) }} · PID {{ task.pid ?? '—' }} + · step {{ task.recovery_step }} + · {{ t('taskLastProgress') }} {{ task.last_progress.step }}/{{ task.last_progress.total_steps }} - - +
+ {{ t('taskTerminalReason') }}: {{ task.terminal_reason }} +
+
+ {{ t('taskLegacyDescription') }} +
+ + +
- + + {{ t('taskResume') }} + {{ t('taskDeleteHistory') }} + + diff --git a/webui/frontend/src/composables/useTrainingStream.ts b/webui/frontend/src/composables/useTrainingStream.ts index 710ae3307..c646faf80 100644 --- a/webui/frontend/src/composables/useTrainingStream.ts +++ b/webui/frontend/src/composables/useTrainingStream.ts @@ -16,6 +16,7 @@ interface WsMessage { epoch?: number | null prompt?: string | null ts?: number | null + attempt_id?: string | null } export function useTrainingStream(taskId: string) { @@ -140,6 +141,7 @@ export function useTrainingStream(taskId: string) { // A new preview image landed in the training output dir. Append // to the gallery (the store dedupes by path). store.recordSample({ + attempt_id: msg.attempt_id ?? null, path: msg.path, step: msg.step ?? null, epoch: msg.epoch ?? null, diff --git a/webui/frontend/src/i18n/cn.ts b/webui/frontend/src/i18n/cn.ts index f9efccf02..dec922b5d 100644 --- a/webui/frontend/src/i18n/cn.ts +++ b/webui/frontend/src/i18n/cn.ts @@ -96,9 +96,31 @@ export default { taskTitle: '任务监控', taskSubtitle: '查看运行中的任务、实时日志与任务管理', taskNoActive: '暂无运行中的任务', + taskNoHistory: '暂无任务历史', + taskNoMatches: '没有符合当前筛选条件的任务', + taskLoadFailed: '任务列表刷新失败,已清空陈旧列表。', taskCancel: '取消', taskShowLogs: '查看日志', taskHideLogs: '隐藏日志', + taskResume: '继续训练', + taskDeleteHistory: '清理记录', + taskDeleteHistoryConfirm: '确定清理这条任务的全部恢复记录吗?训练权重和断点不会删除。', + taskResumedCount: '已恢复 {n} 次', + taskShowAttempts: '展开运行记录', + taskHideAttempts: '收起运行记录', + taskAttemptLabel: '第 {n} 次运行', + taskAttemptResumedFrom: '从 step {step} 恢复', + taskAttemptExitCode: '退出码 {code}', + taskLastProgress: '最后进度', + taskTerminalReason: '终态原因', + taskLegacy: '旧任务', + taskLegacyDescription: '该任务缺少不可变配置快照,继续前请确认当前配置与数据仍匹配。', + taskLegacyResumeConfirm: '这是没有不可变配置快照的旧任务。请确认当前训练配置、数据集和预处理清单与原任务一致,再继续训练。', + taskFilterAll: '全部', + taskFilterActive: '运行中', + taskFilterSuccess: '成功', + taskFilterFailed: '失败', + taskFilterCancelled: '已取消', taskState: '状态', taskConnecting: '连接中...', taskPauseQueue: '暂停队列', @@ -107,6 +129,10 @@ export default { taskShutdownDaemon: '关闭守护进程', taskShutdownDaemonDesc: '将完全退出训练守护进程。若守护进程同时托管 WebUI,本界面也会一并关闭。', taskShutdownKillJobs: '同时终止正在运行的任务', + taskShutdownMode: '关闭模式', + taskShutdownDetach: '分离并重启时重新接管(保留任务)', + taskShutdownCooperative: '协作停止并保存断点', + taskShutdownForce: '强制终止(不保证最新断点)', taskShutdownConfirm: '关闭并退出', // Training Dashboard @@ -617,6 +643,12 @@ export default { notifyTestLaunched: '测试推理已启动', notifyTaskStarted: '{command} 已启动', notifyTaskStartFailed: '{command} 启动失败', + notifyTaskStopping: '已请求停止,正在保存状态后退出', + notifyTaskCancelFailed: '任务取消失败', + notifyTaskResumed: '已创建继续训练任务', + notifyTaskResumeFailed: '继续训练失败', + notifyTaskDeleted: '历史任务已删除', + notifyTaskDeleteFailed: '删除历史任务失败', notifyQueuePaused: '队列已暂停 — 新任务将等待', notifyQueueResumed: '队列已恢复', notifyQueuePauseFailed: '暂停队列失败', diff --git a/webui/frontend/src/i18n/en.ts b/webui/frontend/src/i18n/en.ts index 130a9aaf7..4a6f45603 100644 --- a/webui/frontend/src/i18n/en.ts +++ b/webui/frontend/src/i18n/en.ts @@ -96,9 +96,31 @@ export default { taskTitle: 'Task Monitor', taskSubtitle: 'View running tasks, streaming logs, and manage execution', taskNoActive: 'No active tasks', + taskNoHistory: 'No task history', + taskNoMatches: 'No tasks match this filter', + taskLoadFailed: 'Could not refresh tasks. The stale list was cleared.', taskCancel: 'Cancel', taskShowLogs: 'Show Logs', taskHideLogs: 'Hide Logs', + taskResume: 'Resume Training', + taskDeleteHistory: 'Delete History', + taskDeleteHistoryConfirm: 'Delete every attempt record in this task? Training weights and resume states will be kept.', + taskResumedCount: 'Resumed {n} times', + taskShowAttempts: 'Show attempts', + taskHideAttempts: 'Hide attempts', + taskAttemptLabel: 'Attempt {n}', + taskAttemptResumedFrom: 'Resumed from step {step}', + taskAttemptExitCode: 'Exit code {code}', + taskLastProgress: 'Last progress', + taskTerminalReason: 'Terminal reason', + taskLegacy: 'Legacy', + taskLegacyDescription: 'This task has no immutable config snapshot. Confirm that the current config and data still match before resuming.', + taskLegacyResumeConfirm: 'This is a legacy task without an immutable config snapshot. Confirm that the training config, dataset, and preprocess manifest still match the original run before resuming.', + taskFilterAll: 'All', + taskFilterActive: 'Active', + taskFilterSuccess: 'Success', + taskFilterFailed: 'Failed', + taskFilterCancelled: 'Cancelled', taskState: 'State', taskConnecting: 'Connecting...', taskPauseQueue: 'Pause Queue', @@ -107,6 +129,10 @@ export default { taskShutdownDaemon: 'Stop Daemon', taskShutdownDaemonDesc: 'This will fully exit the training daemon. If the daemon hosts the WebUI, this interface will also close.', taskShutdownKillJobs: 'Also terminate running jobs', + taskShutdownMode: 'Shutdown mode', + taskShutdownDetach: 'Detach (reattach jobs after restart)', + taskShutdownCooperative: 'Cooperative stop (save recovery state)', + taskShutdownForce: 'Force (latest checkpoint not guaranteed)', taskShutdownConfirm: 'Stop & Exit', // Training Dashboard @@ -617,6 +643,12 @@ export default { notifyTestLaunched: 'Test inference started', notifyTaskStarted: '{command} started', notifyTaskStartFailed: 'Failed to start {command}', + notifyTaskStopping: 'Stop requested; saving state before exit', + notifyTaskCancelFailed: 'Failed to cancel task', + notifyTaskResumed: 'Training resumed', + notifyTaskResumeFailed: 'Failed to resume task', + notifyTaskDeleted: 'History deleted', + notifyTaskDeleteFailed: 'Failed to delete history', notifyQueuePaused: 'Queue paused — new jobs will wait', notifyQueueResumed: 'Queue resumed', notifyQueuePauseFailed: 'Failed to pause the queue', diff --git a/webui/frontend/src/i18n/ja.ts b/webui/frontend/src/i18n/ja.ts index 5734d2b32..aec0abc1a 100644 --- a/webui/frontend/src/i18n/ja.ts +++ b/webui/frontend/src/i18n/ja.ts @@ -96,9 +96,31 @@ export default { taskTitle: 'タスクモニター', taskSubtitle: '実行中のタスク、リアルタイムログ、タスク管理', taskNoActive: 'アクティブなタスクなし', + taskNoHistory: 'タスク履歴なし', + taskNoMatches: 'このフィルターに一致するタスクなし', + taskLoadFailed: 'タスク一覧を更新できないため、古い一覧をクリアしました。', taskCancel: 'キャンセル', taskShowLogs: 'ログ表示', taskHideLogs: 'ログ非表示', + taskResume: 'トレーニングを再開', + taskDeleteHistory: '履歴を削除', + taskDeleteHistoryConfirm: 'このタスクのすべての実行履歴を削除しますか?学習済み重みと再開状態は保持されます。', + taskResumedCount: '{n} 回再開済み', + taskShowAttempts: '実行履歴を表示', + taskHideAttempts: '実行履歴を非表示', + taskAttemptLabel: '実行 {n}', + taskAttemptResumedFrom: 'step {step} から再開', + taskAttemptExitCode: '終了コード {code}', + taskLastProgress: '最終進捗', + taskTerminalReason: '終了理由', + taskLegacy: '旧タスク', + taskLegacyDescription: 'このタスクには不変の設定スナップショットがありません。再開前に現在の設定とデータが一致することを確認してください。', + taskLegacyResumeConfirm: 'これは不変の設定スナップショットがない旧タスクです。再開する前に、設定、データセット、前処理マニフェストが元の実行と一致することを確認してください。', + taskFilterAll: 'すべて', + taskFilterActive: '実行中', + taskFilterSuccess: '成功', + taskFilterFailed: '失敗', + taskFilterCancelled: 'キャンセル済み', taskState: '状態', taskConnecting: '接続中...', taskPauseQueue: '一時停止', @@ -107,6 +129,10 @@ export default { taskShutdownDaemon: 'デーモン停止', taskShutdownDaemonDesc: '学習デーモンを完全に終了します。デーモンが WebUI をホストしている場合、この画面も閉じます。', taskShutdownKillJobs: '実行中のタスクも終了', + taskShutdownMode: '停止モード', + taskShutdownDetach: '切り離し(再起動後にタスクを再接続)', + taskShutdownCooperative: '協調停止(再開状態を保存)', + taskShutdownForce: '強制停止(最新状態は保証されません)', taskShutdownConfirm: '停止して終了', // Training Dashboard @@ -617,6 +643,12 @@ export default { notifyTestLaunched: 'テスト推論開始', notifyTaskStarted: '{command} 開始', notifyTaskStartFailed: '{command} 開始失敗', + notifyTaskStopping: '停止を要求しました。状態を保存して終了します', + notifyTaskCancelFailed: 'タスクのキャンセルに失敗しました', + notifyTaskResumed: '再開タスクを作成しました', + notifyTaskResumeFailed: '再開に失敗しました', + notifyTaskDeleted: '履歴を削除しました', + notifyTaskDeleteFailed: '履歴の削除に失敗しました', notifyQueuePaused: 'キュー一時停止 — 新規タスクは待機', notifyQueueResumed: 'キュー再開', notifyQueuePauseFailed: 'キュー一時停止失敗', diff --git a/webui/frontend/src/i18n/ko.ts b/webui/frontend/src/i18n/ko.ts index 0f8a7bf04..55d7711b5 100644 --- a/webui/frontend/src/i18n/ko.ts +++ b/webui/frontend/src/i18n/ko.ts @@ -96,9 +96,31 @@ export default { taskTitle: '작업 모니터', taskSubtitle: '실행 중인 작업, 실시간 로그 및 작업 관리', taskNoActive: '활성 작업 없음', + taskNoHistory: '작업 기록 없음', + taskNoMatches: '이 필터와 일치하는 작업 없음', + taskLoadFailed: '작업 목록을 새로 고칠 수 없어 오래된 목록을 비웠습니다.', taskCancel: '취소', taskShowLogs: '로그 보기', taskHideLogs: '로그 숨기기', + taskResume: '학습 재개', + taskDeleteHistory: '기록 삭제', + taskDeleteHistoryConfirm: '이 작업의 모든 실행 기록을 삭제할까요? 학습 가중치와 재개 상태는 유지됩니다.', + taskResumedCount: '{n}회 재개됨', + taskShowAttempts: '실행 기록 표시', + taskHideAttempts: '실행 기록 숨기기', + taskAttemptLabel: '실행 {n}', + taskAttemptResumedFrom: 'step {step}에서 재개', + taskAttemptExitCode: '종료 코드 {code}', + taskLastProgress: '마지막 진행', + taskTerminalReason: '종료 이유', + taskLegacy: '레거시', + taskLegacyDescription: '이 작업에는 변경 불가능한 설정 스냅샷이 없습니다. 재개하기 전에 현재 설정과 데이터가 일치하는지 확인하세요.', + taskLegacyResumeConfirm: '변경 불가능한 설정 스냅샷이 없는 레거시 작업입니다. 재개하기 전에 학습 설정, 데이터셋, 전처리 매니페스트가 원래 실행과 일치하는지 확인하세요.', + taskFilterAll: '전체', + taskFilterActive: '실행 중', + taskFilterSuccess: '성공', + taskFilterFailed: '실패', + taskFilterCancelled: '취소됨', taskState: '상태', taskConnecting: '연결 중...', taskPauseQueue: '일시정지', @@ -107,6 +129,10 @@ export default { taskShutdownDaemon: '데몬 종료', taskShutdownDaemonDesc: '학습 데몬을 완전히 종료합니다. 데몬이 WebUI를 호스팅 중이면 이 화면도 함께 닫힙니다.', taskShutdownKillJobs: '실행 중인 작업도 종료', + taskShutdownMode: '종료 모드', + taskShutdownDetach: '분리 (재시작 후 작업 재연결)', + taskShutdownCooperative: '협조적 중지 (복구 상태 저장)', + taskShutdownForce: '강제 종료 (최신 체크포인트 보장 안 됨)', taskShutdownConfirm: '종료', // Training Dashboard @@ -617,6 +643,12 @@ export default { notifyTestLaunched: '테스트 추론 시작됨', notifyTaskStarted: '{command} 시작됨', notifyTaskStartFailed: '{command} 시작 실패', + notifyTaskStopping: '중지를 요청했습니다. 상태를 저장한 후 종료합니다', + notifyTaskCancelFailed: '작업을 취소하지 못했습니다', + notifyTaskResumed: '재개 작업을 생성했습니다', + notifyTaskResumeFailed: '재개에 실패했습니다', + notifyTaskDeleted: '기록을 삭제했습니다', + notifyTaskDeleteFailed: '기록 삭제에 실패했습니다', notifyQueuePaused: '큐 일시정지 — 새 작업 대기', notifyQueueResumed: '큐 재개됨', notifyQueuePauseFailed: '큐 일시정지 실패', diff --git a/webui/frontend/src/stores/stagedResolution.ts b/webui/frontend/src/stores/stagedResolution.ts index 31b5d9228..63a701add 100644 --- a/webui/frontend/src/stores/stagedResolution.ts +++ b/webui/frontend/src/stores/stagedResolution.ts @@ -219,7 +219,7 @@ export const useStagedResolutionStore = defineStore('stagedResolution', () => { const task = taskStore.tasks.find(item => item.task_id === taskId) if (!task) return preprocessTaskState.value = task.state - if (task.state === 'pending' || task.state === 'running') { + if (task.state === 'pending' || task.state === 'running' || task.state === 'stopping') { await fetchStatus() return } diff --git a/webui/frontend/src/stores/task.ts b/webui/frontend/src/stores/task.ts index 72ded2edf..a9f23f757 100644 --- a/webui/frontend/src/stores/task.ts +++ b/webui/frontend/src/stores/task.ts @@ -3,10 +3,24 @@ import { ref } from 'vue' import { useNotifyStore } from './notify' import { useI18n } from '../composables/useI18n' +export interface TaskAttempt { + job_id: string + attempt_index: number + state: string + started_at: string | null + ended_at: string | null + recovery_step: number | null + exit_code: number | null + terminal_reason: string | null +} + export interface TaskInfo { task_id: string + current_job_id: string | null + attempt_count: number + attempts: TaskAttempt[] command: string - state: 'pending' | 'running' | 'success' | 'failed' | 'cancelled' + state: 'pending' | 'running' | 'stopping' | 'success' | 'failed' | 'cancelled' pid: number | null started_at: string | null output_lines: number @@ -15,12 +29,30 @@ export interface TaskInfo { // many queued jobs finish before this one starts (1, 2, … in FIFO order). // null/undefined for running or terminal tasks. queue_position?: number | null + recovery_step?: number | null + resume_state?: string | null + terminal_reason?: string | null + resumable?: boolean + legacy?: boolean + exit_code?: number | null + last_progress?: { + step?: number | null + total_steps?: number | null + epoch?: number | null + } } +export type TaskFilter = 'all' | 'active' | 'success' | 'failed' | 'cancelled' + export const useTaskStore = defineStore('task', () => { const tasks = ref([]) const commands = ref>({}) const loading = ref(false) + const totalTasks = ref(0) + const page = ref(1) + const pageSize = ref(25) + const taskFilter = ref('all') + const taskListError = ref(false) // Queue state — driven by GET /api/tasks/queue/status (polled alongside // fetchTasks in the Tasks view). const daemonPaused = ref(false) @@ -32,14 +64,35 @@ export const useTaskStore = defineStore('task', () => { const notify = useNotifyStore() const { t } = useI18n() - async function fetchTasks() { + async function fetchTasks(options: { + state?: TaskFilter + page?: number + pageSize?: number + } = {}) { try { - const res = await fetch('/api/tasks') + if (options.state !== undefined) taskFilter.value = options.state + if (options.page !== undefined) page.value = Math.max(1, options.page) + if (options.pageSize !== undefined) pageSize.value = Math.max(1, options.pageSize) + const params = new URLSearchParams({ + limit: String(pageSize.value), + offset: String((page.value - 1) * pageSize.value), + }) + if (taskFilter.value !== 'all') params.set('state', taskFilter.value) + const res = await fetch(`/api/tasks?${params.toString()}`) if (!res.ok) throw new Error(`HTTP ${res.status}`) const data = await res.json() - if (Array.isArray(data)) tasks.value = data + if (Array.isArray(data)) { + tasks.value = data + taskListError.value = false + const headerTotal = Number(res.headers.get('X-Total-Count')) + totalTasks.value = Number.isFinite(headerTotal) ? headerTotal : data.length + } } catch { - // silently ignore + // Do not leave a stale page visible when a filter/poll request failed. + // Otherwise the toggle changes while the old task cards remain on screen. + tasks.value = [] + totalTasks.value = 0 + taskListError.value = true } } @@ -77,7 +130,12 @@ export const useTaskStore = defineStore('task', () => { queue_position: positions[task.task_id] ?? null, })) } catch { - // daemon unreachable on this tick — leave the last known state. + daemonUp.value = false + daemonPaused.value = false + if (!daemonWasDown.value) { + notify.show(t('notifyDaemonDown'), 'warning') + } + daemonWasDown.value = true } } @@ -105,12 +163,15 @@ export const useTaskStore = defineStore('task', () => { } } - async function shutdownDaemon(killJobs: boolean = true): Promise { + async function shutdownDaemon(modeOrKill: boolean | 'detach' | 'cooperative-stop' | 'force' = true): Promise { + const mode = typeof modeOrKill === 'boolean' + ? (modeOrKill ? 'force' : 'detach') + : modeOrKill try { await fetch('/api/tasks/daemon/shutdown', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ kill_jobs: killJobs }), + body: JSON.stringify({ kill_jobs: mode === 'force', mode }), }) // A clean response is best-case — but the daemon may host the WebUI as a // sidecar and tree-kill this server on shutdown, so a network error here @@ -137,7 +198,7 @@ export const useTaskStore = defineStore('task', () => { }) if (!res.ok) throw new Error(`HTTP ${res.status}`) const data = await res.json() - await fetchTasks() + await fetchTasks({ page: 1 }) await fetchQueueStatus() return data.task_id || null } catch { @@ -147,14 +208,75 @@ export const useTaskStore = defineStore('task', () => { } } - async function cancelTask(taskId: string) { + async function cancelTask(taskId: string): Promise { try { const res = await fetch(`/api/tasks/${taskId}`, { method: 'DELETE' }) - if (!res.ok) throw new Error(`HTTP ${res.status}`) + if (!res.ok) { + let detail = `HTTP ${res.status}` + try { + const body = await res.json() + if (body?.detail) detail = body.detail + } catch { + // Keep the HTTP status when the server did not return JSON. + } + throw new Error(detail) + } + tasks.value = tasks.value.map((task) => + task.task_id === taskId ? { ...task, state: 'stopping' } : task, + ) + notify.show(t('notifyTaskStopping'), 'info') await fetchTasks() await fetchQueueStatus() + return true } catch { - // silently ignore + notify.show(t('notifyTaskCancelFailed'), 'error') + return false + } + } + + async function resumeTask(taskId: string): Promise { + const source = tasks.value.find((task) => task.task_id === taskId) + if (source?.legacy && !window.confirm(t('taskLegacyResumeConfirm'))) return false + let failureDetail = '' + try { + const res = await fetch(`/api/tasks/${taskId}/resume`, { method: 'POST' }) + if (!res.ok) { + failureDetail = `HTTP ${res.status}` + try { + const body = await res.json() + if (body?.detail) failureDetail = String(body.detail) + } catch { + // Keep the HTTP status when the response is not JSON. + } + throw new Error(failureDetail) + } + await poll() + notify.show(t('notifyTaskResumed'), 'success') + return true + } catch { + notify.show( + failureDetail ? `${t('notifyTaskResumeFailed')}: ${failureDetail}` : t('notifyTaskResumeFailed'), + 'error', + 6000, + ) + return false + } + } + + async function deleteHistory(taskId: string): Promise { + if (!window.confirm(t('taskDeleteHistoryConfirm'))) return false + try { + const res = await fetch(`/api/tasks/${taskId}/history`, { method: 'DELETE' }) + if (!res.ok) throw new Error(`HTTP ${res.status}`) + tasks.value = tasks.value.filter((task) => task.task_id !== taskId) + totalTasks.value = Math.max(0, totalTasks.value - 1) + if (tasks.value.length === 0 && page.value > 1) page.value -= 1 + await fetchTasks() + notify.show(t('notifyTaskDeleted'), 'success') + return true + } catch { + notify.show(t('notifyTaskDeleteFailed'), 'error') + return false } } @@ -166,6 +288,11 @@ export const useTaskStore = defineStore('task', () => { tasks, commands, loading, + totalTasks, + page, + pageSize, + taskFilter, + taskListError, daemonPaused, daemonUp, fetchTasks, @@ -176,6 +303,8 @@ export const useTaskStore = defineStore('task', () => { shutdownDaemon, startTask, cancelTask, + resumeTask, + deleteHistory, poll, } }) diff --git a/webui/frontend/src/stores/training.ts b/webui/frontend/src/stores/training.ts index ca63ad74b..25e23637f 100644 --- a/webui/frontend/src/stores/training.ts +++ b/webui/frontend/src/stores/training.ts @@ -2,6 +2,7 @@ import { defineStore } from 'pinia' import { ref } from 'vue' export interface SampleInfo { + attempt_id: string | null path: string // absolute on-disk path emitted by the training process filename: string // basename used for the FileResponse URL step: number | null @@ -84,14 +85,15 @@ export const useTrainingStore = defineStore('training', () => { const connected = ref(false) const done = ref(false) - // Set of on-disk paths we've already recorded — path is the natural - // dedupe key (training process writes one PNG per sample event). + // Attempt + path is the durable key: resumed attempts use independent + // sample directories and may legitimately emit the same basename. const seenSamplePaths = new Set() function recordSample(info: Omit) { if (!info.path) return - if (seenSamplePaths.has(info.path)) return - seenSamplePaths.add(info.path) + const sampleKey = `${info.attempt_id || ''}\u0000${info.path}` + if (seenSamplePaths.has(sampleKey)) return + seenSamplePaths.add(sampleKey) const sample: SampleInfo = { ...info, @@ -105,7 +107,9 @@ export const useTrainingStore = defineStore('training', () => { const overflow = metrics.value.sample_history.length - MAX_SAMPLE_HISTORY if (overflow > 0) { const dropped = metrics.value.sample_history.splice(0, overflow) - for (const d of dropped) seenSamplePaths.delete(d.path) + for (const d of dropped) { + seenSamplePaths.delete(`${d.attempt_id || ''}\u0000${d.path}`) + } } } @@ -152,6 +156,7 @@ export const useTrainingStore = defineStore('training', () => { // and ``latest_sample`` ends up pointing at the newest file. for (const it of items.slice().reverse()) { recordSample({ + attempt_id: it.attempt_id || null, path: it.path, step: null, epoch: null, diff --git a/webui/frontend/src/views/AdapterView.vue b/webui/frontend/src/views/AdapterView.vue index ed41090bf..980de9b19 100644 --- a/webui/frontend/src/views/AdapterView.vue +++ b/webui/frontend/src/views/AdapterView.vue @@ -171,7 +171,7 @@ > @@ -203,7 +203,7 @@ function isRunning(command: string) { } function stateColor(state: string) { - if (state === 'running') return 'info' + if (state === 'running' || state === 'stopping') return 'info' if (state === 'success') return 'success' if (state === 'failed') return 'error' return undefined diff --git a/webui/frontend/src/views/DistillView.vue b/webui/frontend/src/views/DistillView.vue index 4c35983c0..11ec3d7ff 100644 --- a/webui/frontend/src/views/DistillView.vue +++ b/webui/frontend/src/views/DistillView.vue @@ -192,7 +192,7 @@