From 93a93030058b71dc4e12d202bead487389e1cddf Mon Sep 17 00:00:00 2001 From: YUFEI JIA <59379871+TATP-233@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:02:04 +0800 Subject: [PATCH] fix(uni-cumps): add stop --gpus selector and actionable stop errors The name-only stop surface trapped users in the recommended one-daemon-per-GPU setup: bare stop was ambiguous, stop --gpus did not exist, and failures never named a next step. - stop accepts --gpus (exclusive with --name/--all), resolving the unique live recorded daemon covering that GPU; zero or multiple matches fail closed and name the candidates - stop/env selection errors point at uni-cumps status; start --name help corrected to gpu-; shell completion updated - README and training guide (en/zh) document the full status -> start -> train -> stop lifecycle, stop --all, the default-name rule, and the orphan-daemon manual recovery path - ADR-0014 (Proposed) amended: --gpus stop selector supersedes the name-only stop surface; discoverability is part of the CLI contract Closes #2610 --- README.md | 18 ++++- .../ADR-0014-cuda-mps-cli-lifecycle-owner.md | 9 +++ docs/sphinx/source/changelog.md | 16 ++++ .../1-training/1-cli_reference.md | 4 +- .../1-training/7-tensor_runtime_production.md | 26 +++++-- .../1-training/1-cli_reference.md | 4 +- .../1-training/7-tensor_runtime_production.md | 22 ++++-- src/unilab/cli_completion.py | 2 +- src/unilab/training/cuda_mps_cli.py | 61 +++++++++++++-- tests/training/test_cuda_mps_cli.py | 75 +++++++++++++++++++ 10 files changed, 209 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 5fbcf94a1..bff2b89b3 100644 --- a/README.md +++ b/README.md @@ -108,19 +108,31 @@ uv run --extra genesis eval --algo flashsac --task g1_motion_tracking --sim gene To use MPS, start one service per selected GPU before training, append `training.cuda_process_sharing=mps` to the training command, and stop those -services manually after training. For example, for GPUs 0 and 1: +services after training. `uv run uni-cumps status` lists visible GPUs and all +recorded services at any point. For example, for GPUs 0 and 1: ```bash +uv run uni-cumps status # visible GPUs + recorded services uv run uni-cumps start --gpus 0 --name train-gpu0 --daemon uv run uni-cumps start --gpus 1 --name train-gpu1 --daemon CUDA_VISIBLE_DEVICES=0,1 \ uv run --extra mjwarp train --algo flashsac --task g1_motion_tracking --sim mjwarp \ training.cuda_process_sharing=mps # Run training, then stop only these services after its processes exit. -uv run uni-cumps stop --name train-gpu0 -uv run uni-cumps stop --name train-gpu1 +uv run uni-cumps stop --name train-gpu0 # or: stop --gpus 0 +uv run uni-cumps stop --name train-gpu1 # or: stop --gpus 1 ``` +`stop` selects by `--name`, by `--gpus`, or implicitly when exactly one live +service exists; `stop --all` stops every service recorded for the current user +on this host. Without `--name`, a started service is labeled +`gpu-`; `status` prints every recorded name with its GPU. UniLab +stops only services it recorded (records live in +`~/.cache/unilab/cuda-mps/`): if a control daemon outlives its record, quit it +manually, e.g. +`echo quit | CUDA_MPS_PIPE_DIRECTORY=/tmp/uni-cumps//pipe CUDA_MPS_LOG_DIRECTORY=/tmp/uni-cumps//log nvidia-cuda-mps-control`, +then remove that `/tmp/uni-cumps/` directory. + An `mps` request binds each rank to its GPU's recorded service automatically (index-form `CUDA_VISIBLE_DEVICES` entries are normalized to GPU UUIDs, which per-GPU MPS servers require, before CUDA initializes); `uni-cumps env` is only diff --git a/docs/sphinx/source/adr/ADR-0014-cuda-mps-cli-lifecycle-owner.md b/docs/sphinx/source/adr/ADR-0014-cuda-mps-cli-lifecycle-owner.md index 6001abcd7..a8c03a1e9 100644 --- a/docs/sphinx/source/adr/ADR-0014-cuda-mps-cli-lifecycle-owner.md +++ b/docs/sphinx/source/adr/ADR-0014-cuda-mps-cli-lifecycle-owner.md @@ -35,6 +35,15 @@ fail-closed probe,也不提前声明 DP 或 task-per-GPU 支持。 `--name` 时使用 `gpu-`;未传 pipe/log 路径时使用 `/tmp/uni-cumps/`(mode `0700`); `env`/`stop` 未传 `--name` 时选择唯一 live record。多 GPU 或多 daemon 歧义必须 显式指定,不自动选择。 +- `stop` 接受 `--gpus ` 选择器(与 `--name`/`--all` 互斥),解析为 + 覆盖该 GPU 的唯一 live record;零个或多个匹配 fail closed 并列出候选名称。 + stop authority 仍然只来自 record,不 attach 到 unmanaged daemon。 + (2026-11-16 依据 issue #2610 增补,取代原"仅 `--name`"的 stop 选择面:推荐的 + 一 GPU 一 daemon 流程使裸 `stop` 必然歧义,用户只能猜 label。) +- 可发现性是 CLI contract 的一部分:所有 `stop`/`env` 选择失败的错误信息必须指出 + 下一步(`uni-cumps status` 列出 record 名称与 GPU);用户文档必须给出 + status → start → train → stop 完整生命周期,以及 record 丢失后 orphan daemon 的 + 手动恢复路径。(2026-11-16,issue #2610。) - Daemon record 以 `(UID, host, name)` 为 scope,持久化 canonical GPU UUID、绝对 pipe/log 路径、control PID 与 Linux `/proc//stat` start-time。PID 单独不可作为 stop authority。 diff --git a/docs/sphinx/source/changelog.md b/docs/sphinx/source/changelog.md index 47ff4ad4d..89d6c54f3 100644 --- a/docs/sphinx/source/changelog.md +++ b/docs/sphinx/source/changelog.md @@ -13,6 +13,22 @@ UniLab 遵循[语义化版本](https://semver.org/)。本共享页面以中英 PyPI 版本变更与未发布变更;发布日期采用 PyPI 上传日期。完整提交历史请参阅 [UniLab 仓库](https://github.com/unilabsim/UniLab)。 +## Unreleased / 未发布 + +### Fixed / 修复 + +- `uni-cumps stop` accepts `--gpus ` to stop the unique + recorded daemon covering one GPU, stop-selection errors name the next + discovery step (`uni-cumps status`), and the README and training guide + document the full status → start → train → stop lifecycle including + orphan-daemon recovery. + `uni-cumps stop` 新增 `--gpus ` 选择器,可停止覆盖指定 GPU + 的唯一已记录 daemon;stop 选择失败的错误信息会指出下一步 + (`uni-cumps status`);README 与训练指南现在给出 + status → start → train → stop 完整生命周期,包括 orphan daemon 的手动恢复 + 路径。 + ([#2610](https://github.com/Motphys/UniLab/issues/2610)). + ## 1.5.11 (2026-11-16) ### Added / 新增 diff --git a/docs/sphinx/source/en/2-user_guide/1-training/1-cli_reference.md b/docs/sphinx/source/en/2-user_guide/1-training/1-cli_reference.md index 6d9a988c4..aec7faa71 100644 --- a/docs/sphinx/source/en/2-user_guide/1-training/1-cli_reference.md +++ b/docs/sphinx/source/en/2-user_guide/1-training/1-cli_reference.md @@ -49,10 +49,12 @@ uv run uni-cumps doctor uv run uni-cumps stop # Multi-GPU or explicitly named deployments. +uv run uni-cumps status # visible GPUs and recorded daemons uv run uni-cumps doctor --gpus uv run uni-cumps start --gpus --name uv run uni-cumps env --name -uv run uni-cumps stop --name +uv run uni-cumps stop --name # or: stop --gpus +uv run uni-cumps stop --all # stop every recorded daemon ``` See the tensor runtime production guide for ownership, concurrency, and topology diff --git a/docs/sphinx/source/en/2-user_guide/1-training/7-tensor_runtime_production.md b/docs/sphinx/source/en/2-user_guide/1-training/7-tensor_runtime_production.md index a5ab10113..739107ffb 100644 --- a/docs/sphinx/source/en/2-user_guide/1-training/7-tensor_runtime_production.md +++ b/docs/sphinx/source/en/2-user_guide/1-training/7-tensor_runtime_production.md @@ -134,11 +134,12 @@ Default values: | daemon name | `gpu-` | | pipe directory | `/tmp/uni-cumps//pipe` (mode `0700`) | | log directory | `/tmp/uni-cumps//log` (mode `0700`) | -| `env`/`stop` target | the sole live UniLab-recorded daemon | +| `env`/`stop` target | the sole live UniLab-recorded daemon; `--name` or `--gpus` selects explicitly | If more than one GPU is visible, `start` and `doctor` require -`--gpus `. Explicit overrides remain available for deployments -that need stable labels or service-owned paths: +`--gpus `, and `stop` accepts the same selector to pick the +recorded daemon covering one GPU. Explicit overrides remain available for +deployments that need stable labels or service-owned paths: ```bash uv run uni-cumps start \ @@ -147,7 +148,8 @@ uv run uni-cumps start \ --pipe-dir /absolute/path/mps/pipe \ --log-dir /absolute/path/mps/log eval "$(uv run uni-cumps env --name )" -uv run uni-cumps stop --name +uv run uni-cumps stop --name # or: stop --gpus +uv run uni-cumps stop --all # stop every recorded daemon ``` Read-only status works without arguments: @@ -157,10 +159,18 @@ uv run uni-cumps status ``` `status` and `doctor` are read-only. `start` creates a user-owned daemon and a -record under the current user's UniLab cache. `stop` uses only that recorded -ownership evidence: UID, host, control PID, and Linux process start-time. It -never sends `quit` to an unmanaged pipe. `env` prints exports for a launcher or -an explicit shell integration; it does not mutate UniLab's parent process. +record under the current user's UniLab cache (`~/.cache/unilab/cuda-mps/`). +`stop` uses only that recorded ownership evidence: UID, host, control PID, and +Linux process start-time. It never sends `quit` to an unmanaged pipe. `env` +prints exports for a launcher or an explicit shell integration; it does not +mutate UniLab's parent process. Stop-selection failures name the next step +(`uv run uni-cumps status` lists every recorded daemon with its GPU). + +If a control daemon outlives its record — for example after the UniLab cache +was cleaned — UniLab cannot stop it. Recover manually by sending `quit` +through its recorded pipe directory, e.g. +`echo quit | CUDA_MPS_PIPE_DIRECTORY=/tmp/uni-cumps//pipe CUDA_MPS_LOG_DIRECTORY=/tmp/uni-cumps//log nvidia-cuda-mps-control`, +then remove that runtime directory. A daemon is not exclusively owned by a training run and may serve multiple clients. A stale record is quarantined before reuse, and logs are retained after diff --git a/docs/sphinx/source/zh_CN/2-user_guide/1-training/1-cli_reference.md b/docs/sphinx/source/zh_CN/2-user_guide/1-training/1-cli_reference.md index 432de0209..5386b5f11 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/1-training/1-cli_reference.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/1-training/1-cli_reference.md @@ -44,10 +44,12 @@ uv run uni-cumps doctor uv run uni-cumps stop # 多 GPU 或需要显式命名的部署。 +uv run uni-cumps status # 可见 GPU 与已记录的 daemon uv run uni-cumps doctor --gpus uv run uni-cumps start --gpus --name uv run uni-cumps env --name -uv run uni-cumps stop --name +uv run uni-cumps stop --name # 或:stop --gpus +uv run uni-cumps stop --all # 停止当前用户所有已记录 daemon ``` ownership、并发和拓扑限制见 tensor runtime 生产指南。训练仍必须显式设置 diff --git a/docs/sphinx/source/zh_CN/2-user_guide/1-training/7-tensor_runtime_production.md b/docs/sphinx/source/zh_CN/2-user_guide/1-training/7-tensor_runtime_production.md index e8b6bc135..97ee535dd 100644 --- a/docs/sphinx/source/zh_CN/2-user_guide/1-training/7-tensor_runtime_production.md +++ b/docs/sphinx/source/zh_CN/2-user_guide/1-training/7-tensor_runtime_production.md @@ -128,10 +128,11 @@ daemon,训练会在 environment/learner/collector 构造前 fail closed,并 | daemon 名称 | `gpu-` | | pipe 目录 | `/tmp/uni-cumps//pipe` (mode `0700`) | | log 目录 | `/tmp/uni-cumps//log` (mode `0700`) | -| `env`/`stop` 目标 | 唯一 live 的 UniLab-recorded daemon | +| `env`/`stop` 目标 | 唯一 live 的 UniLab-recorded daemon;可用 `--name` 或 `--gpus` 显式选择 | 如果可见 GPU 多于一张,`start` 与 `doctor` 必须显式传 -`--gpus `。需要稳定 label 或服务自有路径的部署仍可显式覆盖: +`--gpus `;`stop` 接受相同的选择器来停止覆盖该 GPU 的已记录 +daemon。需要稳定 label 或服务自有路径的部署仍可显式覆盖: ```bash uv run uni-cumps start \ @@ -140,7 +141,8 @@ uv run uni-cumps start \ --pipe-dir /absolute/path/mps/pipe \ --log-dir /absolute/path/mps/log eval "$(uv run uni-cumps env --name )" -uv run uni-cumps stop --name +uv run uni-cumps stop --name # 或:stop --gpus +uv run uni-cumps stop --all # 停止当前用户所有已记录 daemon ``` 只读 status 无参数即可使用: @@ -150,10 +152,16 @@ uv run uni-cumps status ``` `status` 与 `doctor` 是只读诊断。`start` 创建 user-owned daemon,并在当前用户的 -UniLab cache 中写入 record。`stop` 只接受 record 中的 ownership evidence:UID、 -host、control PID 与 Linux process start-time;它绝不会向 unmanaged pipe 发送 -`quit`。`env` 输出给 launcher 或显式 shell 集成使用的环境变量,不修改 UniLab -父进程环境。 +UniLab cache(`~/.cache/unilab/cuda-mps/`)中写入 record。`stop` 只接受 record +中的 ownership evidence:UID、host、control PID 与 Linux process start-time;它 +绝不会向 unmanaged pipe 发送 `quit`。`env` 输出给 launcher 或显式 shell 集成使用 +的环境变量,不修改 UniLab 父进程环境。stop 选择失败时,错误信息会给出下一步 +(`uv run uni-cumps status` 列出所有已记录 daemon 及其 GPU)。 + +如果 control daemon 比它的 record 活得更久——例如 UniLab cache 被清理之后—— +UniLab 无法再停止它。此时需要手动恢复:通过它记录的 pipe 目录发送 `quit`,例如 +`echo quit | CUDA_MPS_PIPE_DIRECTORY=/tmp/uni-cumps//pipe CUDA_MPS_LOG_DIRECTORY=/tmp/uni-cumps//log nvidia-cuda-mps-control`, +然后删除对应的 runtime 目录。 daemon 不是某个训练 run 的独占资源,可以服务多个 client。stale record 会在复用前 被 quarantine;stop 后保留日志。 diff --git a/src/unilab/cli_completion.py b/src/unilab/cli_completion.py index d3754f9f4..dd6907515 100644 --- a/src/unilab/cli_completion.py +++ b/src/unilab/cli_completion.py @@ -489,7 +489,7 @@ def _demo_completions( "status": ("--json",), "doctor": ("--gpus", "--json"), "start": ("--gpus", "--name", "--pipe-dir", "--log-dir", "--daemon", "--foreground"), - "stop": ("--name", "--all"), + "stop": ("--name", "--gpus", "--all"), "env": ("--name", "--shell", "--json"), } diff --git a/src/unilab/training/cuda_mps_cli.py b/src/unilab/training/cuda_mps_cli.py index a8d35efcd..e9c10860b 100644 --- a/src/unilab/training/cuda_mps_cli.py +++ b/src/unilab/training/cuda_mps_cli.py @@ -296,7 +296,9 @@ def _validate_record_identity( def _daemon_record_path(root: Path, name: str) -> Path: if not _name_is_safe(name): raise CudaMpsError( - "Daemon name must match [a-z0-9][a-z0-9-]{0,63}; use a label, not a path." + "Daemon name must match [a-z0-9][a-z0-9-]{0,63}; use a label, not a path. " + "Run `uv run uni-cumps status` to list recorded daemon names, or select the " + "daemon by GPU with --gpus." ) return root / name / "daemon.json" @@ -362,14 +364,16 @@ def read_daemon_record( else f"multiple live daemons exist: {', '.join(daemon.name for daemon in live)}" ) raise CudaMpsError( - f"Cannot select a default daemon because {detail}; pass --name explicitly." + f"Cannot select a default daemon because {detail}; pass --name or --gpus " + "explicitly. Run `uv run uni-cumps status` to list recorded daemons." ) path = _daemon_record_path(_record_root(root), name) try: payload = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError as exc: raise CudaMpsError( - f"No UniLab CUDA MPS daemon record named {name!r}. UniLab can stop only recorded daemons." + f"No UniLab CUDA MPS daemon record named {name!r}. UniLab can stop only recorded " + "daemons. Run `uv run uni-cumps status` to list recorded daemon names and GPUs." ) from exc except (OSError, json.JSONDecodeError) as exc: raise CudaMpsError(f"Could not read daemon record {path}: {exc}.") from exc @@ -961,6 +965,40 @@ def select_environment_for_gpu( return environment_for_daemon(matching[0]) +def select_daemon_for_gpu( + selector: str, + *, + root: Path | None = None, + run_command: RunCommand = subprocess.run, + proc_stat: Callable[[int], tuple[int, int] | None] = _proc_stat, +) -> DaemonRecord: + """Select the unique recorded daemon covering one GPU index or UUID. + + Stop-facing selection: a stale sole candidate is returned so the caller's + stop path cleans its record and reports the stale state; ambiguity between + live daemons fails closed and names the candidates. + """ + + plan = resolve_topology(selector, run_command=run_command) + uuid = plan.gpus[0].uuid + candidates = [daemon for daemon in list_daemon_records(root) if uuid in daemon.gpu_uuids] + live = [daemon for daemon in candidates if daemon_process_is_live(daemon, proc_stat=proc_stat)] + if len(live) > 1: + names = ", ".join(daemon.name for daemon in live) + raise CudaMpsError( + f"Multiple live UniLab-recorded daemons cover GPU {uuid}: {names}. " + "Pass --name explicitly." + ) + if live: + return live[0] + if len(candidates) == 1: + return candidates[0] + raise CudaMpsError( + f"No live UniLab-recorded CUDA MPS daemon covers GPU {uuid}. " + "Run `uv run uni-cumps status` to list recorded daemons and their GPUs." + ) + + def stop_daemon( name: str, *, @@ -1106,7 +1144,7 @@ def _mps_parser() -> argparse.ArgumentParser: help="one GPU index/UUID (default: the sole visible GPU)", ) start_parser.add_argument( - "--name", default=None, help="daemon label (default: gpu-)" + "--name", default=None, help="daemon label (default: gpu-)" ) start_parser.add_argument( "--pipe-dir", default=None, help="pipe directory (default: user-owned UniLab cache)" @@ -1118,10 +1156,16 @@ def _mps_parser() -> argparse.ArgumentParser: start_mode.add_argument("--daemon", action="store_true", help="run the control daemon detached") start_mode.add_argument("--foreground", action="store_true", help="run control in this process") stop_parser = subparsers.add_parser("stop", help="stop a UniLab-recorded daemon") - stop_parser.add_argument( + stop_select = stop_parser.add_mutually_exclusive_group() + stop_select.add_argument( "--name", default=None, help="daemon label (default: the sole live daemon)" ) - stop_parser.add_argument("--all", action="store_true", help="stop all current-user records") + stop_select.add_argument( + "--gpus", + default=None, + help="stop the unique recorded daemon covering this GPU index/UUID", + ) + stop_select.add_argument("--all", action="store_true", help="stop all current-user records") env_parser = subparsers.add_parser( "env", help="print launcher environment for a recorded daemon" ) @@ -1191,7 +1235,10 @@ def main(argv: Sequence[str] | None = None) -> int: if args.all: stopped = stop_all_daemons() else: - stopped = (stop_daemon(args.name),) + name = args.name + if args.gpus is not None: + name = select_daemon_for_gpu(args.gpus).name + stopped = (stop_daemon(name),) _print_json({"stopped": [daemon.to_dict() for daemon in stopped]}) return 0 diff --git a/tests/training/test_cuda_mps_cli.py b/tests/training/test_cuda_mps_cli.py index 6173b1f45..044cc4770 100644 --- a/tests/training/test_cuda_mps_cli.py +++ b/tests/training/test_cuda_mps_cli.py @@ -686,6 +686,76 @@ def run(command: list[str], **_kwargs: Any) -> Completed: assert (linux_host / "b" / "daemon.json").exists() +def test_select_daemon_for_gpu_resolves_unique_live_record(linux_host: Path) -> None: + write_record(linux_host, daemon_record(linux_host, name="prod")) + + by_index = mps.select_daemon_for_gpu( + "0", root=linux_host, run_command=fake_run(), proc_stat=live_proc_stat() + ) + by_bare_uuid = mps.select_daemon_for_gpu( + GPU_A.removeprefix("GPU-"), + root=linux_host, + run_command=fake_run(), + proc_stat=live_proc_stat(), + ) + + assert by_index.name == "prod" + assert by_bare_uuid.name == "prod" + + +def test_select_daemon_for_gpu_fails_closed_on_zero_or_multiple_matches( + linux_host: Path, +) -> None: + write_record(linux_host, daemon_record(linux_host, name="prod")) + + with pytest.raises(mps.CudaMpsError, match="uni-cumps status"): + mps.select_daemon_for_gpu( + "1", root=linux_host, run_command=fake_run(), proc_stat=live_proc_stat() + ) + + other = daemon_record(linux_host, name="other", pid=4243, process_start_ticks=778) + write_record(linux_host, other) + with pytest.raises(mps.CudaMpsError, match="Multiple live UniLab-recorded daemons"): + mps.select_daemon_for_gpu( + "0", + root=linux_host, + run_command=fake_run(), + proc_stat=lambda pid: (pid, 777 if pid == 4242 else 778), + ) + + +def test_select_daemon_for_gpu_returns_sole_stale_record_for_cleanup( + linux_host: Path, +) -> None: + write_record(linux_host, daemon_record(linux_host, name="prod")) + + selected = mps.select_daemon_for_gpu( + "0", root=linux_host, run_command=fake_run(), proc_stat=lambda _pid: None + ) + + assert selected.name == "prod" + + +def test_stop_selection_errors_point_to_status(linux_host: Path) -> None: + with pytest.raises(mps.CudaMpsError, match=r"uni-cumps status"): + mps.stop_daemon("unknown", root=linux_host, run_command=fake_run()) + with pytest.raises(mps.CudaMpsError, match=r"uni-cumps status"): + mps._daemon_record_path(linux_host, "/tmp/not-a-label") + with pytest.raises(mps.CudaMpsError, match=r"uni-cumps status"): + mps.read_daemon_record(linux_host, None) + + +def test_cli_parser_stop_gpu_selector_is_exclusive_with_name_and_all() -> None: + args = mps._parser().parse_args(["stop", "--gpus", "0"]) + + assert args.command == "stop" + assert args.gpus == "0" + with pytest.raises(SystemExit): + mps._parser().parse_args(["stop", "--gpus", "0", "--name", "prod"]) + with pytest.raises(SystemExit): + mps._parser().parse_args(["stop", "--gpus", "0", "--all"]) + + def test_doctor_uses_sole_live_managed_daemon_pipe( linux_host: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -844,6 +914,11 @@ def test_completion_offers_cumps_commands() -> None: "--shell", "--json", ] + assert cli_completion.complete_words(["uv", "run", "uni-cumps", "stop", "-"], 4) == [ + "--name", + "--gpus", + "--all", + ] def test_regular_file_control_pipe_is_rejected_by_probe(linux_host: Path) -> None: