From 1fda094dfe8c4a5b20e0abc53038d40ad1bfd52a Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 11:23:47 -0500 Subject: [PATCH 01/12] feat(models): uploads up to 25 GB resume after a drop --- .github/workflows/build-and-test.yml | 6 + CHANGELOG.md | 10 + README.md | 4 +- comfy_cli/command/assets_library.py | 205 ++++++++--- comfy_cli/command/model_upload_resume.py | 251 ++++++++++++++ comfy_cli/command/models/upload.py | 4 +- ...oads-only-for-the-same-file-and-account.md | 76 +++++ .../comfy_cli/command/test_assets_library.py | 247 +++++++++++++- .../command/test_model_upload_resume.py | 322 ++++++++++++++++++ 9 files changed, 1065 insertions(+), 60 deletions(-) create mode 100644 comfy_cli/command/model_upload_resume.py create mode 100644 docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md create mode 100644 tests/comfy_cli/command/test_model_upload_resume.py diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 0f5da884c..c9c2e2f9f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -62,6 +62,12 @@ jobs: run: | uv run --locked --extra dev pytest tests/comfy_cli/command/test_jobs_pid_alive.py -v + - name: Test model upload and resume + env: + PYTHONPATH: ${{ github.workspace }} + run: | + uv run --locked --extra dev pytest tests/comfy_cli/command/test_assets_library.py tests/comfy_cli/command/test_model_upload_resume.py + - name: Test e2e env: PYTHONPATH: ${{ github.workspace }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f505eb04..2516354bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,16 @@ history. ## [Unreleased] +### Changed + +- Both `comfy assets library upload` and `comfy model upload` request resumable + model uploads up to 25 GB. Dropped connections retry from storage's held + bytes, and running the command again within seven days continues a saved + upload for the same file, account, workspace and folder. Older servers keep + the single-PUT upload. File approval, block checks and the final stored-hash + check still apply. The final storage check no longer stops after fifteen + minutes. + ### Added - The `comfy-run-on-cloud` agent skill, installed with the other bundled diff --git a/README.md b/README.md index 11d64ca35..447f42fb2 100644 --- a/README.md +++ b/README.md @@ -444,7 +444,9 @@ the bisect tool can help you pinpoint the custom node that causes the issue. `comfy model upload --type --where cloud ?[--dry-run] ?[--workspace ] ?[--expected-sha256 ] ?[--yes]` - - Puts one local `.safetensors` file (up to 2 GB) into your Comfy Cloud model library, so a workflow run on Comfy Cloud can load it. `--type` is the model folder, e.g. `loras` or `checkpoints`; `comfy model list-folders --where cloud` lists them. + - Puts one local `.safetensors` file (up to 25 GB on servers with resumable uploads) into your Comfy Cloud model library, so a workflow run on Comfy Cloud can load it. `--type` is the model folder, e.g. `loras` or `checkpoints`; `comfy model list-folders --where cloud` lists them. + - Uploads use 64 MiB pieces when the server supports them. A dropped connection retries from the bytes storage holds. If the command stops, run it again with the same file, model folder, account and workspace within seven days to continue. The CLI hashes the file again before resuming. Older servers keep using one PUT and enforce their own size limit. Upload sessions are credentials, kept in a private `model-uploads` directory under the CLI config folder and removed after completion. + - After 30 minutes without progress, the command exits with the held byte count. The final check on Comfy Cloud can take longer than fifteen minutes; the command waits until it ends. - Only the file you name is uploaded, and only the bytes hashed before the upload: each block is checked against its hash before it is sent. The upload succeeds only once Comfy Cloud's hash of the stored file matches your file's. - `--dry-run` uploads nothing. It reports the file's `sha256` and whether Comfy Cloud already has the file. - The output's `loader_path` is the value a loader node takes for the model on Comfy Cloud, as Comfy Cloud's record of the model gives it. It is null for a dry run that would upload, or when Comfy Cloud gives none. diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index 7ee9bf5e0..e0aff125d 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -346,9 +346,9 @@ def ensure_cmd( # What Comfy Cloud takes from `upload` (POST /api/assets/uploads): one -# `.safetensors` file of 1 byte to 2 GiB under a bare name of at most 255 bytes. +# `.safetensors` file under a bare name of at most 255 bytes. Start enforces +# the size limit (up to 25 GiB when the server supports resumable uploads). # A pickle format (.ckpt, .pt, .bin) is refused: it can run code when loaded. -_MAX_UPLOAD_BYTES = 2 * 1024**3 _MAX_FILENAME_BYTES = 255 # The first upload plus one fresh one after a PUT that got no answer or a 5xx. # Fresh, not a second PUT to the same URL: a signed URL lasts 15 minutes, which @@ -358,9 +358,8 @@ def ensure_cmd( # id names a fresh object, so it can only be this upload's: the bytes landed. _ALREADY_WRITTEN = 412 _PUT_TIMEOUT = (10, 600) -# Storing the file (hashing up to 2 GiB) normally takes a minute or two. +# Large uploads can take more than fifteen minutes to check and store. _TASK_POLL_SECONDS = 2.0 -_TASK_WAIT_SECONDS = 15 * 60.0 # A task read that gets a 5xx or no answer is retried for this long from the # first failure in a row; a 4xx ends the wait at once. _UNANSWERED_WINDOW_SECONDS = 60.0 @@ -370,7 +369,7 @@ def ensure_cmd( "MODEL_STORAGE_LIMIT_REACHED": "delete models you no longer need, or move to a plan with more model storage", "UNSUPPORTED_FILE_TYPE": "convert the model to .safetensors", "UNSUPPORTED_MODEL_TYPE": "pass a --type from `model_upload_folders` in GET /api/features", - "FILE_TOO_LARGE": "files can be up to 2 GB", + "FILE_TOO_LARGE": "use a file within the size limit in the server message", } # Any form Comfy Cloud reports a model's content hash in: bare hex, canonical # ``blake3:`` or the stored ``.``, any case. @@ -423,11 +422,14 @@ class _HashedBytes: since are not sent. """ - def __init__(self, raw, digests) -> None: + def __init__(self, raw, digests, offset: int = 0) -> None: self._raw, self._digests = raw, digests self._pending = b"" - self._next_block = 0 - self._given = 0 + self._next_block = offset // digests.block_size + self._skip = offset % digests.block_size + self._given = offset + if offset: + raw.seek(self._next_block * digests.block_size) def read(self, size: int = -1) -> bytes: from blake3 import blake3 @@ -439,7 +441,8 @@ def read(self, size: int = -1) -> bytes: block = self._raw.read(min(d.block_size, d.size - self._next_block * d.block_size)) if blake3(block).digest() != d.blocks[self._next_block]: raise _BlockChanged(f"block {self._next_block} of the file is not the one hashed") - self._pending, self._next_block = block, self._next_block + 1 + self._pending, self._next_block = block[self._skip :], self._next_block + 1 + self._skip = 0 n = len(self._pending) if size is None or size < 0 else size out, self._pending = self._pending[:n], self._pending[n:] self._given += len(out) @@ -498,8 +501,6 @@ def _local_refusal(path: Path) -> tuple[str, str] | None: return _file_problem(path, e) if size < 1: return "empty_file", f"{path} is empty" - if size > _MAX_UPLOAD_BYTES: - return "file_too_large", f"{name} is {size} bytes; files can be up to 2 GB ({_MAX_UPLOAD_BYTES} bytes)" return None @@ -718,6 +719,70 @@ def read(self, size: int = -1) -> bytes: raise _PutFailed(resp.status_code) +def _read_range(path, digests, offset: int, length: int) -> bytes: + try: + with path.open("rb") as raw: + reader = _HashedBytes(raw, digests, offset) + blocks = [] + while length: + block = reader.read(length) + if not block: + raise _FileChanged() + blocks.append(block) + length -= len(block) + return b"".join(blocks) + except _BlockChanged: + raise _FileChanged() from None + except OSError as e: + raise _FileUnreadable(e) from None + + +def _upload_identity(target, sign_in) -> dict | None: + """Only server-confirmed account and workspace ids permit saved-session reuse.""" + import http.client + import urllib.error + + try: + _, user = _send(target, sign_in, target.url("user"), strict_json=True) + workspace = _current_workspace(target, sign_in) + except (urllib.error.URLError, OSError, http.client.HTTPException, ResponseUnparseable): + return None + account = user.get("id") if isinstance(user, dict) else None + if not isinstance(account, str) or not account or not workspace: + return None + return {"account": account, "workspace": workspace} + + +def _complete_upload(renderer, target, sign_in, upload_id: str, body: dict, *, resumable: bool) -> dict: + import http.client + import urllib.error + import urllib.parse + + from comfy_cli.command.model_upload_resume import STALL_SECONDS, SessionExpired + + parts = ("assets", "uploads", urllib.parse.quote(upload_id, safe=""), "complete") + if not resumable: + return _call(renderer, target, "complete", parts, body, sign_in=sign_in) + started, delay = _clock(), 1 + while True: + try: + _, parsed = _send(target, sign_in, target.url(*parts), method="POST", body=body, strict_json=True) + except (urllib.error.URLError, OSError, http.client.HTTPException) as e: + code = e.code if isinstance(e, urllib.error.HTTPError) else None + if code == 404: + raise SessionExpired() from None + if (code is not None and code != 429 and code < 500) or _clock() - started >= STALL_SECONDS: + raise _http_failure(renderer, e, "complete", sign_in) from None + _sleep(min(delay, STALL_SECONDS - (_clock() - started))) + delay = min(delay * 2, 60) + except ResponseUnparseable: + raise _unexpected(renderer, "/api/assets/uploads/complete", "body was not valid JSON") from None + else: + if not isinstance(parsed, dict): + raise _unexpected(renderer, "/api/assets/uploads/complete", "expected a JSON object") + return parsed + + def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: """Read the task until it ends, riding out a brief 5xx, 429 or dropped connection.""" import http.client @@ -727,7 +792,7 @@ def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: from comfy_cli.command._cloud_errors import retry_after_from_headers url = target.url("tasks", urllib.parse.quote(task_id, safe="")) - deadline = _clock() + _TASK_WAIT_SECONDS + started, last_notice = _clock(), _clock() unanswered_since: float | None = None status = None while True: @@ -744,8 +809,8 @@ def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: raise _http_failure(renderer, e, "task", sign_in) from e retry_after = retry_after_from_headers(getattr(e, "headers", None)) if code == 429 else None if retry_after is not None: - # Never past the end of the window or of the whole wait. - left = min(unanswered_since + _UNANSWERED_WINDOW_SECONDS, deadline) - now + # Never past the end of the unanswered window. + left = unanswered_since + _UNANSWERED_WINDOW_SECONDS - now pause = max(_TASK_POLL_SECONDS, min(retry_after, left)) except ResponseUnparseable as e: raise _unexpected(renderer, "/api/tasks", "body was not valid JSON", task_id=task_id) from e @@ -756,14 +821,9 @@ def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: status = task["status"] if status in _TASK_DONE: return task - if _clock() >= deadline: - renderer.error( - code="asset_upload_timeout", - message=f"Comfy Cloud was still storing the file after {int(_TASK_WAIT_SECONDS)}s", - hint="storing goes on without this command; check `comfy assets library ls --tags models` later", - details={"task_id": task_id, "status": status}, - ) - raise typer.Exit(code=1) + if renderer.is_pretty() and _clock() - last_notice >= 30: + renderer.info(f"Comfy Cloud is still checking and storing the file ({int(_clock() - started)}s so far).") + last_notice = _clock() _sleep(pause) @@ -859,7 +919,7 @@ def _loader_path(asset: dict | None) -> str | None: @app.command( "upload", - help="Upload one named .safetensors model file (up to 2 GB) into your Comfy Cloud model library. " + help="Upload one named .safetensors model file (up to 25 GB) into your Comfy Cloud model library. Run the same command again to resume a stopped upload. " "Succeeds only once Comfy Cloud's hash of the stored file matches this file's; --dry-run checks without uploading.", ) @tracking.track_command("assets") @@ -906,9 +966,16 @@ def run_upload( the folder; without it the file is uploaded under that name, as a new version of it. """ - import urllib.parse - from comfy_cli.command.build_upload_progress import UploadProgressReporter + from comfy_cli.command.model_upload_resume import ( + SessionExpired, + SessionStore, + TransferFailed, + UploadPaused, + transfer, + valid_start, + ) + from comfy_cli.config_manager import ConfigManager from comfy_cli.credentials import DataPlaneToken from comfy_cli.hashing import file_digests from comfy_cli.interaction import confirm @@ -987,6 +1054,7 @@ def refuse(reason: str, message: str, **details: Any) -> typer.Exit: # One read gives the hashes, the size and each block's digest; the PUT # sends only bytes that match these blocks (`_HashedBytes`). digests = file_digests(file) + modified_ns = file.stat().st_mtime_ns except OSError as e: # removed or locked since it was checked; nothing is sent yet raise refuse(*_file_problem(file, e)) from None local_hash, sha256, size = digests.blake3, digests.sha256, digests.size @@ -1063,31 +1131,74 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None tags = ["models", folder] reporter = UploadProgressReporter(renderer) reporter.plan([item], already_held=0) + store = SessionStore( + Path(ConfigManager().get_config_path()) / "model-uploads", + { + "server": target.url(), + "path": str(file.resolve()), + "size": size, + "mtime_ns": modified_ns, + "hash": local_hash, + "folder": folder, + }, + ) + identity = _upload_identity(target, sign_in) if store.has_saved() else None + saved = store.load(identity) for attempt in range(1, _UPLOAD_STARTS + 1): - start = _call( + resumed = saved is not None + start = saved or _call( renderer, target, "start", ("assets", "uploads"), - {"filename": item.filename, "size_bytes": item.size_bytes, "tags": tags}, + {"filename": item.filename, "size_bytes": item.size_bytes, "tags": tags, "resumable": True}, sign_in=sign_in, ) - upload_id, upload_url, headers = start.get("upload_id"), start.get("upload_url"), start.get("headers") - if not ( - isinstance(upload_id, str) - and upload_id - and isinstance(upload_url, str) - and upload_url - and start.get("method") == "PUT" - and isinstance(headers, dict) - and all(isinstance(k, str) and isinstance(v, str) for k, v in headers.items()) - ): - # Never the URL itself in the error: it is a credential. + saved = None + if not valid_start(start): raise _unexpected(renderer, "/api/assets/uploads", "expected upload_id, upload_url, method PUT and headers") + upload_id, upload_url, headers = start["upload_id"], start["upload_url"], start["headers"] + resumable = start.get("upload_mode") == "resumable" + if resumable and not resumed: + identity = identity or _upload_identity(target, sign_in) + if not store.save(start, identity): + renderer.warn("This upload cannot be saved for a later run; connection retries still work.") try: with reporter.uploading(item, 1, 1) as progress: - _put(upload_url, headers, file, progress, digests) + if resumable: + transfer( + start, + size, + lambda offset, length: _read_range(file, digests, offset, length), + progress, + resumed=resumed, + clock=_clock, + sleep=_sleep, + ) + else: + _put(upload_url, headers, file, progress, digests) + task_id = _complete_upload( + renderer, + target, + sign_in, + upload_id, + {"filename": item.filename, "tags": tags}, + resumable=resumable, + ).get("task_id") + if not isinstance(task_id, str) or not task_id: + raise _unexpected(renderer, "/api/assets/uploads/complete", "no task_id", upload_id=upload_id) + store.clear() break + except UploadPaused as e: + from comfy_cli.output.progress import human_bytes + + renderer.error( + code="asset_upload_failed", + message=f"Upload paused at {human_bytes(e.args[0])} of {human_bytes(size)}. " + "Run the same command again to continue.", + details={"reason": "upload_paused", "upload_id": upload_id, "bytes_held": e.args[0]}, + ) + raise typer.Exit(code=1) from None except _FileUnreadable as e: renderer.error( code="asset_upload_failed", @@ -1098,6 +1209,7 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None ) raise typer.Exit(code=1) from None except _FileChanged: + store.clear() # Only the blocks before the changed one were sent, and the # upload is never completed, so the staged bytes never become an asset. renderer.error( @@ -1116,9 +1228,12 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None details={"reason": "tls_setup", "upload_id": upload_id, "attempts": attempt}, ) raise typer.Exit(code=1) from None - except _PutFailed as e: - status = e.args[0] - if (status is None or status >= 500) and attempt < _UPLOAD_STARTS: + except (_PutFailed, TransferFailed, SessionExpired) as e: + expired = isinstance(e, SessionExpired) + status = 404 if expired else e.args[0] + if expired: + store.clear() + if (expired or (not resumable and (status is None or status >= 500))) and attempt < _UPLOAD_STARTS: renderer.warn("The upload to storage did not go through; starting a fresh upload.") continue answer = "no response" if status is None else f"HTTP {status}" @@ -1130,12 +1245,6 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None ) raise typer.Exit(code=1) from None - complete = ("assets", "uploads", urllib.parse.quote(upload_id, safe=""), "complete") - task_id = _call( - renderer, target, "complete", complete, {"filename": item.filename, "tags": tags}, sign_in=sign_in - ).get("task_id") - if not isinstance(task_id, str) or not task_id: - raise _unexpected(renderer, "/api/assets/uploads/complete", "no task_id", upload_id=upload_id) if renderer.is_pretty(): renderer.info("Uploaded; Comfy Cloud is checking and storing the file.") task = _wait_for_task(renderer, target, task_id, sign_in) diff --git a/comfy_cli/command/model_upload_resume.py b/comfy_cli/command/model_upload_resume.py new file mode 100644 index 000000000..eda13a0ba --- /dev/null +++ b/comfy_cli/command/model_upload_resume.py @@ -0,0 +1,251 @@ +"""Private, expiring model-upload sessions and the bucket's acknowledged byte range.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +import tempfile +import time +from datetime import datetime +from pathlib import Path + +import requests + +CHUNK_BYTES = 64 * 1024**2 +LIFETIME = 7 * 24 * 60 * 60 +STALL_SECONDS = 30 * 60 +_MAX_ENTRIES = 32 +_RANGE = re.compile(r"bytes=0-([0-9]+)$") + + +class SessionExpired(Exception): + """Storage or complete no longer knows this upload; start again.""" + + +class TransferFailed(Exception): + """Only a status is safe to expose; never keep a transport exception.""" + + +class UploadPaused(Exception): + """The argument is the last byte count acknowledged by storage.""" + + +def _expiry(start: dict) -> float: + try: + return datetime.fromisoformat(start["expires_at"].replace("Z", "+00:00")).timestamp() + except (KeyError, TypeError, ValueError, OverflowError, AttributeError): + return 0.0 + + +def chunk_size(start: dict) -> int: + value = start.get("chunk_size_bytes") + if type(value) is not int or value < 256 * 1024 or value > CHUNK_BYTES or value % (256 * 1024): + raise TransferFailed(None) + return value + + +class SessionStore: + """At most 32 bounded JSON files, written atomically with user-only access. + + No credential is used to identify the account: the caller supplies the ids + returned by the authenticated user and workspace endpoints. Unknown identity + disables persistence. Failure to secure or write the directory stops saving, + but does not stop the current transfer. + """ + + def __init__(self, directory: Path, binding: dict) -> None: + self.directory, self.binding = directory, binding + self.path: Path | None = None + + def _entries(self) -> list[Path]: + if self.directory.is_symlink(): + return [] + try: + entries = list(self.directory.glob("*.json")) + temporary = sorted(self.directory.glob(".upload-*"), key=lambda p: p.lstat().st_mtime) + for index, path in enumerate(temporary): + if index < len(temporary) - _MAX_ENTRIES or time.time() - path.lstat().st_mtime >= LIFETIME: + path.unlink(missing_ok=True) + valid = [] + now = time.time() + for path in entries: + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + continue + if info.st_size > 16384 or now - info.st_mtime >= LIFETIME: + path.unlink(missing_ok=True) + else: + valid.append(path) + return sorted(valid, key=lambda p: p.stat().st_mtime) + except OSError: + return [] + + def has_saved(self) -> bool: + return bool(self._entries()) + + def _secure(self) -> None: + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + if self.directory.is_symlink(): + raise OSError("session directory is a symlink") + self.directory.chmod(0o700) + + def load(self, identity: dict | None) -> dict | None: + if identity is None: + return None + try: + self._secure() + except OSError: + return None + for path in reversed(self._entries()): + try: + info = path.lstat() + if os.name != "nt" and (info.st_uid != os.getuid() or info.st_mode & 0o077): + continue + with path.open(encoding="utf-8") as stream: + saved = json.load(stream) + if not isinstance(saved, dict) or not isinstance(saved.get("start"), dict): + continue + created = saved.get("created_at") + if not isinstance(created, (float, int)) or not math.isfinite(created) or created > time.time(): + continue + start = saved["start"] + if min(created + LIFETIME, _expiry(start)) <= time.time(): + path.unlink(missing_ok=True) + continue + if saved.get("binding") == self.binding and saved.get("identity") == identity: + # Validate before any request to a saved credential address. + if start.get("upload_mode") != "resumable" or not valid_start(start): + continue + chunk_size(start) + self.path = path + return start + except (OSError, ValueError, KeyError, TypeError, TransferFailed): + continue + return None + + def save(self, start: dict, identity: dict | None) -> bool: + if identity is None or _expiry(start) <= time.time(): + return False + temporary = None + try: + self._secure() + key = hashlib.sha256(json.dumps([self.binding, identity], sort_keys=True).encode()).hexdigest() + path = self.directory / f"{key}.json" + entries = self._entries() + for old in entries[: max(0, len(entries) - _MAX_ENTRIES + 1)]: + old.unlink(missing_ok=True) + fd, temporary = tempfile.mkstemp(prefix=".upload-", dir=self.directory) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump( + {"binding": self.binding, "identity": identity, "created_at": time.time(), "start": start}, stream + ) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + self.path = path + return True + except (OSError, ValueError): + return False + finally: + if temporary is not None: + try: + Path(temporary).unlink(missing_ok=True) + except OSError: + pass + + def clear(self) -> None: + if self.path is not None: + try: + self.path.unlink(missing_ok=True) + except OSError: + pass + self.path = None + + +def valid_start(start: dict) -> bool: + from urllib.parse import urlsplit + + try: + url = urlsplit(start.get("upload_url", "")) + return ( + isinstance(start.get("upload_id"), str) + and bool(start["upload_id"]) + and url.scheme == "https" + and bool(url.hostname) + and not url.username + and not url.password + and start.get("method") == "PUT" + and isinstance(start.get("headers"), dict) + and all(isinstance(k, str) and isinstance(v, str) for k, v in start["headers"].items()) + and start.get("upload_mode") in (None, "resumable") + ) + except (TypeError, ValueError): + return False + + +def _request(start: dict, data: bytes, content_range: str) -> tuple[int | None, str | None]: + try: + response = requests.put( + start["upload_url"], + data=data, + headers={**start["headers"], "Content-Length": str(len(data)), "Content-Range": content_range}, + timeout=(10, 60), + allow_redirects=False, + ) + except OSError: + # Requests exceptions may quote the credential, as may TLS setup errors. + return None, None + try: + return response.status_code, response.headers.get("Range") + finally: + response.close() + + +def transfer( + start: dict, size: int, read_range, progress, *, resumed: bool, clock=time.monotonic, sleep=time.sleep +) -> None: + """Query after every uncertain PUT; only acknowledged bytes advance progress. + + ``read_range`` checks each whole hashing block before returning a piece. + It may start inside a block, because a connection can drop at any byte. + """ + piece_size = chunk_size(start) + offset, query, delay, last_progress = 0, resumed, 1, clock() + while True: + if clock() - last_progress >= STALL_SECONDS: + raise UploadPaused(offset) + end = offset if query else min(offset + piece_size, size) + body = b"" if query else read_range(offset, end - offset) + content_range = f"bytes */{size}" if query else f"bytes {offset}-{end - 1}/{size}" + status, held = _request(start, body, content_range) + if status in (200, 201, 412): + progress(size - offset) + return + if status in (404, 410): + raise SessionExpired() + if status == 308: + match = _RANGE.fullmatch(held) if isinstance(held, str) else None + if held is not None and match is None: + raise TransferFailed(308) + acknowledged = int(match.group(1)) + 1 if match else 0 + if acknowledged < offset or acknowledged > size or (not query and acknowledged > end): + raise TransferFailed(308) + if acknowledged > offset: + progress(acknowledged - offset) + offset, last_progress, delay = acknowledged, clock(), 1 + if offset == size: + return + query = False + continue + # A fresh empty status is normal; a piece with no progress backs off. + if query: + query = False + continue + elif status is not None and status != 429 and status < 500: + raise TransferFailed(status) + sleep(min(delay, max(0, STALL_SECONDS - (clock() - last_progress)))) + delay, query = min(delay * 2, 60), True diff --git a/comfy_cli/command/models/upload.py b/comfy_cli/command/models/upload.py index 965e2d4fd..42bb7b2e6 100644 --- a/comfy_cli/command/models/upload.py +++ b/comfy_cli/command/models/upload.py @@ -21,9 +21,9 @@ @app.command( "upload", - help="Upload one named .safetensors model file (up to 2 GB) to your Comfy Cloud model library, so a " + help="Upload one named .safetensors model file (up to 25 GB) to your Comfy Cloud model library, so a " "workflow run on Comfy Cloud can load it. Uploads only the file named; succeeds only once Comfy Cloud's " - "hash of the stored file matches this file's. Run with --dry-run first to see its sha256 and whether " + "hash of the stored file matches this file's. Dropped uploads resume when you run the same command again. Run with --dry-run first to see its sha256 and whether " "Comfy Cloud already has it.", ) @tracking.track_command("model") diff --git a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md new file mode 100644 index 000000000..3c9a84f6c --- /dev/null +++ b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md @@ -0,0 +1,76 @@ +# ADR-TRANSFER-0012: Resume Model Uploads Only for the Same File and Account + +Date: 2026-11-06 + +## Status + +Proposed + +Relation: Builds on [ADR-TRANSFER-0011](TRANSFER-0011-upload-only-the-named-model-and-prove-cloud-stored-its-bytes.md). +Approval, name-reuse checks, checking each hashing block before sending it, and +proving the stored hash all remain in effect. The success envelope is unchanged. + +## Context + +A user uploads a large checkpoint and their connection drops. Sending one PUT +again loses the bytes already sent. A storage session can continue that upload, +but its address is a credential and must not follow a later sign-in or changed +file into another upload. + +## Decision + +Both model upload commands request resumable uploads. A response without an +upload mode keeps the existing single-PUT path. The server decides the size +limit; the CLI does not keep a separate upper limit. + +The CLI owns saved sessions in a private directory under its configuration +folder. An atomic replacement writes each session with user-only permissions. +At most 32 session records and 32 interrupted-save temporary files remain. +Sessions expire at the earlier of the server's expiry or seven days after +creation. Oversized, malformed, expired and insecure records are never reused. +On Windows, file access follows the user's configuration-directory ACL; POSIX +also checks ownership and mode 0600. + +Reuse requires authenticated user and workspace ids from Cloud, the same server, +resolved path, folder, hashed size, modified time and freshly computed content +hash. Unknown identity disables persistence. A failure to save is reported and +still permits retries within that process. + +Storage's status query decides where to resume. Each piece is at most 64 MiB. +A resumed offset inside a hashing block still checks that whole block before +sending any part of it. Only acknowledged bytes advance progress. Transport +errors keep no exception text because it can contain the session credential. +Expired sessions restart; thirty minutes without acknowledged progress pauses. +Completion deletes the session after a valid task response, and the command +waits for the task without a total deadline. Repeated unanswered task polls +still have a bounded retry window. + +## Alternatives considered + +- Keep one PUT and raise only the size limit. A drop still restarts the file. +- Reuse state by path alone. This could send bytes for a different file, + account, workspace or folder. +- Trust size and modified time alone. Files can change without either changing. +- Save session addresses in ordinary config or output. That exposes credentials. + +## Consequences + +Reruns hash the whole file again. Storage receives only newly needed pieces, +and no temporary copy of a multi-GB model is created. State is local to one +machine; simultaneous invocations are not coordinated and may start separate +uploads. Losing saved state loses the ability to resume that session, but a +new upload still works. A task may continue after the command loses its sign-in. + +## Validation + +The model-upload tests use a fake bucket for mid-piece drops, process reruns, +final-piece recovery, refusal, expiry, transient errors and byte verification. +State tests cover each identity field, expiration, permissions and bounds. +Existing command tests retain approval and stored-hash assertions. CI runs +these tests on Linux, macOS and Windows. Real multi-GB storage is not exercised +by the fake bucket. + +## Revisit triggers + +Revisit when storage changes its resumable protocol or lifetime, when users +need to resume across machines, or when concurrent uploads must share a session. diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index b9ae2bbd4..c900b8570 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -44,7 +44,12 @@ @pytest.fixture(autouse=True) -def reset_singleton(): +def reset_singleton(monkeypatch, tmp_path): + from comfy_cli import constants + + monkeypatch.setattr( + constants, "DEFAULT_CONFIG", {key: str(tmp_path / "config") for key in constants.DEFAULT_CONFIG} + ) reset_renderer_for_testing() yield reset_renderer_for_testing() @@ -969,7 +974,12 @@ def test_uploads_completes_and_reports_the_stored_asset( "/api/tasks/task-1", "/api/assets/asset-9", ] - assert cloud.calls[2][2] == {"filename": "style.safetensors", "size_bytes": 16, "tags": ["models", "loras"]} + assert cloud.calls[2][2] == { + "filename": "style.safetensors", + "size_bytes": 16, + "tags": ["models", "loras"], + "resumable": True, + } assert cloud.calls[3][2] == {"filename": "style.safetensors", "tags": ["models", "loras"]} (put,) = storage.puts assert put["url"] == f"{_SIGNED}1" and put["headers"] == _PUT_HEADERS and put["allow_redirects"] is False @@ -986,7 +996,6 @@ def test_uploads_completes_and_reports_the_stored_asset( ("x" * 250 + ".safetensors", None, "bad_filename"), ("style.safetensors", None, "file_not_found"), ("style.safetensors", 0, "empty_file"), - ("style.safetensors", 2 * 1024**3 + 1, "file_too_large"), ], ) def test_a_file_comfy_cloud_would_refuse_is_refused_before_any_request( @@ -1003,10 +1012,11 @@ def test_a_file_comfy_cloud_would_refuse_is_refused_before_any_request( assert env["error"]["details"]["reason"] == reason assert cloud.calls == [] and storage.puts == [] - def test_exactly_2_gib_is_accepted_locally(self, tmp_path): + @pytest.mark.parametrize("size", [2 * 1024**3 + 1, 25 * 1024**3, 25 * 1024**3 + 1]) + def test_size_limit_is_left_to_the_server(self, tmp_path, size): path = tmp_path / "big.SAFETENSORS" with path.open("wb") as f: - f.truncate(2 * 1024**3) + f.truncate(size) assert assets_library._local_refusal(path) is None def test_unknown_type_is_refused_listing_the_valid_ones_before_the_upload( @@ -1137,13 +1147,13 @@ def test_a_task_4xx_ends_the_wait_at_once(self, cloud_target, monkeypatch, capsy assert rc == 1 and cloud.paths().count("/api/tasks/task-1") == 1 - def test_a_task_that_never_ends_times_out(self, cloud_target, monkeypatch, capsys, model_file, no_wait): + def test_a_task_can_take_longer_than_fifteen_minutes(self, cloud_target, monkeypatch, capsys, model_file, no_wait): routes = dict(_UPLOAD_ROUTES) - routes[("GET", "/api/tasks/task-1")] = {"id": "task-1", "status": "running"} + routes[("GET", "/api/tasks/task-1")] = [{"id": "task-1", "status": "running"}] * 500 + [_DONE] env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) - assert rc == 1 and env["error"]["code"] == "asset_upload_timeout" - assert env["error"]["details"] == {"task_id": "task-1", "status": "running"} + assert rc == 0 and env["data"]["id"] == "asset-9" + assert no_wait[0] > 15 * 60 @pytest.mark.parametrize( ("route", "answer"), @@ -2348,3 +2358,222 @@ def test_it_is_registered_under_the_model_noun_with_a_schema(self): assert "upload" in {c.name for c in models.app.registered_commands} assert COMMAND_SCHEMAS["comfy model upload"] == "model_upload" assert STREAM_EVENT_SCHEMAS["comfy model upload"] == "build_push_event" + + +# --- Resumable uploads through both public commands ---------------------------- + + +def _resumable_started(n=1): + import time + from datetime import datetime, timezone + + return { + **_started(n), + "upload_mode": "resumable", + "chunk_size_bytes": 64 * 1024**2, + "expires_at": datetime.fromtimestamp(time.time() + 7 * 24 * 3600, timezone.utc).isoformat(), + } + + +def _resume_routes(): + return { + **_UPLOAD_ROUTES, + ("POST", "/api/assets/uploads"): [_resumable_started(1), _resumable_started(2)], + ("GET", "/api/user"): {"id": "user-1"}, + ("GET", "/api/workspaces/current"): {"id": "ws-1"}, + } + + +class _ResumableBucket: + def __init__(self, *, crash=False, held=b"", status=None): + self.crash, self.held, self.status = crash, held, status + self.puts = [] + + def __call__(self, url, *, data, headers, **kw): + self.puts.append({"url": url, "headers": headers, "body": data, **kw}) + assert kw["allow_redirects"] is False + if self.crash: + self.held = data[:5] + self.crash = False + raise KeyboardInterrupt() + if not headers["Content-Range"].startswith("bytes */"): + offset = int(headers["Content-Range"].split()[1].split("-")[0]) + assert offset == len(self.held) + self.held += data + code = self.status or (200 if len(self.held) == 16 else 308) + return SimpleNamespace( + status_code=code, + headers={"Range": f"bytes=0-{len(self.held) - 1}"} if self.held else {}, + close=lambda: None, + ) + + +class TestResumableCommands: + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + def test_killed_upload_resumes_the_same_session_on_rerun( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model + ): + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + ( + ["--yes", "--workspace", "ws-1", "--expected-sha256", _MODEL_SHA256] if model else [] + ) + bucket = _ResumableBucket(crash=True) + # CliRunner converts KeyboardInterrupt to an exit; the session has already been saved. + monkeypatch.setattr("requests.put", bucket) + cloud = _Cloud(_resume_routes()) + monkeypatch.setattr("urllib.request.urlopen", cloud) + from comfy_cli.command import models + + result = CliRunner().invoke(models.app if model else assets_library.app, ["upload", *args, "--where", "cloud"]) + assert result.exit_code != 0 + capsys.readouterr() + bucket.puts.clear() + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=bucket) + assert rc == 0 and env["data"]["uploaded"] is True + assert "/api/assets/uploads" not in cloud.paths() + assert bucket.puts[0]["headers"]["Content-Range"] == "bytes */16" + assert bucket.puts[1]["headers"]["Content-Range"] == "bytes 5-15/16" + assert bucket.held == _MODEL_BYTES + assert "SECRETSIG" not in everything + from comfy_cli.config_manager import ConfigManager + + assert list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) == [] + + def test_crash_after_final_piece_completes_without_resending( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _cloud_error(403, "NO", "try again") + bucket = _ResumableBucket() + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=bucket) + assert rc == 1 and bucket.held == _MODEL_BYTES + bucket.puts.clear() + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], _resume_routes(), [], storage=bucket + ) + assert rc == 0 and "/api/assets/uploads" not in cloud.paths() + assert [p["headers"]["Content-Range"] for p in bucket.puts] == ["bytes */16"] + + @pytest.mark.parametrize("expired_at", ["bucket", "complete"]) + def test_expired_session_starts_fresh(self, cloud_target, monkeypatch, capsys, model_file, no_wait, expired_at): + routes = _resume_routes() + if expired_at == "complete": + routes[("POST", _COMPLETE_1)] = _http_error(404) + bucket = _ResumableBucket(status=404 if expired_at == "bucket" else None) + original = bucket.__call__ + + def put(url, **kw): + if url.endswith("2"): + bucket.status = None + bucket.held = b"" + return original(url, **kw) + + monkeypatch.setattr("requests.put", put) + # _upload accepts a callable in place of its storage fake. + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=put + ) + assert rc == 0 and cloud.paths("POST").count("/api/assets/uploads") == 2 + + @pytest.mark.parametrize("change", ["workspace", "account", "same-size-bytes", "folder", "mtime", "size", "server"]) + def test_rerun_with_a_changed_file_or_identity_starts_fresh( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, change + ): + bucket = _ResumableBucket(status=403) + args = [str(model_file), "--type", "loras"] + env, rc, *_ = _upload(monkeypatch, capsys, args, _resume_routes(), [], storage=bucket) + assert rc == 1 + routes = _resume_routes() + if change == "workspace": + routes[("GET", "/api/workspaces/current")] = {"id": "ws-2"} + elif change == "account": + routes[("GET", "/api/user")] = {"id": "user-2"} + elif change == "same-size-bytes": + info = model_file.stat() + model_file.write_bytes(b"X" + _MODEL_BYTES[1:]) + os.utime(model_file, ns=(info.st_atime_ns, info.st_mtime_ns)) + elif change == "mtime": + info = model_file.stat() + os.utime(model_file, ns=(info.st_atime_ns, info.st_mtime_ns + 1000000000)) + elif change == "size": + model_file.write_bytes(_MODEL_BYTES + b"x") + elif change == "folder": + args[-1] = "checkpoints" + else: + from dataclasses import replace + + monkeypatch.setattr( + "comfy_cli.target.resolve_target", lambda **kw: replace(cloud_target, base_url="https://other.example") + ) + # A refusal at storage is enough to see whether start was called, regardless of stored hash. + bucket.held = b"" + env, rc, cloud, _, _ = _upload(monkeypatch, capsys, args, routes, [], storage=bucket) + assert rc == 1 and "/api/assets/uploads" in cloud.paths() + assert not bucket.puts[-1]["headers"]["Content-Range"].startswith("bytes */") + + def test_resumable_still_checks_changed_blocks_and_stored_hash( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + routes = _resume_routes() + routes[("POST", "/api/assets/uploads")] = _then( + lambda: model_file.write_bytes(b"X" + _MODEL_BYTES[1:]), _resumable_started() + ) + bucket = _ResumableBucket() + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=bucket + ) + assert rc == 1 and env["error"]["details"]["reason"] == "file_changed" + assert bucket.puts == [] and _COMPLETE_1 not in cloud.paths() + model_file.write_bytes(_MODEL_BYTES) + routes = _resume_routes() + routes[("GET", "/api/tasks/task-1")] = {**_DONE, "result": {**_DONE["result"], "hash": "b" * 64}} + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=bucket) + assert rc == 1 and env["error"]["code"] == "asset_upload_unverified" + + def test_complete_retries_transient_errors(self, cloud_target, monkeypatch, capsys, model_file, no_wait): + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = [_http_error(429), _http_error(503), {"task_id": "task-1"}] + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=_ResumableBucket() + ) + assert rc == 0 and cloud.paths().count(_COMPLETE_1) == 3 + + +class TestResumableRefusals: + def test_server_size_refusal_prints_its_message_before_sending(self, cloud_target, monkeypatch, capsys, model_file): + routes = _resume_routes() + routes[("POST", "/api/assets/uploads")] = _cloud_error(400, "FILE_TOO_LARGE", "Model files can be up to 25 GB.") + env, rc, _, storage, _ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, []) + assert rc == 1 and env["error"]["message"] == "Model files can be up to 25 GB." + assert storage.puts == [] + + @pytest.mark.parametrize("endpoint", ["user", "workspaces/current"]) + def test_unknown_identity_does_not_save_a_session( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, endpoint + ): + routes = _resume_routes() + routes[("GET", "/api/" + endpoint)] = {} + bucket = _ResumableBucket() + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=bucket) + assert rc == 0 + from comfy_cli.config_manager import ConfigManager + + assert list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) == [] + + def test_paused_upload_redacts_the_credential_and_keeps_the_state( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + import requests + + def lost(*a, **kw): + raise requests.ConnectionError("SECRETSIG") + + env, rc, _, _, everything = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], _resume_routes(), [], storage=lost + ) + assert rc == 1 and env["error"]["details"]["reason"] == "upload_paused" + assert "SECRETSIG" not in everything + assert "Run the same command again" in env["error"]["message"] + from comfy_cli.config_manager import ConfigManager + + assert len(list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json"))) == 1 diff --git a/tests/comfy_cli/command/test_model_upload_resume.py b/tests/comfy_cli/command/test_model_upload_resume.py new file mode 100644 index 000000000..bd55ea8b7 --- /dev/null +++ b/tests/comfy_cli/command/test_model_upload_resume.py @@ -0,0 +1,322 @@ +"""A fake bucket holds bytes across connection drops and process reruns.""" + +import json +import os +import time +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest +import requests + +from comfy_cli.command import assets_library as uploads +from comfy_cli.command import model_upload_resume as resume +from comfy_cli.hashing import file_digests + + +@pytest.fixture +def session(): + return { + "upload_id": "upload-1", + "upload_url": "https://bucket.example/upload?secret=SESSION", + "upload_mode": "resumable", + "method": "PUT", + "headers": {}, + "chunk_size_bytes": 256 * 1024, + "expires_at": datetime.fromtimestamp(time.time() + resume.LIFETIME, timezone.utc).isoformat(), + } + + +class Bucket: + def __init__(self, *, held=b"", drop=None, lose_final=False): + self.held, self.drop, self.lose_final = held, drop, lose_final + self.calls = [] + + def __call__(self, url, *, data, headers, timeout, allow_redirects): + assert allow_redirects is False + assert headers["Content-Length"] == str(len(data)) + content_range = headers["Content-Range"] + self.calls.append(content_range) + size = int(content_range.split("/")[1]) + if not content_range.startswith("bytes */"): + offset = int(content_range.split()[1].split("-")[0]) + assert offset == len(self.held) + if self.drop is not None: + self.held += data[: self.drop] + self.drop = None + raise requests.ConnectionError(url) + self.held += data + if self.lose_final and len(self.held) == size: + self.lose_final = False + raise requests.ConnectionError(url) + code = 200 if len(self.held) == size else 308 + response_headers = {"Range": f"bytes=0-{len(self.held) - 1}"} if self.held else {} + return SimpleNamespace(status_code=code, headers=response_headers, close=lambda: None) + + +@pytest.fixture +def file(tmp_path): + path = tmp_path / "checkpoint.safetensors" + path.write_bytes(bytes(range(256)) * 2500) + return path + + +@pytest.fixture +def clock(): + now = [0.0] + return lambda: now[0], lambda s: now.__setitem__(0, now[0] + s) + + +def send(session, file, clock, *, resumed=False): + digests = file_digests(file, block_size=4096) + progress = [] + resume.transfer( + session, + digests.size, + lambda o, n: uploads._read_range(file, digests, o, n), + progress.append, + resumed=resumed, + clock=clock[0], + sleep=clock[1], + ) + return progress + + +def test_a_drop_mid_piece_queries_the_held_range_before_sending(session, file, clock, monkeypatch): + bucket = Bucket(drop=12345) + monkeypatch.setattr(requests, "put", bucket) + assert sum(send(session, file, clock)) == file.stat().st_size + assert bucket.held == file.read_bytes() + assert bucket.calls[:3] == [ + f"bytes 0-262143/{file.stat().st_size}", + f"bytes */{file.stat().st_size}", + f"bytes 12345-274488/{file.stat().st_size}", + ] + + +def test_a_rerun_queries_before_sending_and_checks_the_partial_hash_block(session, file, clock, monkeypatch): + bucket = Bucket(held=file.read_bytes()[:12345]) + monkeypatch.setattr(requests, "put", bucket) + assert sum(send(session, file, clock, resumed=True)) == file.stat().st_size + assert bucket.held == file.read_bytes() + assert bucket.calls[0] == f"bytes */{file.stat().st_size}" + + +@pytest.mark.parametrize("rerun", [False, True]) +def test_last_piece_without_an_answer_goes_to_complete(session, file, clock, monkeypatch, rerun): + bucket = Bucket(held=file.read_bytes() if rerun else b"", lose_final=not rerun) + monkeypatch.setattr(requests, "put", bucket) + send(session, file, clock, resumed=rerun) + assert bucket.calls[-1] == f"bytes */{file.stat().st_size}" + if rerun: + assert len(bucket.calls) == 1 + + +@pytest.mark.parametrize("status", [429, 500, 503, None]) +def test_transient_errors_back_off_and_query(session, file, clock, monkeypatch, status): + bucket = Bucket() + calls = [] + + def put(*args, **kw): + calls.append(kw["headers"]["Content-Range"]) + if len(calls) == 1: + if status is None: + raise requests.ConnectionError("SESSION") + return SimpleNamespace(status_code=status, headers={}, close=lambda: None) + return bucket(*args, **kw) + + monkeypatch.setattr(requests, "put", put) + send(session, file, clock) + assert calls[1].startswith("bytes */") + assert clock[0]() == 1 + + +@pytest.mark.parametrize( + "code,exception", + [ + (404, resume.SessionExpired), + (410, resume.SessionExpired), + (403, resume.TransferFailed), + (307, resume.TransferFailed), + ], +) +def test_expiry_or_refusal_never_follows_a_redirect(session, file, clock, monkeypatch, code, exception): + monkeypatch.setattr( + requests, "put", lambda *a, **k: SimpleNamespace(status_code=code, headers={}, close=lambda: None) + ) + with pytest.raises(exception): + send(session, file, clock) + + +def test_412_goes_to_complete(session, file, clock, monkeypatch): + monkeypatch.setattr( + requests, "put", lambda *a, **k: SimpleNamespace(status_code=412, headers={}, close=lambda: None) + ) + assert sum(send(session, file, clock)) == file.stat().st_size + + +@pytest.mark.parametrize("held", ["bad", "bytes=1-20", "bytes=0-9999999"]) +def test_bad_ranges_are_refused(session, file, clock, monkeypatch, held): + monkeypatch.setattr( + requests, "put", lambda *a, **k: SimpleNamespace(status_code=308, headers={"Range": held}, close=lambda: None) + ) + with pytest.raises(resume.TransferFailed): + send(session, file, clock) + + +def test_thirty_minutes_without_acknowledged_bytes_pauses_without_credentials(session, file, clock, monkeypatch): + def lost(*a, **kw): + raise requests.ConnectionError(session["upload_url"]) + + monkeypatch.setattr(requests, "put", lost) + with pytest.raises(resume.UploadPaused) as error: + send(session, file, clock) + assert error.value.args == (0,) + assert "SESSION" not in str(error.value) + assert clock[0]() == resume.STALL_SECONDS + + +def test_a_changed_partial_block_sends_none_of_its_bytes(file): + digests = file_digests(file, block_size=4096) + original = file.read_bytes() + file.write_bytes(b"X" + original[1:]) + with pytest.raises(uploads._FileChanged): + uploads._read_range(file, digests, 123, 10) + + +def test_shortened_file_is_refused(file): + digests = file_digests(file, block_size=4096) + file.write_bytes(b"x") + with pytest.raises(uploads._FileChanged): + uploads._read_range(file, digests, 123, 10) + + +def test_state_is_private_atomic_and_round_trips(tmp_path, session): + store = resume.SessionStore(tmp_path / "sessions", {"path": "model", "hash": "hash"}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + assert store.load(identity) == session + if os.name != "nt": + assert store.path.stat().st_mode & 0o777 == 0o600 + assert store.directory.stat().st_mode & 0o777 == 0o700 + assert not list(store.directory.glob(".upload-*")) + store.clear() + assert store.load(identity) is None + + +@pytest.mark.parametrize("field", ["server", "path", "size", "mtime_ns", "hash", "folder"]) +def test_changed_file_or_destination_never_reuses_state(tmp_path, session, field): + binding = { + "server": "https://example/api", + "path": "model", + "size": 10, + "mtime_ns": 123, + "hash": "a", + "folder": "loras", + } + identity = {"account": "user", "workspace": "ws"} + store = resume.SessionStore(tmp_path / "sessions", binding) + assert store.save(session, identity) + changed = {**binding, field: "different"} + assert resume.SessionStore(store.directory, changed).load(identity) is None + + +@pytest.mark.parametrize( + "identity", [None, {"account": "other", "workspace": "ws"}, {"account": "user", "workspace": "other"}] +) +def test_unknown_or_different_identity_never_reuses_state(tmp_path, session, identity): + store = resume.SessionStore(tmp_path / "sessions", {}) + assert store.save(session, {"account": "user", "workspace": "ws"}) + assert store.load(identity) is None + + +def test_expired_and_old_state_is_pruned(tmp_path, session, monkeypatch): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + now = time.time() + monkeypatch.setattr(resume.time, "time", lambda: now + resume.LIFETIME + 1) + assert store.load(identity) is None + assert list(store.directory.glob("*.json")) == [] + + +def test_directory_is_bounded_and_invalid_json_is_ignored(tmp_path, session): + identity = {"account": "user", "workspace": "ws"} + for i in range(resume._MAX_ENTRIES + 5): + store = resume.SessionStore(tmp_path / "sessions", {"path": str(i)}) + assert store.save(session, identity) + assert len(list(store.directory.glob("*.json"))) == resume._MAX_ENTRIES + store.path.write_text("{broken") + assert store.load(identity) is None + + +def test_state_never_follows_a_symlink(tmp_path, session): + actual = tmp_path / "actual" + actual.mkdir() + link = tmp_path / "link" + try: + link.symlink_to(actual, target_is_directory=True) + except OSError: + pytest.skip("symlink permission unavailable") + store = resume.SessionStore(link, {}) + assert not store.save(session, {"account": "user", "workspace": "ws"}) + assert list(actual.iterdir()) == [] + + +def test_unknown_identity_is_never_persisted(tmp_path, session): + store = resume.SessionStore(tmp_path / "sessions", {}) + assert not store.save(session, None) + assert not store.directory.exists() + + +def test_state_expiry_cannot_be_extended_by_a_future_server_expiry(tmp_path, session, monkeypatch): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save({**session, "expires_at": "2099-01-01T00:00:00Z"}, identity) + saved = json.loads(store.path.read_text()) + saved["created_at"] = time.time() - resume.LIFETIME - 1 + store.path.write_text(json.dumps(saved)) + assert store.load(identity) is None + + +@pytest.mark.parametrize("piece", [None, 0, True, 123, 128 * 1024**2]) +def test_invalid_piece_size_is_refused_before_sending(session, file, clock, monkeypatch, piece): + bucket = Bucket() + monkeypatch.setattr(requests, "put", bucket) + with pytest.raises(resume.TransferFailed): + send({**session, "chunk_size_bytes": piece}, file, clock) + assert bucket.calls == [] + + +def test_range_cannot_go_backwards(session, file, clock, monkeypatch): + responses = iter(["bytes=0-262143", "bytes=0-5"]) + monkeypatch.setattr( + requests, + "put", + lambda *a, **k: SimpleNamespace(status_code=308, headers={"Range": next(responses)}, close=lambda: None), + ) + with pytest.raises(resume.TransferFailed): + send(session, file, clock) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX permission bits") +def test_world_readable_state_is_not_reused(tmp_path, session): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + store.path.chmod(0o644) + assert store.load(identity) is None + + +def test_failed_atomic_save_keeps_the_previous_session(tmp_path, session, monkeypatch): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + + def fail(*a): + raise OSError("disk full") + + monkeypatch.setattr(resume.os, "replace", fail) + assert not store.save({**session, "upload_id": "other"}, identity) + assert store.load(identity) == session + assert not list(store.directory.glob(".upload-*")) From 244a1a1abede4610697e25d225f945345bef091e Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 11:25:22 -0500 Subject: [PATCH 02/12] test(models): invoke the model command in the resume regression --- tests/comfy_cli/command/test_assets_library.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index c900b8570..2adc16222 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -2422,10 +2422,11 @@ def test_killed_upload_resumes_the_same_session_on_rerun( monkeypatch.setattr("requests.put", bucket) cloud = _Cloud(_resume_routes()) monkeypatch.setattr("urllib.request.urlopen", cloud) - from comfy_cli.command import models + from comfy_cli.command.models import models result = CliRunner().invoke(models.app if model else assets_library.app, ["upload", *args, "--where", "cloud"]) assert result.exit_code != 0 + assert bucket.held == _MODEL_BYTES[:5] capsys.readouterr() bucket.puts.clear() env, rc, cloud, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=bucket) From 13f01f2bc4ea0efe597bdc9b616996624fc3e41d Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 11:43:26 -0500 Subject: [PATCH 03/12] fix(models): preserve saved uploads and report setup errors --- comfy_cli/command/assets_library.py | 12 +--- comfy_cli/command/model_upload_resume.py | 28 +++++++--- comfy_cli/error_codes.py | 11 ++-- ...oads-only-for-the-same-file-and-account.md | 6 +- .../comfy_cli/command/test_assets_library.py | 55 +++++++++++++++++-- .../command/test_model_upload_resume.py | 34 ++++++++++++ tests/comfy_cli/output/test_discovery.py | 12 ++++ 7 files changed, 129 insertions(+), 29 deletions(-) diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index e0aff125d..16c501f76 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -24,6 +24,7 @@ handle_cloud_http_error, http_request, ) +from comfy_cli.command.model_upload_resume import TlsSetupFailed as _TlsSetupFailed from comfy_cli.output.renderer import get_renderer app = typer.Typer(help="Browse your Comfy Cloud asset library (list, borrow, upload).") @@ -400,10 +401,6 @@ class _FileUnreadable(Exception): """The file could not be opened or read while it was being sent; ``args[0]`` is the ``OSError``.""" -class _TlsSetupFailed(Exception): - """This machine's TLS settings stopped the PUT before it connected; ``args[0]`` is the message to show.""" - - class _FileChanged(Exception): """A block of the file read for the PUT is not the block hashed before it; ``args[0]`` is the block's index.""" @@ -656,8 +653,6 @@ def _put(upload_url: str, headers: dict, path: Path, progress, digests) -> None: exists) and raises ``_TlsSetupFailed`` with its message; a retry would fail the same way too. """ - import urllib.parse - import requests from comfy_cli.builder_api import _CountingReader @@ -711,10 +706,7 @@ def read(self, size: int = -1) -> bytes: # TLS setup, e.g. REQUESTS_CA_BUNDLE naming a removed file. Its # text names a certificate path, but is never shown if it quotes # the signed URL. - text, query = str(e) or type(e).__name__, urllib.parse.urlsplit(upload_url).query - if upload_url in text or (query and query in text): - text = type(e).__name__ - raise _TlsSetupFailed(text) from None + raise _TlsSetupFailed(e, upload_url) from None if not (200 <= resp.status_code < 300 or resp.status_code == _ALREADY_WRITTEN): raise _PutFailed(resp.status_code) diff --git a/comfy_cli/command/model_upload_resume.py b/comfy_cli/command/model_upload_resume.py index eda13a0ba..bf811beeb 100644 --- a/comfy_cli/command/model_upload_resume.py +++ b/comfy_cli/command/model_upload_resume.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib import json import math import os @@ -12,6 +11,8 @@ import time from datetime import datetime from pathlib import Path +from urllib.parse import urlsplit +from uuid import uuid4 import requests @@ -30,6 +31,16 @@ class TransferFailed(Exception): """Only a status is safe to expose; never keep a transport exception.""" +class TlsSetupFailed(Exception): + """A permanent local TLS setup error, with session credentials removed.""" + + def __init__(self, error: OSError, url: str) -> None: + text, query = str(error) or type(error).__name__, urlsplit(url).query + if url in text or (query and query in text): + text = type(error).__name__ + super().__init__(text) + + class UploadPaused(Exception): """The argument is the last byte count acknowledged by storage.""" @@ -133,8 +144,9 @@ def save(self, start: dict, identity: dict | None) -> bool: temporary = None try: self._secure() - key = hashlib.sha256(json.dumps([self.binding, identity], sort_keys=True).encode()).hexdigest() - path = self.directory / f"{key}.json" + # Two commands may both start before either has saved a session. + # Each owns a separate record, including when their bindings match. + path = self.directory / f"{uuid4().hex}.json" entries = self._entries() for old in entries[: max(0, len(entries) - _MAX_ENTRIES + 1)]: old.unlink(missing_ok=True) @@ -167,8 +179,8 @@ def clear(self) -> None: def valid_start(start: dict) -> bool: - from urllib.parse import urlsplit - + if not isinstance(start.get("upload_url"), str): + return False try: url = urlsplit(start.get("upload_url", "")) return ( @@ -196,9 +208,11 @@ def _request(start: dict, data: bytes, content_range: str) -> tuple[int | None, timeout=(10, 60), allow_redirects=False, ) - except OSError: - # Requests exceptions may quote the credential, as may TLS setup errors. + except requests.RequestException: + # Transport exceptions may quote the session credential. return None, None + except OSError as error: + raise TlsSetupFailed(error, start["upload_url"]) from None try: return response.status_code, response.headers.get("Range") finally: diff --git a/comfy_cli/error_codes.py b/comfy_cli/error_codes.py index 162466b12..fef56c03b 100644 --- a/comfy_cli/error_codes.py +++ b/comfy_cli/error_codes.py @@ -498,7 +498,7 @@ class ErrorCode: "asset_upload_invalid", "`model upload` or `assets library upload` refused the file or `--type` before uploading anything: " "`details.reason` is unsupported_file_type (not .safetensors), bad_filename, file_not_found, " - "file_unreadable (it could not be opened or its size read), empty_file, file_too_large (over 2 GB), " + "file_unreadable (it could not be opened or its size read), empty_file, " "unsupported_model_type (`details.valid_types` lists the folders the workspace takes), " "bad_expected_sha256 (`--expected-sha256` is not 64 hex), file_changed (the file's sha256, " "`details.sha256`, is not `details.expected_sha256`: it changed since the dry run that was approved), " @@ -529,13 +529,15 @@ class ErrorCode: "Comfy Cloud refused a model upload; the message is the server's. `details.server_code` is its " "code, e.g. FEATURE_DISABLED (uploads are off for this workspace), MODEL_STORAGE_LIMIT_REACHED " "(the message names the cap), UNSUPPORTED_FILE_TYPE, UNSUPPORTED_MODEL_TYPE or FILE_TOO_LARGE. " - "Retrying the same upload gets the same answer.", + "Cloud decides the size limit. Retrying the same upload gets the same answer.", "act on the server's message; do not retry unchanged", ), ErrorCode( "asset_upload_failed", "A model upload did not end in a stored asset: sending the bytes to storage failed after a fresh " - "retry (`details.status`, null when no answer came), the local file could not be read while it " + "retry (`details.status`, null when no answer came), a resumable transfer paused after thirty minutes " + "without acknowledged progress (`details.reason` upload_paused; `details.bytes_held` counts the bytes " + "storage holds, and saved state permits a later run to continue), the local file could not be read while it " "was being sent (`details.reason` file_unreadable; not retried), this machine's TLS settings stopped " "the PUT before it connected (`details.reason` tls_setup, e.g. REQUESTS_CA_BUNDLE naming a missing " "file; not retried), a block of the file read for sending was not the block hashed before it " @@ -543,7 +545,8 @@ class ErrorCode: "none of the changed bytes were, and the upload was not completed, so no asset was added), or " "Comfy Cloud's storing task failed or was cancelled (the message carries the task's error, e.g. not a " "valid .safetensors file).", - "retry for a storage failure; for an unreadable file or a task failure fix the file the message names; " + "retry for a storage failure; for upload_paused run the same command again to continue a saved upload; " + "for an unreadable file or a task failure fix the file the message names; " "for tls_setup fix the certificate setting the message names; for file_changed wait until nothing " "writes the file, then run the dry run again", ), diff --git a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md index 3c9a84f6c..7af7d8f76 100644 --- a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md +++ b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md @@ -58,8 +58,10 @@ still have a bounded retry window. Reruns hash the whole file again. Storage receives only newly needed pieces, and no temporary copy of a multi-GB model is created. State is local to one machine; simultaneous invocations are not coordinated and may start separate -uploads. Losing saved state loses the ability to resume that session, but a -new upload still works. A task may continue after the command loses its sign-in. +uploads. Each new upload has a separate record, so completing one does not +delete another upload's saved state. Losing saved state loses the ability to +resume that session, but a new upload still works. A task may continue after +the command loses its sign-in. ## Validation diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 2adc16222..a25590c2e 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -1004,7 +1004,7 @@ def test_a_file_comfy_cloud_would_refuse_is_refused_before_any_request( path = tmp_path / name if size is not None: with path.open("wb") as f: - f.truncate(size) # sparse: no 2 GB written to disk + f.truncate(size) env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, [str(path), "--type", "loras"], {}, []) @@ -1013,10 +1013,19 @@ def test_a_file_comfy_cloud_would_refuse_is_refused_before_any_request( assert cloud.calls == [] and storage.puts == [] @pytest.mark.parametrize("size", [2 * 1024**3 + 1, 25 * 1024**3, 25 * 1024**3 + 1]) - def test_size_limit_is_left_to_the_server(self, tmp_path, size): + def test_size_limit_is_left_to_the_server(self, tmp_path, monkeypatch, size): path = tmp_path / "big.SAFETENSORS" - with path.open("wb") as f: - f.truncate(size) + path.write_bytes(b"x") + real_stat = Path.stat + + def stat_with_large_size(candidate, *args, **kwargs): + info = real_stat(candidate, *args, **kwargs) + if candidate == path: + return os.stat_result((*info[:6], size, *info[7:])) + return info + + # Extending a file can allocate its full size on Windows. + monkeypatch.setattr(Path, "stat", stat_with_large_size) assert assets_library._local_refusal(path) is None def test_unknown_type_is_refused_listing_the_valid_ones_before_the_upload( @@ -1747,8 +1756,9 @@ def __call__(self, url, data=None, headers=None, **kw): assert "Transfer-Encoding" not in storage.prepared assert storage.puts[0]["body"] == model_file.read_bytes() + @pytest.mark.parametrize("resumable", [False, True], ids=["single-put", "resumable"]) def test_a_missing_ca_bundle_ends_the_upload_with_a_tls_setup_error( - self, cloud_target, monkeypatch, model_file, tmp_path, no_wait + self, cloud_target, monkeypatch, model_file, tmp_path, no_wait, resumable ): # The real Requests adapter: with REQUESTS_CA_BUNDLE naming a removed # file, it raises a plain OSError while setting up TLS, before any @@ -1769,7 +1779,7 @@ def __call__(self, url, **kw): return real_put(url, **kw) monkeypatch.setenv("REQUESTS_CA_BUNDLE", str(tmp_path / "removed-ca-bundle.pem")) - cloud, storage = _Cloud(dict(_UPLOAD_ROUTES)), _RealPut([]) + cloud, storage = _Cloud(_resume_routes() if resumable else dict(_UPLOAD_ROUTES)), _RealPut([]) monkeypatch.setattr("urllib.request.urlopen", cloud) monkeypatch.setattr("requests.put", storage) result = CliRunner().invoke( @@ -2541,6 +2551,39 @@ def test_complete_retries_transient_errors(self, cloud_target, monkeypatch, caps class TestResumableRefusals: + @pytest.mark.parametrize("url", [1, True, {}, []]) + @pytest.mark.parametrize("resumable", [False, True]) + def test_invalid_start_url_returns_the_cloud_error( + self, cloud_target, monkeypatch, capsys, model_file, url, resumable + ): + routes = _resume_routes() + start = _resumable_started() if resumable else _started() + routes[("POST", "/api/assets/uploads")] = {**start, "upload_url": url} + env, rc, _, storage, everything = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, []) + assert rc == 1 and env["error"]["code"] == "cloud_http_error" + assert storage.puts == [] + assert "SECRETSIG" not in everything + + @pytest.mark.parametrize("quote", ["url", "query"]) + def test_tls_setup_error_never_displays_session_credentials( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, quote + ): + from urllib.parse import urlsplit + + attempts = [] + + def failed(url, **kw): + attempts.append(url) + raise OSError(url if quote == "url" else urlsplit(url).query) + + env, rc, cloud, _, everything = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], _resume_routes(), [], storage=failed + ) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert len(attempts) == 1 + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything + def test_server_size_refusal_prints_its_message_before_sending(self, cloud_target, monkeypatch, capsys, model_file): routes = _resume_routes() routes[("POST", "/api/assets/uploads")] = _cloud_error(400, "FILE_TOO_LARGE", "Model files can be up to 25 GB.") diff --git a/tests/comfy_cli/command/test_model_upload_resume.py b/tests/comfy_cli/command/test_model_upload_resume.py index bd55ea8b7..a4976cad4 100644 --- a/tests/comfy_cli/command/test_model_upload_resume.py +++ b/tests/comfy_cli/command/test_model_upload_resume.py @@ -320,3 +320,37 @@ def fail(*a): assert not store.save({**session, "upload_id": "other"}, identity) assert store.load(identity) == session assert not list(store.directory.glob(".upload-*")) + + +@pytest.mark.parametrize("url", [1, True, {}, [], b"https://bucket.example/upload"]) +def test_invalid_url_types_are_refused(session, url): + assert not resume.valid_start({**session, "upload_url": url}) + + +@pytest.mark.parametrize("url", [1, True, {}, []]) +def test_malformed_saved_url_is_ignored(tmp_path, session, url): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + saved = json.loads(store.path.read_text()) + saved["start"]["upload_url"] = url + store.path.write_text(json.dumps(saved)) + assert resume.SessionStore(store.directory, {}).load(identity) is None + + +def test_overlapping_uploads_keep_separate_sessions(tmp_path, session): + identity = {"account": "user", "workspace": "ws"} + first = resume.SessionStore(tmp_path / "sessions", {"path": "model", "hash": "hash"}) + second = resume.SessionStore(first.directory, first.binding) + assert first.load(identity) is None + assert second.load(identity) is None + other_session = {**session, "upload_id": "upload-2", "upload_url": "https://bucket.example/other"} + assert first.save(session, identity) + assert second.save(other_session, identity) + assert first.path != second.path + first.clear() + rerun = resume.SessionStore(first.directory, first.binding) + assert rerun.load(identity) == other_session + assert rerun.path == second.path + rerun.clear() + assert not list(first.directory.glob("*.json")) diff --git a/tests/comfy_cli/output/test_discovery.py b/tests/comfy_cli/output/test_discovery.py index 4e682b9c9..f84cb2e93 100644 --- a/tests/comfy_cli/output/test_discovery.py +++ b/tests/comfy_cli/output/test_discovery.py @@ -104,6 +104,18 @@ def test_discover_includes_error_codes_from_markdown(): assert "cql_no_graph" in codes # the loader's "no source available" survives +def test_model_upload_guidance_leaves_size_limits_to_cloud_and_explains_resume(): + from comfy_cli.discovery import load_error_codes + + codes = {row["code"]: row for row in load_error_codes()} + assert "file_too_large" not in codes["asset_upload_invalid"]["meaning"] + assert "FILE_TOO_LARGE" in codes["asset_upload_refused"]["meaning"] + paused = codes["asset_upload_failed"] + assert "upload_paused" in paused["meaning"] + assert "bytes_held" in paused["meaning"] + assert "same command" in paused["hint"] + + def test_discover_capabilities_flags(): envelope = _run_cli(["--json", "discover"]) caps = envelope["data"]["capabilities"] From 7bbd40cad675077cde1cdf63e9a1dc4deded486f Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 12:07:17 -0500 Subject: [PATCH 04/12] fix(models): keep the retired upload timeout code discoverable --- CHANGELOG.md | 3 +- comfy_cli/error_codes.py | 9 +++--- comfy_cli/schemas/error_codes.md | 5 +++- .../output/test_error_code_registry.py | 29 ++++++++++++++----- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2516354bc..1a79e8ce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,8 @@ history. upload for the same file, account, workspace and folder. Older servers keep the single-PUT upload. File approval, block checks and the final stored-hash check still apply. The final storage check no longer stops after fifteen - minutes. + minutes. The retired `asset_upload_timeout` error code stays discoverable + for compatibility with older versions. ### Added diff --git a/comfy_cli/error_codes.py b/comfy_cli/error_codes.py index fef56c03b..1599ed432 100644 --- a/comfy_cli/error_codes.py +++ b/comfy_cli/error_codes.py @@ -5,11 +5,11 @@ - ``tests/comfy_cli/output/test_error_code_registry.py``: every raised code is registered - every registered code is raised somewhere + every registered code is raised somewhere or explicitly retired That makes this module the canonical contract for agents. Agents fetch the -list via ``comfy discover`` and branch on the codes; if you rename, deprecate, -or remove one, you're breaking the contract and the tests fail before merge. +list via ``comfy discover`` and branch on the codes. Retired codes stay +registered for compatibility; never rename, repurpose or remove one. Codes are snake_case and match ``^[a-z][a-z0-9_]*$``. @@ -552,7 +552,8 @@ class ErrorCode: ), ErrorCode( "asset_upload_timeout", - "The model file was uploaded, but Comfy Cloud had not finished storing it when " + "Retired: current versions wait until storing ends. Older versions raised this when " + "the model file was uploaded, but Comfy Cloud had not finished storing it when " "`model upload` or `assets library upload` stopped waiting. `details.task_id` is the storing task, " "which goes on.", "check `comfy assets library ls --tags models` later instead of uploading again", diff --git a/comfy_cli/schemas/error_codes.md b/comfy_cli/schemas/error_codes.md index abe22e53c..ee3cedac7 100644 --- a/comfy_cli/schemas/error_codes.md +++ b/comfy_cli/schemas/error_codes.md @@ -27,7 +27,10 @@ comfy --json discover | jq '.data.error_codes[]' Three tests pin the contract: - Every `renderer.error(code="…")` site is registered. -- Every registered code is raised somewhere (no dead codes). +- Every registered code is raised somewhere or explicitly retired in the + registry test. Retired codes stay discoverable for compatibility and must + no longer be emitted. `asset_upload_timeout` is retired because model + uploads now wait until storing ends instead of stopping after fifteen minutes. - A code emitted only as a warning (an object in a `warnings` list of a successful result) is not registered. Registered codes are failure categories; a warning keeps its `code` string in `warnings[]`. Warning-only diff --git a/tests/comfy_cli/output/test_error_code_registry.py b/tests/comfy_cli/output/test_error_code_registry.py index ffbb58536..eee678c45 100644 --- a/tests/comfy_cli/output/test_error_code_registry.py +++ b/tests/comfy_cli/output/test_error_code_registry.py @@ -9,9 +9,9 @@ 1. Every code raised in source is in the registry. A typo or a fresh code added without registry update fails the test and surfaces the typo. -2. Every code in the registry is raised somewhere. A code that's - deprecated or removed but left in the registry fails the test, forcing - the dead entry to be deleted. +2. Every code in the registry is raised somewhere or explicitly retired. + Retired codes stay registered under ADR-AUTOMATION-0003's append-only + contract; an accidental unused entry still fails the test. Warning objects (``{"code": …, "message": …}`` placed in a ``warnings`` list) are collected separately from failures. Registered codes are failure @@ -256,6 +256,11 @@ def _collect_codes() -> tuple[dict[str, list[Path]], dict[str, list[Path]]]: ) +# The storage wait no longer has a total deadline. Keep the published code +# discoverable for clients that also handle output from older CLI versions. +RETIRED_ERROR_REGISTRATIONS = frozenset({"asset_upload_timeout"}) + + @pytest.fixture(scope="module") def collected_codes() -> tuple[dict[str, list[Path]], dict[str, list[Path]]]: return _collect_codes() @@ -343,18 +348,28 @@ def test_every_registered_code_is_raised(raised_codes, warned_codes): """If this fails: a code in the registry is no longer raised anywhere. A legacy warning registration (:data:`LEGACY_WARNING_REGISTRATIONS`) - counts as used while its warning is still emitted. + counts as used while its warning is still emitted. Explicitly retired + codes stay registered for compatibility but must no longer be emitted. - Fix: delete the dead entry from the registry, or wire the code up. + Fix: wire up an accidental unused code, or explicitly retire a published + code whose behavior was removed. Never delete a published registration. """ dead = [ c for c in error_codes.all_codes() - if c not in raised_codes and not (c in LEGACY_WARNING_REGISTRATIONS and c in warned_codes) + if c not in raised_codes + and c not in RETIRED_ERROR_REGISTRATIONS + and not (c in LEGACY_WARNING_REGISTRATIONS and c in warned_codes) ] assert not dead, ( - f"Registered but never raised:\n{dead}\nEither delete these from comfy_cli/error_codes.REGISTRY or use them." + f"Registered but never raised:\n{dead}\nUse these codes or explicitly retire their published registrations." + ) + invalid_retired = sorted( + c + for c in RETIRED_ERROR_REGISTRATIONS + if not error_codes.is_registered(c) or c in raised_codes or c in warned_codes ) + assert not invalid_retired, f"Retired codes must stay registered and never be emitted: {invalid_retired}" def test_every_build_code_has_a_first_call_site(raised_codes): From 68eb1e60471a360004e848a9392d55c98c81426e Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 14:12:19 -0500 Subject: [PATCH 05/12] fix(models): report whether paused uploads have saved sessions --- CHANGELOG.md | 8 ++-- README.md | 2 +- comfy_cli/command/assets_library.py | 19 ++++++-- comfy_cli/error_codes.py | 6 ++- .../comfy_cli/command/test_assets_library.py | 47 +++++++++++++++++-- .../command/test_model_upload_resume.py | 15 ++++++ tests/comfy_cli/output/test_discovery.py | 2 + 7 files changed, 85 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a79e8ce7..922966113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,9 +19,11 @@ history. - Both `comfy assets library upload` and `comfy model upload` request resumable model uploads up to 25 GB. Dropped connections retry from storage's held - bytes, and running the command again within seven days continues a saved - upload for the same file, account, workspace and folder. Older servers keep - the single-PUT upload. File approval, block checks and the final stored-hash + bytes. Running the command again continues a saved upload for the same file, + account, workspace and folder while its session remains valid. Sessions expire + no later than seven days after creation; the server can expire them earlier. + A paused upload reports whether its session was saved for a later run. Older + servers keep the single-PUT upload. File approval, block checks and the final stored-hash check still apply. The final storage check no longer stops after fifteen minutes. The retired `asset_upload_timeout` error code stays discoverable for compatibility with older versions. diff --git a/README.md b/README.md index 447f42fb2..daa58a3be 100644 --- a/README.md +++ b/README.md @@ -445,7 +445,7 @@ the bisect tool can help you pinpoint the custom node that causes the issue. `comfy model upload --type --where cloud ?[--dry-run] ?[--workspace ] ?[--expected-sha256 ] ?[--yes]` - Puts one local `.safetensors` file (up to 25 GB on servers with resumable uploads) into your Comfy Cloud model library, so a workflow run on Comfy Cloud can load it. `--type` is the model folder, e.g. `loras` or `checkpoints`; `comfy model list-folders --where cloud` lists them. - - Uploads use 64 MiB pieces when the server supports them. A dropped connection retries from the bytes storage holds. If the command stops, run it again with the same file, model folder, account and workspace within seven days to continue. The CLI hashes the file again before resuming. Older servers keep using one PUT and enforce their own size limit. Upload sessions are credentials, kept in a private `model-uploads` directory under the CLI config folder and removed after completion. + - Uploads use 64 MiB pieces when the server supports them. A dropped connection retries from the bytes storage holds. If the command stops, run it again with the same file, model folder, account and workspace to continue while its saved session remains valid. Sessions expire no later than seven days after creation; the server can expire them earlier. A paused upload tells you whether its session was saved. Without saved state, a rerun starts a new upload. The CLI hashes the file again before resuming. Older servers keep using one PUT and enforce their own size limit. Upload sessions are credentials, kept in a private `model-uploads` directory under the CLI config folder and removed after completion. - After 30 minutes without progress, the command exits with the held byte count. The final check on Comfy Cloud can take longer than fifteen minutes; the command waits until it ends. - Only the file you name is uploaded, and only the bytes hashed before the upload: each block is checked against its hash before it is sent. The upload succeeds only once Comfy Cloud's hash of the stored file matches your file's. - `--dry-run` uploads nothing. It reports the file's `sha256` and whether Comfy Cloud already has the file. diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index 16c501f76..4b9bfc33c 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -1138,6 +1138,7 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None saved = store.load(identity) for attempt in range(1, _UPLOAD_STARTS + 1): resumed = saved is not None + session_saved = resumed start = saved or _call( renderer, target, @@ -1153,7 +1154,8 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None resumable = start.get("upload_mode") == "resumable" if resumable and not resumed: identity = identity or _upload_identity(target, sign_in) - if not store.save(start, identity): + session_saved = store.save(start, identity) + if not session_saved: renderer.warn("This upload cannot be saved for a later run; connection retries still work.") try: with reporter.uploading(item, 1, 1) as progress: @@ -1184,11 +1186,20 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None except UploadPaused as e: from comfy_cli.output.progress import human_bytes + next_run = ( + "Run the same command again to continue while the saved session remains valid." + if session_saved + else "This session was not saved. Run the same command again to start a new upload." + ) renderer.error( code="asset_upload_failed", - message=f"Upload paused at {human_bytes(e.args[0])} of {human_bytes(size)}. " - "Run the same command again to continue.", - details={"reason": "upload_paused", "upload_id": upload_id, "bytes_held": e.args[0]}, + message=f"Upload paused at {human_bytes(e.args[0])} of {human_bytes(size)}. {next_run}", + details={ + "reason": "upload_paused", + "upload_id": upload_id, + "bytes_held": e.args[0], + "session_saved": session_saved, + }, ) raise typer.Exit(code=1) from None except _FileUnreadable as e: diff --git a/comfy_cli/error_codes.py b/comfy_cli/error_codes.py index 1599ed432..d633978cd 100644 --- a/comfy_cli/error_codes.py +++ b/comfy_cli/error_codes.py @@ -537,7 +537,8 @@ class ErrorCode: "A model upload did not end in a stored asset: sending the bytes to storage failed after a fresh " "retry (`details.status`, null when no answer came), a resumable transfer paused after thirty minutes " "without acknowledged progress (`details.reason` upload_paused; `details.bytes_held` counts the bytes " - "storage holds, and saved state permits a later run to continue), the local file could not be read while it " + "storage holds; `details.session_saved` says whether the session was saved for a later run), " + "the local file could not be read while it " "was being sent (`details.reason` file_unreadable; not retried), this machine's TLS settings stopped " "the PUT before it connected (`details.reason` tls_setup, e.g. REQUESTS_CA_BUNDLE naming a missing " "file; not retried), a block of the file read for sending was not the block hashed before it " @@ -545,7 +546,8 @@ class ErrorCode: "none of the changed bytes were, and the upload was not completed, so no asset was added), or " "Comfy Cloud's storing task failed or was cancelled (the message carries the task's error, e.g. not a " "valid .safetensors file).", - "retry for a storage failure; for upload_paused run the same command again to continue a saved upload; " + "retry for a storage failure; for upload_paused run the same command again to continue only when " + "session_saved is true and the session remains valid; otherwise start a new upload; " "for an unreadable file or a task failure fix the file the message names; " "for tls_setup fix the certificate setting the message names; for file_changed wait until nothing " "writes the file, then run the dry run again", diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index a25590c2e..d6ca2449b 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -2604,20 +2604,59 @@ def test_unknown_identity_does_not_save_a_session( assert list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) == [] + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) def test_paused_upload_redacts_the_credential_and_keeps_the_state( - self, cloud_target, monkeypatch, capsys, model_file, no_wait + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model ): import requests def lost(*a, **kw): raise requests.ConnectionError("SECRETSIG") - env, rc, _, _, everything = _upload( - monkeypatch, capsys, [str(model_file), "--type", "loras"], _resume_routes(), [], storage=lost - ) + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, _, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=lost) assert rc == 1 and env["error"]["details"]["reason"] == "upload_paused" + assert env["error"]["details"]["session_saved"] is True assert "SECRETSIG" not in everything assert "Run the same command again" in env["error"]["message"] from comfy_cli.config_manager import ConfigManager assert len(list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json"))) == 1 + + env, rc, cloud, _, _ = run(monkeypatch, capsys, args, _resume_routes(), [], storage=lost) + assert rc == 1 and env["error"]["details"]["session_saved"] is True + assert "/api/assets/uploads" not in cloud.paths() + + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + @pytest.mark.parametrize("unsaved", ["user", "workspaces/current", "write-failed"]) + def test_paused_upload_without_saved_state_explains_it_must_start_again( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model, unsaved + ): + import requests + + from comfy_cli.command.model_upload_resume import SessionStore + from comfy_cli.config_manager import ConfigManager + + def lost(*a, **kw): + raise requests.ConnectionError("SECRETSIG") + + def cannot_save(self): + raise OSError("permission denied") + + routes = _resume_routes() + if unsaved == "write-failed": + monkeypatch.setattr(SessionStore, "_secure", cannot_save) + else: + routes[("GET", "/api/" + unsaved)] = {} + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, routes, [], storage=lost) + assert rc == 1 and env["error"]["details"]["reason"] == "upload_paused" + assert env["error"]["details"]["session_saved"] is False + assert "not saved" in env["error"]["message"] + assert "start a new upload" in env["error"]["message"] + assert "to continue" not in env["error"]["message"] + assert "SECRETSIG" not in everything + assert _COMPLETE_1 not in cloud.paths() + assert list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) == [] diff --git a/tests/comfy_cli/command/test_model_upload_resume.py b/tests/comfy_cli/command/test_model_upload_resume.py index a4976cad4..73a694ec8 100644 --- a/tests/comfy_cli/command/test_model_upload_resume.py +++ b/tests/comfy_cli/command/test_model_upload_resume.py @@ -263,6 +263,21 @@ def test_state_never_follows_a_symlink(tmp_path, session): assert list(actual.iterdir()) == [] +def test_corrupt_newer_session_does_not_hide_valid_session(tmp_path, session): + identity = {"account": "user", "workspace": "ws"} + first = resume.SessionStore(tmp_path / "sessions", {"path": "model"}) + second = resume.SessionStore(first.directory, first.binding) + assert first.save(session, identity) + assert second.save(session, identity) + second.path.write_text("{broken") + now = time.time() + os.utime(first.path, (now - 10, now - 10)) + os.utime(second.path, (now, now)) + loaded = resume.SessionStore(first.directory, first.binding) + assert loaded.load(identity) == session + assert loaded.path == first.path + + def test_unknown_identity_is_never_persisted(tmp_path, session): store = resume.SessionStore(tmp_path / "sessions", {}) assert not store.save(session, None) diff --git a/tests/comfy_cli/output/test_discovery.py b/tests/comfy_cli/output/test_discovery.py index f84cb2e93..f7f4b3ae4 100644 --- a/tests/comfy_cli/output/test_discovery.py +++ b/tests/comfy_cli/output/test_discovery.py @@ -113,7 +113,9 @@ def test_model_upload_guidance_leaves_size_limits_to_cloud_and_explains_resume() paused = codes["asset_upload_failed"] assert "upload_paused" in paused["meaning"] assert "bytes_held" in paused["meaning"] + assert "session_saved" in paused["meaning"] assert "same command" in paused["hint"] + assert "start a new upload" in paused["hint"] def test_discover_capabilities_flags(): From 7e21efb46e6dcf69bbdab2bdd56976a184535558 Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 14:20:36 -0500 Subject: [PATCH 06/12] fix(models): recover refused uploads and report storage tasks --- CHANGELOG.md | 5 +- README.md | 2 +- comfy_cli/command/assets_library.py | 21 ++- comfy_cli/command/model_upload_resume.py | 11 +- comfy_cli/schemas/build_push_event.json | 9 +- ...oads-only-for-the-same-file-and-account.md | 9 +- docs/json-output.md | 14 +- .../comfy_cli/command/test_assets_library.py | 137 +++++++++++++++++- .../command/test_model_upload_resume.py | 9 +- 9 files changed, 198 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 922966113..0d60b896e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,10 @@ history. servers keep the single-PUT upload. File approval, block checks and the final stored-hash check still apply. The final storage check no longer stops after fifteen minutes. The retired `asset_upload_timeout` error code stays discoverable - for compatibility with older versions. + for compatibility with older versions. The task id is reported before the + storage wait. Refused resumable sessions are discarded and restarted once; + certificate failures stop immediately. Legacy single-PUT storage addresses + retain HTTP compatibility. ### Added diff --git a/README.md b/README.md index daa58a3be..1f680289c 100644 --- a/README.md +++ b/README.md @@ -446,7 +446,7 @@ the bisect tool can help you pinpoint the custom node that causes the issue. - Puts one local `.safetensors` file (up to 25 GB on servers with resumable uploads) into your Comfy Cloud model library, so a workflow run on Comfy Cloud can load it. `--type` is the model folder, e.g. `loras` or `checkpoints`; `comfy model list-folders --where cloud` lists them. - Uploads use 64 MiB pieces when the server supports them. A dropped connection retries from the bytes storage holds. If the command stops, run it again with the same file, model folder, account and workspace to continue while its saved session remains valid. Sessions expire no later than seven days after creation; the server can expire them earlier. A paused upload tells you whether its session was saved. Without saved state, a rerun starts a new upload. The CLI hashes the file again before resuming. Older servers keep using one PUT and enforce their own size limit. Upload sessions are credentials, kept in a private `model-uploads` directory under the CLI config folder and removed after completion. - - After 30 minutes without progress, the command exits with the held byte count. The final check on Comfy Cloud can take longer than fifteen minutes; the command waits until it ends. + - After 30 minutes without progress, the command exits with the held byte count. The final check on Comfy Cloud can take longer than fifteen minutes; the command waits until it ends. It reports the task id before waiting. In JSON mode, the `upload_storing` event goes to stderr; in streaming mode it goes to stdout. If the wait is interrupted, the id identifies the task at Cloud's `GET /api/tasks/` endpoint. - Only the file you name is uploaded, and only the bytes hashed before the upload: each block is checked against its hash before it is sent. The upload succeeds only once Comfy Cloud's hash of the stored file matches your file's. - `--dry-run` uploads nothing. It reports the file's `sha256` and whether Comfy Cloud already has the file. - The output's `loader_path` is the value a loader node takes for the model on Comfy Cloud, as Comfy Cloud's record of the model gives it. It is null for a dry run that would upload, or when Comfy Cloud gives none. diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index 4b9bfc33c..417396b21 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -1226,17 +1226,27 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None renderer.error( code="asset_upload_failed", message=f"could not set up a secure connection to storage ({e.args[0]})", - hint="point REQUESTS_CA_BUNDLE (or CURL_CA_BUNDLE) at a certificate bundle that exists, or unset it, " - "then upload again; nothing was added to the library", + hint="check the storage certificate and proxy, and point REQUESTS_CA_BUNDLE (or CURL_CA_BUNDLE) " + "at a trusted certificate bundle that exists, or unset it; then upload again; " + "nothing was added to the library", details={"reason": "tls_setup", "upload_id": upload_id, "attempts": attempt}, ) raise typer.Exit(code=1) from None except (_PutFailed, TransferFailed, SessionExpired) as e: expired = isinstance(e, SessionExpired) status = 404 if expired else e.args[0] - if expired: + refused = ( + resumable + and isinstance(e, TransferFailed) + and status is not None + and 400 <= status < 500 + and status != 429 + ) + if expired or refused: store.clear() - if (expired or (not resumable and (status is None or status >= 500))) and attempt < _UPLOAD_STARTS: + if ( + expired or refused or (not resumable and (status is None or status >= 500)) + ) and attempt < _UPLOAD_STARTS: renderer.warn("The upload to storage did not go through; starting a fresh upload.") continue answer = "no response" if status is None else f"HTTP {status}" @@ -1249,7 +1259,8 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None raise typer.Exit(code=1) from None if renderer.is_pretty(): - renderer.info("Uploaded; Comfy Cloud is checking and storing the file.") + renderer.info(f"Uploaded; Comfy Cloud is checking and storing the file. Task id: {task_id}") + renderer.progress_event("upload_storing", task_id=task_id) task = _wait_for_task(renderer, target, task_id, sign_in) result = task.get("result") if isinstance(task.get("result"), dict) else {} diff --git a/comfy_cli/command/model_upload_resume.py b/comfy_cli/command/model_upload_resume.py index bf811beeb..48f307881 100644 --- a/comfy_cli/command/model_upload_resume.py +++ b/comfy_cli/command/model_upload_resume.py @@ -186,10 +186,11 @@ def valid_start(start: dict) -> bool: return ( isinstance(start.get("upload_id"), str) and bool(start["upload_id"]) - and url.scheme == "https" - and bool(url.hostname) - and not url.username - and not url.password + and bool(start["upload_url"]) + and ( + start.get("upload_mode") is None + or (url.scheme == "https" and bool(url.hostname) and not url.username and not url.password) + ) and start.get("method") == "PUT" and isinstance(start.get("headers"), dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in start["headers"].items()) @@ -208,6 +209,8 @@ def _request(start: dict, data: bytes, content_range: str) -> tuple[int | None, timeout=(10, 60), allow_redirects=False, ) + except requests.exceptions.SSLError as error: + raise TlsSetupFailed(error, start["upload_url"]) from None except requests.RequestException: # Transport exceptions may quote the session credential. return None, None diff --git a/comfy_cli/schemas/build_push_event.json b/comfy_cli/schemas/build_push_event.json index 031341494..56bd85dd7 100644 --- a/comfy_cli/schemas/build_push_event.json +++ b/comfy_cli/schemas/build_push_event.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://comfy.org/schemas/build_push_event.json", "title": "comfy build push upload progress event", - "description": "One JSON object per line while `comfy build push` uploads model files and packaged custom nodes, or `comfy model upload` / `comfy assets library upload` uploads one model file. Under `--json-stream` the lines are on stdout, ahead of the final envelope (build_push.json schema in .data; model_upload.json for `model upload`, assets_library.json for `assets library upload`). Under `--json` they are on stderr, so stdout stays the single envelope. A push opens with one `upload_plan`; each file that transfers gets an `upload_progress` when it starts and about every two seconds after, then one `upload_complete`. A file the builder already held gets only an `upload_complete` with `deduplicated: true`. A file whose upload fails gets no `upload_complete`: the error envelope follows.", + "description": "One JSON object per line while `comfy build push` uploads model files and packaged custom nodes, or `comfy model upload` / `comfy assets library upload` uploads one model file. Under `--json-stream` the lines are on stdout, ahead of the final envelope (build_push.json schema in .data; model_upload.json for `model upload`, assets_library.json for `assets library upload`). Under `--json` they are on stderr, so stdout stays the single envelope. A push opens with one `upload_plan`; each file that transfers gets an `upload_progress` when it starts and about every two seconds after, then one `upload_complete`. A file the builder already held gets only an `upload_complete` with `deduplicated: true`. A file whose upload fails gets no `upload_complete`: the error envelope follows. Model uploads then emit one `upload_storing` with `task_id` before waiting for the storage task.", "type": "object", "required": ["schema", "type"], "additionalProperties": true, @@ -13,8 +13,9 @@ }, "type": { "type": "string", - "description": "Open-ended: this comfy-cli version emits upload_plan, upload_progress and upload_complete (the shapes below), and adding a type is an additive change that does NOT bump `event/1`. Ignore types you do not recognise rather than treat them as a validation failure." + "description": "Open-ended: this comfy-cli version emits upload_plan, upload_progress, upload_complete and upload_storing (the shapes below), and adding a type is an additive change that does NOT bump `event/1`. Ignore types you do not recognise rather than treat them as a validation failure." }, + "task_id": {"type": "string", "minLength": 1, "description": "upload_storing: the model storage task to look up if the wait is interrupted."}, "files": {"type": "integer", "minimum": 0, "description": "upload_plan: files this push still has to send."}, "already_held": {"type": "integer", "minimum": 0, "description": "upload_plan: local models and nodes that need no upload, because an earlier push stored them or a public source link stands in for the bytes."}, "file": {"type": "string", "description": "The model's filename, or `.zip` for a packaged custom node."}, @@ -32,6 +33,10 @@ "deduplicated": {"type": "boolean", "description": "upload_complete: true when the builder already held these bytes and nothing was sent."} }, "allOf": [ + { + "if": {"required": ["type"], "properties": {"type": {"const": "upload_storing"}}}, + "then": {"required": ["task_id"]} + }, { "if": {"required": ["type"], "properties": {"type": {"const": "upload_plan"}}}, "then": {"required": ["files", "bytes_total", "already_held"]} diff --git a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md index 7af7d8f76..2fe64d44c 100644 --- a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md +++ b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md @@ -40,10 +40,15 @@ Storage's status query decides where to resume. Each piece is at most 64 MiB. A resumed offset inside a hashing block still checks that whole block before sending any part of it. Only acknowledged bytes advance progress. Transport errors keep no exception text because it can contain the session credential. -Expired sessions restart; thirty minutes without acknowledged progress pauses. +Expired sessions and permanent storage refusals discard the saved record and +restart once; thirty minutes without acknowledged progress pauses. Certificate +verification failures stop immediately with credential-safe TLS guidance. Completion deletes the session after a valid task response, and the command waits for the task without a total deadline. Repeated unanswered task polls -still have a bounded retry window. +still have a bounded retry window. The task id is emitted before waiting, so +a caller can look it up if the command stops. HTTPS with a host and no userinfo +is required for resumable and saved sessions; legacy single-PUT addresses retain +the existing validation for compatibility with storage emulators. ## Alternatives considered diff --git a/docs/json-output.md b/docs/json-output.md index 0c9c8ddc2..1ae6863bb 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -487,7 +487,7 @@ time. If the callback never arrives, the terminal envelope is an {"schema": "event/1", "type": "login_url", "url": "https://api.comfy.org/oauth/authorize?...", "timeout_s": 300} ``` -### `upload_plan`, `upload_progress`, `upload_complete` +### `upload_plan`, `upload_progress`, `upload_complete`, `upload_storing` Emitted by `comfy build push` while it uploads model files and packaged custom nodes, and by `comfy model upload` and `comfy assets library upload` while they @@ -513,6 +513,18 @@ when there is nothing to upload. Each file that transfers gets an {"schema": "event/1", "type": "upload_complete", "file": "model.safetensors", "kind": "model", "index": 1, "of": 3, "bytes_total": 23761671782, "seconds": 498.2, "bytes_per_second": 47695045, "deduplicated": false, "overall_bytes_done": 23761671782, "overall_bytes_total": 56908316672} ``` +After transferring the bytes, `comfy model upload` and `comfy assets library +upload` emit `upload_storing` as soon as Cloud returns a storage task id. It +arrives before task polling and follows the same stdout/stderr rules above. +If the wait is interrupted, look up that id at Cloud's +`GET /api/tasks/` endpoint. +The final success envelope still requires a completed task and a matching +stored hash. + +```json +{"schema": "event/1", "type": "upload_storing", "task_id": "task-1"} +``` + `bytes_per_second` on a progress event is measured over the last ten seconds and is sampled on a timer, not on bytes moving, so a stalled connection keeps reporting and its rate falls to `0` with `eta_seconds: null`. That is how to diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index d6ca2449b..0e50a63c2 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -2406,6 +2406,8 @@ def __call__(self, url, *, data, headers, **kw): self.held = data[:5] self.crash = False raise KeyboardInterrupt() + if self.status is not None: + return SimpleNamespace(status_code=self.status, headers={}, close=lambda: None) if not headers["Content-Range"].startswith("bytes */"): offset = int(headers["Content-Range"].split()[1].split("-")[0]) assert offset == len(self.held) @@ -2486,14 +2488,72 @@ def put(url, **kw): ) assert rc == 0 and cloud.paths("POST").count("/api/assets/uploads") == 2 + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + @pytest.mark.parametrize("status", [400, 403, 422]) + @pytest.mark.parametrize("saved", [False, True], ids=["fresh", "saved"]) + def test_refused_session_is_discarded_and_restarted_once( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model, status, saved + ): + from comfy_cli.config_manager import ConfigManager + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + directory = Path(ConfigManager().get_config_path()) / "model-uploads" + if saved: + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + env, rc, *_ = run(monkeypatch, capsys, args, routes, [], storage=_ResumableBucket()) + assert rc == 1 and len(list(directory.glob("*.json"))) == 1 + bucket = _ResumableBucket() + refused = [] + + def put(url, **kw): + if url.endswith("1"): + refused.append(kw["headers"]["Content-Range"]) + return SimpleNamespace(status_code=status, headers={}, close=lambda: None) + return bucket(url, **kw) + + routes = _resume_routes() + if saved: + routes[("POST", "/api/assets/uploads")] = _resumable_started(2) + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, routes, [], storage=put) + assert rc == 0 and env["data"]["uploaded"] is True + assert cloud.paths("POST").count("/api/assets/uploads") == (1 if saved else 2) + assert refused == ["bytes */16" if saved else "bytes 0-15/16"] + assert bucket.held == _MODEL_BYTES + assert not list(directory.glob("*.json")) + assert "SECRETSIG" not in everything + + def test_repeated_storage_refusal_is_bounded_and_not_saved_for_the_next_run( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + from comfy_cli.config_manager import ConfigManager + + def refused(*args, **kw): + return SimpleNamespace(status_code=403, headers={}, close=lambda: None) + + for _ in range(2): + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], _resume_routes(), [], storage=refused + ) + assert rc == 1 and env["error"]["details"]["status"] == 403 + assert cloud.paths("POST").count("/api/assets/uploads") == 2 + assert not list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) + @pytest.mark.parametrize("change", ["workspace", "account", "same-size-bytes", "folder", "mtime", "size", "server"]) def test_rerun_with_a_changed_file_or_identity_starts_fresh( self, cloud_target, monkeypatch, capsys, model_file, no_wait, change ): - bucket = _ResumableBucket(status=403) + from comfy_cli.config_manager import ConfigManager + + bucket = _ResumableBucket() args = [str(model_file), "--type", "loras"] - env, rc, *_ = _upload(monkeypatch, capsys, args, _resume_routes(), [], storage=bucket) + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + env, rc, *_ = _upload(monkeypatch, capsys, args, routes, [], storage=bucket) assert rc == 1 + assert len(list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json"))) == 1 + bucket.status = 403 routes = _resume_routes() if change == "workspace": routes[("GET", "/api/workspaces/current")] = {"id": "ws-2"} @@ -2551,6 +2611,48 @@ def test_complete_retries_transient_errors(self, cloud_target, monkeypatch, caps class TestResumableRefusals: + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + def test_legacy_single_put_keeps_http_storage_compatibility( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model + ): + routes = dict(_UPLOAD_ROUTES) + address = "http://127.0.0.1:9023/bucket/model?signature=SECRETSIG" + routes[("POST", "/api/assets/uploads")] = {**_started(), "upload_url": address} + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, _, storage, everything = run(monkeypatch, capsys, args, routes, [200]) + assert rc == 0 and env["data"]["uploaded"] is True + assert storage.puts[0]["url"] == address + assert storage.puts[0]["body"] == _MODEL_BYTES + assert "SECRETSIG" not in everything + + @pytest.mark.parametrize("quote", ["url", "query"]) + @pytest.mark.parametrize("saved", [False, True], ids=["piece", "status-query"]) + def test_certificate_failure_stops_at_once_without_session_credentials( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, quote, saved + ): + import requests + from urllib.parse import urlsplit + + args = [str(model_file), "--type", "loras"] + if saved: + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + env, rc, *_ = _upload(monkeypatch, capsys, args, routes, [], storage=_ResumableBucket()) + assert rc == 1 + attempts = [] + + def failed(url, **kw): + attempts.append(kw["headers"]["Content-Range"]) + raise requests.exceptions.SSLError(url if quote == "url" else urlsplit(url).query) + + env, rc, cloud, _, everything = _upload(monkeypatch, capsys, args, _resume_routes(), [], storage=failed) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert attempts == ["bytes */16" if saved else "bytes 0-15/16"] + assert no_wait[0] == 0 + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything + @pytest.mark.parametrize("url", [1, True, {}, []]) @pytest.mark.parametrize("resumable", [False, True]) def test_invalid_start_url_returns_the_cloud_error( @@ -2660,3 +2762,34 @@ def cannot_save(self): assert "SECRETSIG" not in everything assert _COMPLETE_1 not in cloud.paths() assert list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) == [] + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("mode", [OutputMode.JSON, OutputMode.NDJSON]) +def test_storing_task_is_reported_before_waiting(cloud_target, monkeypatch, model_file, model, mode): + import jsonschema + + from comfy_cli.command.models import models + + monkeypatch.setattr("urllib.request.urlopen", _Cloud(_resume_routes())) + monkeypatch.setattr("requests.put", _ResumableBucket()) + + def interrupted(*args, **kw): + raise KeyboardInterrupt() + + monkeypatch.setattr(assets_library, "_wait_for_task", interrupted) + set_renderer(Renderer(mode=mode)) + result = CliRunner().invoke( + models.app if model else assets_library.app, + ["upload", str(model_file), "--type", "loras", "--where", "cloud", *(["--yes"] if model else [])], + ) + assert result.exit_code != 0 + stream = result.stdout if mode is OutputMode.NDJSON else result.stderr + events = [json.loads(line) for line in stream.splitlines() if line.startswith("{")] + storing = [event for event in events if event["type"] == "upload_storing"] + assert storing == [{"schema": "event/1", "type": "upload_storing", "task_id": "task-1"}] + schema = json.loads((Path(assets_library.__file__).parents[1] / "schemas" / "build_push_event.json").read_text()) + jsonschema.validate(storing[0], schema) + if mode is OutputMode.JSON: + assert result.stdout == "" + assert "SECRETSIG" not in result.output diff --git a/tests/comfy_cli/command/test_model_upload_resume.py b/tests/comfy_cli/command/test_model_upload_resume.py index 73a694ec8..82cd36021 100644 --- a/tests/comfy_cli/command/test_model_upload_resume.py +++ b/tests/comfy_cli/command/test_model_upload_resume.py @@ -342,7 +342,7 @@ def test_invalid_url_types_are_refused(session, url): assert not resume.valid_start({**session, "upload_url": url}) -@pytest.mark.parametrize("url", [1, True, {}, []]) +@pytest.mark.parametrize("url", [1, True, {}, [], "http://bucket.example/upload"]) def test_malformed_saved_url_is_ignored(tmp_path, session, url): store = resume.SessionStore(tmp_path / "sessions", {}) identity = {"account": "user", "workspace": "ws"} @@ -369,3 +369,10 @@ def test_overlapping_uploads_keep_separate_sessions(tmp_path, session): assert rerun.path == second.path rerun.clear() assert not list(first.directory.glob("*.json")) + + +@pytest.mark.parametrize( + "url", ["http://bucket.example/upload", "https:///missing-host", "https://user:pass@bucket.example/upload"] +) +def test_resumable_addresses_still_require_https_without_userinfo(session, url): + assert not resume.valid_start({**session, "upload_url": url}) From 07452814b7512017c0b336259f6633825bd829fb Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 14:21:59 -0500 Subject: [PATCH 07/12] style(tests): sort certificate regression imports --- tests/comfy_cli/command/test_assets_library.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 0e50a63c2..84684349b 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -2631,9 +2631,10 @@ def test_legacy_single_put_keeps_http_storage_compatibility( def test_certificate_failure_stops_at_once_without_session_credentials( self, cloud_target, monkeypatch, capsys, model_file, no_wait, quote, saved ): - import requests from urllib.parse import urlsplit + import requests + args = [str(model_file), "--type", "loras"] if saved: routes = _resume_routes() From f804f1ab46a13d734dfbae63a944dd5a0ef62255 Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 14:33:49 -0500 Subject: [PATCH 08/12] fix(models): preserve resume through transient storage failures --- CHANGELOG.md | 5 +- comfy_cli/command/assets_library.py | 15 +-- comfy_cli/command/build_upload_progress.py | 7 + comfy_cli/command/model_upload_resume.py | 42 ++++-- ...oads-only-for-the-same-file-and-account.md | 4 + docs/json-output.md | 3 +- .../comfy_cli/command/test_assets_library.py | 122 +++++++++++++++++- .../command/test_model_upload_resume.py | 2 +- 8 files changed, 177 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d60b896e..a99db7b01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,10 @@ history. minutes. The retired `asset_upload_timeout` error code stays discoverable for compatibility with older versions. The task id is reported before the storage wait. Refused resumable sessions are discarded and restarted once; - certificate failures stop immediately. Legacy single-PUT storage addresses + certificate failures stop immediately without transport exception text. + HTTP timeouts and transient TLS interruptions query the held bytes and retry. + A closed progress pipe does not stop the final stored-hash check. + Legacy single-PUT storage addresses retain HTTP compatibility. ### Added diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index 417396b21..614673712 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -650,7 +650,7 @@ def _put(upload_url: str, headers: dict, path: Path, progress, digests) -> None: or read raises ``_FileUnreadable`` instead: a fresh upload would fail the same way, so it is not a transport failure to retry. Any other ``OSError`` comes from setting up TLS (a CA bundle path that no longer - exists) and raises ``_TlsSetupFailed`` with its message; a retry would + exists) and raises ``_TlsSetupFailed`` with fixed TLS guidance; a retry would fail the same way too. """ import requests @@ -703,10 +703,9 @@ def read(self, size: int = -1) -> bytes: if isinstance(e, requests.RequestException): raise _PutFailed(None) from None # Neither the file nor the transport (Requests wraps those): the - # TLS setup, e.g. REQUESTS_CA_BUNDLE naming a removed file. Its - # text names a certificate path, but is never shown if it quotes - # the signed URL. - raise _TlsSetupFailed(e, upload_url) from None + # TLS setup, e.g. REQUESTS_CA_BUNDLE naming a removed file. Never + # retain its text: Requests may normalize a credential in the URL. + raise _TlsSetupFailed() from None if not (200 <= resp.status_code < 300 or resp.status_code == _ALREADY_WRITTEN): raise _PutFailed(resp.status_code) @@ -1240,7 +1239,7 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None and isinstance(e, TransferFailed) and status is not None and 400 <= status < 500 - and status != 429 + and status not in (408, 429) ) if expired or refused: store.clear() @@ -1258,9 +1257,7 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None ) raise typer.Exit(code=1) from None - if renderer.is_pretty(): - renderer.info(f"Uploaded; Comfy Cloud is checking and storing the file. Task id: {task_id}") - renderer.progress_event("upload_storing", task_id=task_id) + reporter.storing(task_id) task = _wait_for_task(renderer, target, task_id, sign_in) result = task.get("result") if isinstance(task.get("result"), dict) else {} diff --git a/comfy_cli/command/build_upload_progress.py b/comfy_cli/command/build_upload_progress.py index b00ab4ca8..c78689f61 100644 --- a/comfy_cli/command/build_upload_progress.py +++ b/comfy_cli/command/build_upload_progress.py @@ -44,6 +44,7 @@ EVENT_PLAN: Final = "upload_plan" EVENT_PROGRESS: Final = "upload_progress" EVENT_COMPLETE: Final = "upload_complete" +EVENT_STORING: Final = "upload_storing" class Clock(Protocol): @@ -152,6 +153,12 @@ def plan(self, uploads: tuple[UploadItem, ...] | list[UploadItem], *, already_he else: self._say(plan_line(len(uploads), self._overall_total, already_held)) + def storing(self, task_id: str) -> None: + """Report the model's storage task without letting a closed pipe stop verification.""" + if self._renderer.is_pretty(): + self._say(f"Uploaded; Comfy Cloud is checking and storing the file. Task id: {task_id}") + self._event(EVENT_STORING, task_id=task_id) + # ----- one file ----- @contextmanager diff --git a/comfy_cli/command/model_upload_resume.py b/comfy_cli/command/model_upload_resume.py index 48f307881..e1a059bf3 100644 --- a/comfy_cli/command/model_upload_resume.py +++ b/comfy_cli/command/model_upload_resume.py @@ -6,6 +6,7 @@ import math import os import re +import ssl import stat import tempfile import time @@ -32,13 +33,10 @@ class TransferFailed(Exception): class TlsSetupFailed(Exception): - """A permanent local TLS setup error, with session credentials removed.""" + """A permanent TLS failure; never retain transport exception text.""" - def __init__(self, error: OSError, url: str) -> None: - text, query = str(error) or type(error).__name__, urlsplit(url).query - if url in text or (query and query in text): - text = type(error).__name__ - super().__init__(text) + def __init__(self) -> None: + super().__init__("certificate verification or CA certificate bundle setup failed") class UploadPaused(Exception): @@ -200,6 +198,28 @@ def valid_start(start: dict) -> bool: return False +def _certificate_failure(error: BaseException) -> bool: + """Recognize native and wrapped certificate errors without reporting their text.""" + pending, seen = [error], set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, ssl.SSLCertVerificationError): + return True + # Some adapters flatten the native error into a message. + text = str(current).lower() + if "certificate_verify_failed" in text or "certificate verify failed" in text: + return True + pending.extend( + value + for value in (*current.args, getattr(current, "reason", None), current.__cause__, current.__context__) + if isinstance(value, BaseException) + ) + return False + + def _request(start: dict, data: bytes, content_range: str) -> tuple[int | None, str | None]: try: response = requests.put( @@ -210,12 +230,14 @@ def _request(start: dict, data: bytes, content_range: str) -> tuple[int | None, allow_redirects=False, ) except requests.exceptions.SSLError as error: - raise TlsSetupFailed(error, start["upload_url"]) from None + if _certificate_failure(error): + raise TlsSetupFailed() from None + return None, None except requests.RequestException: # Transport exceptions may quote the session credential. return None, None - except OSError as error: - raise TlsSetupFailed(error, start["upload_url"]) from None + except OSError: + raise TlsSetupFailed() from None try: return response.status_code, response.headers.get("Range") finally: @@ -262,7 +284,7 @@ def transfer( if query: query = False continue - elif status is not None and status != 429 and status < 500: + elif status is not None and status not in (408, 429) and status < 500: raise TransferFailed(status) sleep(min(delay, max(0, STALL_SECONDS - (clock() - last_progress)))) delay, query = min(delay * 2, 60), True diff --git a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md index 2fe64d44c..53094139e 100644 --- a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md +++ b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md @@ -43,12 +43,16 @@ errors keep no exception text because it can contain the session credential. Expired sessions and permanent storage refusals discard the saved record and restart once; thirty minutes without acknowledged progress pauses. Certificate verification failures stop immediately with credential-safe TLS guidance. +HTTP 408 and transient TLS failures query held bytes and retry within the same +session. TLS setup diagnostics use fixed text, including for normalized URLs. Completion deletes the session after a valid task response, and the command waits for the task without a total deadline. Repeated unanswered task polls still have a bounded retry window. The task id is emitted before waiting, so a caller can look it up if the command stops. HTTPS with a host and no userinfo is required for resumable and saved sessions; legacy single-PUT addresses retain the existing validation for compatibility with storage emulators. +Progress output failures mute reporting without stopping task polling or the +stored-hash check. ## Alternatives considered diff --git a/docs/json-output.md b/docs/json-output.md index 1ae6863bb..3021c2ed0 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -519,7 +519,8 @@ arrives before task polling and follows the same stdout/stderr rules above. If the wait is interrupted, look up that id at Cloud's `GET /api/tasks/` endpoint. The final success envelope still requires a completed task and a matching -stored hash. +stored hash. A closed progress pipe mutes progress reporting; task polling and +the stored-hash check still run. ```json {"schema": "event/1", "type": "upload_storing", "task_id": "task-1"} diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 84684349b..73957fbaa 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -2645,7 +2645,8 @@ def test_certificate_failure_stops_at_once_without_session_credentials( def failed(url, **kw): attempts.append(kw["headers"]["Content-Range"]) - raise requests.exceptions.SSLError(url if quote == "url" else urlsplit(url).query) + address = url if quote == "url" else urlsplit(url).query + raise requests.exceptions.SSLError(f"[SSL: CERTIFICATE_VERIFY_FAILED] {address}") env, rc, cloud, _, everything = _upload(monkeypatch, capsys, args, _resume_routes(), [], storage=failed) assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" @@ -2794,3 +2795,122 @@ def interrupted(*args, **kw): if mode is OutputMode.JSON: assert result.stdout == "" assert "SECRETSIG" not in result.output + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("mismatch", [False, True], ids=["verified", "wrong-hash"]) +@pytest.mark.parametrize("mode", [OutputMode.JSON, OutputMode.NDJSON, OutputMode.PRETTY]) +def test_closed_progress_pipe_still_polls_and_verifies(cloud_target, monkeypatch, model_file, model, mismatch, mode): + from comfy_cli.command.models import models + + routes = _resume_routes() + if mismatch: + routes[("GET", "/api/tasks/task-1")] = {**_DONE, "result": {**_DONE["result"], "hash": "b" * 64}} + cloud = _Cloud(routes) + monkeypatch.setattr("urllib.request.urlopen", cloud) + monkeypatch.setattr("requests.put", _ResumableBucket()) + refused = [] + original_event = Renderer.progress_event + original_info = Renderer.info + + def event(self, type, **fields): + if type != "upload_plan": + refused.append(type) + raise BrokenPipeError() + return original_event(self, type, **fields) + + def info(self, message, **kw): + if "Task id:" in message: + refused.append("storing") + raise BrokenPipeError() + return original_info(self, message, **kw) + + monkeypatch.setattr(Renderer, "progress_event", event) + monkeypatch.setattr(Renderer, "info", info) + set_renderer(Renderer(mode=mode)) + result = CliRunner().invoke( + models.app if model else assets_library.app, + ["upload", str(model_file), "--type", "loras", "--where", "cloud", *(["--yes"] if model else [])], + ) + assert "/api/tasks/task-1" in cloud.paths(), repr(result.exception) + assert result.exit_code == (1 if mismatch else 0), repr(result.exception) + assert refused + if mode is not OutputMode.PRETTY: + env = json.loads(result.stdout.strip().splitlines()[-1]) + assert env["ok"] is (not mismatch) + if mismatch: + assert env["error"]["code"] == "asset_upload_unverified" + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("failure", ["tls-eof", "408"]) +@pytest.mark.parametrize("saved", [False, True], ids=["piece", "status-query"]) +def test_transient_storage_failure_keeps_session_and_queries_offset( + cloud_target, monkeypatch, capsys, model_file, no_wait, model, failure, saved +): + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, SSLError + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + bucket = _ResumableBucket() + if saved: + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + env, rc, *_ = run(monkeypatch, capsys, args, routes, [], storage=bucket) + assert rc == 1 + calls = [] + + def put(url, **kw): + calls.append((url, kw["headers"]["Content-Range"])) + if len(calls) == 1: + if failure == "tls-eof": + raise requests.exceptions.SSLError(MaxRetryError(None, url, SSLError(ssl.SSLEOFError("TLS EOF")))) + return SimpleNamespace(status_code=408, headers={}, close=lambda: None) + if len(calls) == 2: + assert no_wait[0] == 1 + return bucket(url, **kw) + + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=put) + assert rc == 0 and env["ok"] is True + assert calls[1][1] == "bytes */16" + assert len({url for url, _ in calls}) == 1 + assert cloud.paths("POST").count("/api/assets/uploads") == (0 if saved else 1) + assert bucket.held == _MODEL_BYTES + assert "SECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("failure", ["certificate", "setup"]) +def test_normalized_tls_error_never_exposes_session_credentials( + cloud_target, monkeypatch, capsys, model_file, no_wait, model, failure +): + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, SSLError + + routes = _resume_routes() + start = {**_resumable_started(), "upload_url": "https://bucket.example/?upload_id=%53ECRETSIG"} + routes[("POST", "/api/assets/uploads")] = start + attempts = [] + + def put(url, **kw): + prepared = requests.Request("PUT", url).prepare().url + attempts.append(prepared) + assert "upload_id=SECRETSIG" in prepared + if failure == "certificate": + reason = ssl.SSLCertVerificationError("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed") + raise requests.exceptions.SSLError(MaxRetryError(None, prepared, SSLError(reason))) + raise OSError(f"TLS setup failed for {prepared}") + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, routes, [], storage=put) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert len(attempts) == 1 and no_wait[0] == 0 + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything + assert "%53ECRETSIG" not in everything diff --git a/tests/comfy_cli/command/test_model_upload_resume.py b/tests/comfy_cli/command/test_model_upload_resume.py index 82cd36021..355b9e0eb 100644 --- a/tests/comfy_cli/command/test_model_upload_resume.py +++ b/tests/comfy_cli/command/test_model_upload_resume.py @@ -112,7 +112,7 @@ def test_last_piece_without_an_answer_goes_to_complete(session, file, clock, mon assert len(bucket.calls) == 1 -@pytest.mark.parametrize("status", [429, 500, 503, None]) +@pytest.mark.parametrize("status", [408, 429, 500, 503, None]) def test_transient_errors_back_off_and_query(session, file, clock, monkeypatch, status): bucket = Bucket() calls = [] From 87e4de02124d6111e8529f741393bcc374a965e0 Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 14:51:11 -0500 Subject: [PATCH 09/12] fix(models): finish verification after pretty output fails --- CHANGELOG.md | 6 +- comfy_cli/command/assets_library.py | 6 +- comfy_cli/command/build_upload_progress.py | 22 ++-- comfy_cli/command/model_upload_resume.py | 24 +++- ...oads-only-for-the-same-file-and-account.md | 13 ++- .../comfy_cli/command/test_assets_library.py | 107 ++++++++++++++++++ .../command/test_model_upload_resume.py | 23 ++++ 7 files changed, 180 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a99db7b01..16b8aa805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,9 +28,11 @@ history. minutes. The retired `asset_upload_timeout` error code stays discoverable for compatibility with older versions. The task id is reported before the storage wait. Refused resumable sessions are discarded and restarted once; - certificate failures stop immediately without transport exception text. + certificate verification and invalid CA-bundle failures stop immediately + without transport exception text. HTTP timeouts and transient TLS interruptions query the held bytes and retry. - A closed progress pipe does not stop the final stored-hash check. + A closed progress pipe or stream does not stop task polling or the final + stored-hash check, including when Rich converts the pipe failure to an exit. Legacy single-PUT storage addresses retain HTTP compatibility. diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index 614673712..440ca5da4 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -774,7 +774,7 @@ def _complete_upload(renderer, target, sign_in, upload_id: str, body: dict, *, r return parsed -def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: +def _wait_for_task(renderer, target, task_id: str, sign_in=None, *, reporter: UploadProgressReporter) -> dict: """Read the task until it ends, riding out a brief 5xx, 429 or dropped connection.""" import http.client import urllib.error @@ -813,7 +813,7 @@ def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: if status in _TASK_DONE: return task if renderer.is_pretty() and _clock() - last_notice >= 30: - renderer.info(f"Comfy Cloud is still checking and storing the file ({int(_clock() - started)}s so far).") + reporter.waiting(int(_clock() - started)) last_notice = _clock() _sleep(pause) @@ -1258,7 +1258,7 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None raise typer.Exit(code=1) from None reporter.storing(task_id) - task = _wait_for_task(renderer, target, task_id, sign_in) + task = _wait_for_task(renderer, target, task_id, sign_in, reporter=reporter) result = task.get("result") if isinstance(task.get("result"), dict) else {} if task["status"] != "completed" or result.get("success") is False: diff --git a/comfy_cli/command/build_upload_progress.py b/comfy_cli/command/build_upload_progress.py index c78689f61..9f5f66f84 100644 --- a/comfy_cli/command/build_upload_progress.py +++ b/comfy_cli/command/build_upload_progress.py @@ -159,6 +159,11 @@ def storing(self, task_id: str) -> None: self._say(f"Uploaded; Comfy Cloud is checking and storing the file. Task id: {task_id}") self._event(EVENT_STORING, task_id=task_id) + def waiting(self, seconds: int) -> None: + """Keep storage polling notices inside the same muted-output boundary.""" + if self._renderer.is_pretty(): + self._say(f"Comfy Cloud is still checking and storing the file ({seconds}s so far).") + # ----- one file ----- @contextmanager @@ -283,15 +288,18 @@ def _event(self, type: str, **fields: Any) -> None: return try: self._renderer.progress_event(type, **fields) - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True def _say(self, message: str) -> None: if self._muted: return + # Rich converts a broken pipe to SystemExit and a closed stream to + # ValueError. Suppress those only at this progress-output boundary; + # task polling and byte verification must still finish. try: self._renderer.info(message) - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True def _open_live(self, transfer: _Transfer) -> None: @@ -323,18 +331,18 @@ def _open_live(self, transfer: _Transfer) -> None: rate="", left="", ) - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True try: live.stop() - except OSError: + except (OSError, ValueError, SystemExit): pass return self._live = live self._live_task = task def _update_live(self, numbers: dict[str, Any]) -> None: - if self._live is None: + if self._live is None or self._muted: return rate = numbers["bytes_per_second"] left = numbers["overall_eta_seconds"] @@ -345,7 +353,7 @@ def _update_live(self, numbers: dict[str, Any]) -> None: rate="" if rate is None else f"{human_bytes(rate)}/s", left="" if left is None else f"{human_seconds(left)} left", ) - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True def _close_live(self) -> None: @@ -354,7 +362,7 @@ def _close_live(self) -> None: return try: live.stop() - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True diff --git a/comfy_cli/command/model_upload_resume.py b/comfy_cli/command/model_upload_resume.py index e1a059bf3..02996b6cc 100644 --- a/comfy_cli/command/model_upload_resume.py +++ b/comfy_cli/command/model_upload_resume.py @@ -198,19 +198,33 @@ def valid_start(start: dict) -> bool: return False -def _certificate_failure(error: BaseException) -> bool: - """Recognize native and wrapped certificate errors without reporting their text.""" +def _tls_setup_failure(error: BaseException) -> bool: + """Recognize certificate verification/loading errors through transport wrappers.""" pending, seen = [error], set() while pending: current = pending.pop() if id(current) in seen: continue seen.add(id(current)) - if isinstance(current, ssl.SSLCertVerificationError): + if isinstance(current, ssl.SSLCertVerificationError) or ( + isinstance(current, ssl.SSLError) and getattr(current, "library", None) in {"X509", "PEM", "ASN1"} + ): return True # Some adapters flatten the native error into a message. text = str(current).lower() - if "certificate_verify_failed" in text or "certificate verify failed" in text: + if any( + marker in text + for marker in ( + "certificate_verify_failed", + "certificate verify failed", + "no_certificate_or_crl_found", + "no certificate or crl found", + "pem lib", + "[pem:", + "[x509:", + "[asn1:", + ) + ): return True pending.extend( value @@ -230,7 +244,7 @@ def _request(start: dict, data: bytes, content_range: str) -> tuple[int | None, allow_redirects=False, ) except requests.exceptions.SSLError as error: - if _certificate_failure(error): + if _tls_setup_failure(error): raise TlsSetupFailed() from None return None, None except requests.RequestException: diff --git a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md index 53094139e..9d65b5bec 100644 --- a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md +++ b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md @@ -42,7 +42,8 @@ sending any part of it. Only acknowledged bytes advance progress. Transport errors keep no exception text because it can contain the session credential. Expired sessions and permanent storage refusals discard the saved record and restart once; thirty minutes without acknowledged progress pauses. Certificate -verification failures stop immediately with credential-safe TLS guidance. +verification and certificate-bundle loading failures stop immediately with +credential-safe TLS guidance. HTTP 408 and transient TLS failures query held bytes and retry within the same session. TLS setup diagnostics use fixed text, including for normalized URLs. Completion deletes the session after a valid task response, and the command @@ -51,8 +52,9 @@ still have a bounded retry window. The task id is emitted before waiting, so a caller can look it up if the command stops. HTTPS with a host and no userinfo is required for resumable and saved sessions; legacy single-PUT addresses retain the existing validation for compatibility with storage emulators. -Progress output failures mute reporting without stopping task polling or the -stored-hash check. +Progress output failures, including Rich converting broken pipes to process-exit +exceptions, mute reporting without stopping task polling or the stored-hash check. +Later storage-wait notices use the same reporting boundary. ## Alternatives considered @@ -77,7 +79,10 @@ the command loses its sign-in. The model-upload tests use a fake bucket for mid-piece drops, process reruns, final-piece recovery, refusal, expiry, transient errors and byte verification. State tests cover each identity field, expiration, permissions and bounds. -Existing command tests retain approval and stored-hash assertions. CI runs +Real Requests and OpenSSL check invalid local certificate bundles. Real Rich +rendering checks closed streams and broken pipes during storage notices and +polling. Tests isolate Rich's process-wide stdout redirection. Existing command +tests retain approval and stored-hash assertions. CI runs these tests on Linux, macOS and Windows. Real multi-GB storage is not exercised by the fake bucket. diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 73957fbaa..5941324a7 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -2914,3 +2914,110 @@ def put(url, **kw): assert _COMPLETE_1 not in cloud.paths() assert "SECRETSIG" not in everything assert "%53ECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("saved", [False, True], ids=["piece", "status-query"]) +def test_invalid_ca_contents_stop_without_backoff( + cloud_target, monkeypatch, capsys, model_file, no_wait, tmp_path, model, saved +): + """Preserve a real OpenSSL loading error through the transport's wrappers.""" + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, SSLError + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + if saved: + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + _, rc, *_ = run(monkeypatch, capsys, args, routes, [], storage=_ResumableBucket()) + assert rc == 1 + bundle = tmp_path / "invalid-ca.pem" + bundle.write_text("not a certificate\n") + with pytest.raises(ssl.SSLError) as native: + ssl.create_default_context(cafile=str(bundle)) + attempts = [] + + def put(url, **kw): + attempts.append(kw["headers"]["Content-Range"]) + raise requests.exceptions.SSLError(MaxRetryError(None, url, SSLError(native.value))) + + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=put) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert len(attempts) == 1 and no_wait[0] == 0 + assert attempts[0].startswith("bytes */") is saved + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("mismatch", [False, True], ids=["verified", "wrong-hash"]) +@pytest.mark.parametrize("failure", ["broken-pipe", "closed-stream"]) +@pytest.mark.parametrize("notice", ["storing", "polling"]) +def test_real_pretty_output_failure_preserves_polling_and_hash_check( + cloud_target, monkeypatch, model_file, no_wait, model, mismatch, failure, notice +): + """Run real Rich rendering; isolate only its process-wide stdout redirection.""" + import rich.console + + from comfy_cli.command.models import models + + # Keep Rich's real BrokenPipeError -> SystemExit conversion, but do not + # let on_broken_pipe redirect pytest's stdout file descriptor to /dev/null. + redirected = [] + + def redirect(fd, target): + redirected.append((fd, target)) + os.close(fd) + + monkeypatch.setattr(rich.console, "os", SimpleNamespace(**{**vars(os), "dup2": redirect})) + monkeypatch.setattr( + rich.console, "sys", SimpleNamespace(**{**vars(sys), "stdout": SimpleNamespace(fileno=lambda: 1)}) + ) + failed_writes = [] + + class Output(io.StringIO): + def write(self, text): + marker = "Task id:" if notice == "storing" else "still checking" + if failed_writes or marker in text: + failed_writes.append(text) + if failure == "broken-pipe": + raise BrokenPipeError() + raise ValueError("I/O operation on closed file.") + return super().write(text) + + renderer = Renderer(mode=OutputMode.PRETTY) + renderer.pretty_stream = Output() + routes = _resume_routes() + routes[("GET", "/api/tasks/task-1")] = [{"id": "task-1", "status": "running"}] * 40 + [ + {**_DONE, "result": {"success": True, "asset_id": "asset-9"}} + ] + verified = [] + original_stored = assets_library._stored_asset + routes[("GET", "/api/assets/asset-9")] = {"id": "asset-9", "hash": "b" * 64 if mismatch else _MODEL_HASH} + + def stored(*args): + verified.append(True) + # The final result has its own output boundary. Restore that output + # only once the task completed and its hash is about to be checked. + renderer.pretty_stream = io.StringIO() + return original_stored(*args) + + monkeypatch.setattr(assets_library, "_stored_asset", stored) + cloud = _Cloud(routes) + monkeypatch.setattr("urllib.request.urlopen", cloud) + monkeypatch.setattr("requests.put", _ResumableBucket()) + set_renderer(renderer) + result = CliRunner().invoke( + models.app if model else assets_library.app, + ["upload", str(model_file), "--type", "loras", "--where", "cloud", *(["--yes"] if model else [])], + ) + assert verified, repr(result.exception) + assert cloud.paths().count("/api/tasks/task-1") == 41 + assert result.exit_code == (1 if mismatch else 0), repr(result.exception) + assert len(failed_writes) == 1 + assert bool(redirected) is (failure == "broken-pipe") + if mismatch: + assert "asset_upload_unverified" in renderer.pretty_stream.getvalue() diff --git a/tests/comfy_cli/command/test_model_upload_resume.py b/tests/comfy_cli/command/test_model_upload_resume.py index 355b9e0eb..687d25d3a 100644 --- a/tests/comfy_cli/command/test_model_upload_resume.py +++ b/tests/comfy_cli/command/test_model_upload_resume.py @@ -376,3 +376,26 @@ def test_overlapping_uploads_keep_separate_sessions(tmp_path, session): ) def test_resumable_addresses_still_require_https_without_userinfo(session, url): assert not resume.valid_start({**session, "upload_url": url}) + + +@pytest.mark.parametrize("setting", ["REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"]) +@pytest.mark.parametrize( + "contents", ["not a certificate\n", "-----BEGIN CERTIFICATE-----\ninvalid\n-----END CERTIFICATE-----\n"] +) +def test_existing_invalid_ca_bundle_is_fatal_with_real_requests(tmp_path, monkeypatch, setting, contents): + """Exercise Requests, urllib3 and OpenSSL without contacting a remote service.""" + import socket + + bundle = tmp_path / "invalid-ca.pem" + bundle.write_text(contents) + monkeypatch.delenv("REQUESTS_CA_BUNDLE", raising=False) + monkeypatch.setenv(setting, str(bundle)) + monkeypatch.setenv("NO_PROXY", "127.0.0.1") + monkeypatch.setenv("no_proxy", "127.0.0.1") + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + start = {"upload_url": f"https://127.0.0.1:{listener.getsockname()[1]}/?secret=SESSION", "headers": {}} + with pytest.raises(resume.TlsSetupFailed) as error: + resume._request(start, b"", "bytes */16") + assert "SESSION" not in str(error.value) From 8617000705d0e04848e8680bf414ec51ffcc195d Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 14:52:21 -0500 Subject: [PATCH 10/12] fix(models): keep the polling reporter import lazy --- comfy_cli/command/assets_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index 440ca5da4..60a006655 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -774,7 +774,7 @@ def _complete_upload(renderer, target, sign_in, upload_id: str, body: dict, *, r return parsed -def _wait_for_task(renderer, target, task_id: str, sign_in=None, *, reporter: UploadProgressReporter) -> dict: +def _wait_for_task(renderer, target, task_id: str, sign_in=None, *, reporter) -> dict: """Read the task until it ends, riding out a brief 5xx, 429 or dropped connection.""" import http.client import urllib.error From 27a8fe1ca519753b671763c94065e119508f1124 Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 15:06:40 -0500 Subject: [PATCH 11/12] fix(models): stop legacy uploads on certificate failures --- CHANGELOG.md | 5 +- comfy_cli/command/assets_library.py | 11 ++-- ...oads-only-for-the-same-file-and-account.md | 4 ++ .../comfy_cli/command/test_assets_library.py | 65 +++++++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16b8aa805..760871f3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,10 @@ history. A closed progress pipe or stream does not stop task polling or the final stored-hash check, including when Rich converts the pipe failure to an exit. Legacy single-PUT storage addresses - retain HTTP compatibility. + retain HTTP compatibility. Certificate verification and invalid CA-bundle + errors on that path also stop immediately with fixed TLS guidance. +- `comfy build push` shares the upload progress reporter. A broken pipe or + closed progress stream now mutes its progress instead of ending the push. ### Added diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index 60a006655..b69a580e4 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -648,14 +648,15 @@ def _put(upload_url: str, headers: dict, path: Path, progress, digests) -> None: A transport exception is dropped, never shown: its text quotes the signed URL, whose query string is a live credential. A file that cannot be opened or read raises ``_FileUnreadable`` instead: a fresh upload would fail - the same way, so it is not a transport failure to retry. Any other - ``OSError`` comes from setting up TLS (a CA bundle path that no longer - exists) and raises ``_TlsSetupFailed`` with fixed TLS guidance; a retry would - fail the same way too. + the same way, so it is not a transport failure to retry. Certificate + verification/loading errors and other TLS setup ``OSError`` failures + raise ``_TlsSetupFailed`` with fixed TLS guidance; a retry would fail the + same way too. Transient TLS errors still permit a fresh upload. """ import requests from comfy_cli.builder_api import _CountingReader + from comfy_cli.command.model_upload_resume import _tls_setup_failure class _Reader(_CountingReader): # urllib3 reports a body that fails to read as a dropped connection, @@ -700,6 +701,8 @@ def read(self, size: int = -1) -> bytes: raise _FileChanged(str(reader.failed)) from None if reader.failed is not None: raise _FileUnreadable(reader.failed) from None + if isinstance(e, requests.exceptions.SSLError) and _tls_setup_failure(e): + raise _TlsSetupFailed() from None if isinstance(e, requests.RequestException): raise _PutFailed(None) from None # Neither the file nor the transport (Requests wraps those): the diff --git a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md index 9d65b5bec..493d74479 100644 --- a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md +++ b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md @@ -44,6 +44,8 @@ Expired sessions and permanent storage refusals discard the saved record and restart once; thirty minutes without acknowledged progress pauses. Certificate verification and certificate-bundle loading failures stop immediately with credential-safe TLS guidance. +The legacy single-PUT path uses the same certificate classifier and guidance; +transient TLS failures there retain the existing bounded fresh-upload retry. HTTP 408 and transient TLS failures query held bytes and retry within the same session. TLS setup diagnostics use fixed text, including for normalized URLs. Completion deletes the session after a valid task response, and the command @@ -55,6 +57,8 @@ the existing validation for compatibility with storage emulators. Progress output failures, including Rich converting broken pipes to process-exit exceptions, mute reporting without stopping task polling or the stored-hash check. Later storage-wait notices use the same reporting boundary. +The shared reporter also keeps `comfy build push` running after a broken pipe +or closed progress stream; its upload protocol is unchanged. ## Alternatives considered diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 5941324a7..5edffd10e 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -2952,6 +2952,71 @@ def put(url, **kw): assert "SECRETSIG" not in everything +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("failure", ["certificate", "bundle", "flattened"]) +def test_legacy_tls_certificate_failure_stops_without_a_fresh_upload( + cloud_target, monkeypatch, capsys, model_file, no_wait, tmp_path, model, failure +): + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, SSLError + + if failure == "bundle": + bundle = tmp_path / "invalid-ca.pem" + bundle.write_text("not a certificate\n") + with pytest.raises(ssl.SSLError) as native: + ssl.create_default_context(cafile=str(bundle)) + reason = native.value + else: + reason = ssl.SSLCertVerificationError("certificate verify failed") + attempts = [] + + def put(url, **kw): + prepared = requests.Request("PUT", url).prepare().url + attempts.append(prepared) + assert "signature=SECRETSIG" in prepared + assert "Content-Range" not in kw["headers"] + wrapped = MaxRetryError(None, prepared, SSLError(reason)) + raise requests.exceptions.SSLError(str(wrapped) if failure == "flattened" else wrapped) + + routes = dict(_UPLOAD_ROUTES) + routes[("POST", "/api/assets/uploads")] = { + **_started(), + "upload_url": "https://bucket.example/?signature=%53ECRETSIG", + } + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, routes, [], storage=put) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert "CA certificate bundle" in env["error"]["message"] + assert env["error"]["details"]["attempts"] == 1 + assert len(attempts) == 1 and no_wait[0] == 0 + assert cloud.paths("POST").count("/api/assets/uploads") == 1 + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything and "%53ECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +def test_legacy_tls_eof_still_retries_with_a_fresh_upload( + cloud_target, monkeypatch, capsys, model_file, no_wait, model +): + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, SSLError + + failure = requests.exceptions.SSLError(MaxRetryError(None, "SECRETSIG", SSLError(ssl.SSLEOFError("TLS EOF")))) + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, cloud, storage, everything = run(monkeypatch, capsys, args, dict(_UPLOAD_ROUTES), [failure, 200]) + assert rc == 0 and env["ok"] is True + assert len(storage.puts) == 2 + assert cloud.paths("POST").count("/api/assets/uploads") == 2 + assert "/api/tasks/task-1" in cloud.paths() + assert "SECRETSIG" not in everything + + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) @pytest.mark.parametrize("mismatch", [False, True], ids=["verified", "wrong-hash"]) @pytest.mark.parametrize("failure", ["broken-pipe", "closed-stream"]) From bbe16c3693565f2007b85139b593daa6984c03dc Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Fri, 6 Nov 2026 15:23:28 -0500 Subject: [PATCH 12/12] fix(models): stop on proxy certificate failures --- CHANGELOG.md | 2 + comfy_cli/command/assets_library.py | 2 +- comfy_cli/command/model_upload_resume.py | 2 +- ...oads-only-for-the-same-file-and-account.md | 2 + .../comfy_cli/command/test_assets_library.py | 40 +++++++++++++------ .../command/test_model_upload_resume.py | 25 ++++++++++++ 6 files changed, 58 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 760871f3b..4cba8e143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,8 @@ history. Legacy single-PUT storage addresses retain HTTP compatibility. Certificate verification and invalid CA-bundle errors on that path also stop immediately with fixed TLS guidance. + Both paths recognize certificate failures wrapped by an HTTPS proxy while + retaining retries for temporary proxy failures. - `comfy build push` shares the upload progress reporter. A broken pipe or closed progress stream now mutes its progress instead of ending the push. diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index b69a580e4..6ed002828 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -701,7 +701,7 @@ def read(self, size: int = -1) -> bytes: raise _FileChanged(str(reader.failed)) from None if reader.failed is not None: raise _FileUnreadable(reader.failed) from None - if isinstance(e, requests.exceptions.SSLError) and _tls_setup_failure(e): + if isinstance(e, (requests.exceptions.SSLError, requests.exceptions.ProxyError)) and _tls_setup_failure(e): raise _TlsSetupFailed() from None if isinstance(e, requests.RequestException): raise _PutFailed(None) from None diff --git a/comfy_cli/command/model_upload_resume.py b/comfy_cli/command/model_upload_resume.py index 02996b6cc..1d4686d57 100644 --- a/comfy_cli/command/model_upload_resume.py +++ b/comfy_cli/command/model_upload_resume.py @@ -243,7 +243,7 @@ def _request(start: dict, data: bytes, content_range: str) -> tuple[int | None, timeout=(10, 60), allow_redirects=False, ) - except requests.exceptions.SSLError as error: + except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as error: if _tls_setup_failure(error): raise TlsSetupFailed() from None return None, None diff --git a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md index 493d74479..d2a5fad38 100644 --- a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md +++ b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md @@ -44,6 +44,8 @@ Expired sessions and permanent storage refusals discard the saved record and restart once; thirty minutes without acknowledged progress pauses. Certificate verification and certificate-bundle loading failures stop immediately with credential-safe TLS guidance. +This includes certificate failures wrapped by an HTTPS proxy; temporary proxy +failures remain retryable. The legacy single-PUT path uses the same certificate classifier and guidance; transient TLS failures there retain the existing bounded fresh-upload retry. HTTP 408 and transient TLS failures query held bytes and retry within the same diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 5edffd10e..8d63f60ce 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -2843,7 +2843,7 @@ def info(self, message, **kw): @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) -@pytest.mark.parametrize("failure", ["tls-eof", "408"]) +@pytest.mark.parametrize("failure", ["tls-eof", "proxy-tls-eof", "proxy-connect", "408"]) @pytest.mark.parametrize("saved", [False, True], ids=["piece", "status-query"]) def test_transient_storage_failure_keeps_session_and_queries_offset( cloud_target, monkeypatch, capsys, model_file, no_wait, model, failure, saved @@ -2851,7 +2851,7 @@ def test_transient_storage_failure_keeps_session_and_queries_offset( import ssl import requests - from urllib3.exceptions import MaxRetryError, SSLError + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError run = _model_upload if model else _upload args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) @@ -2868,6 +2868,9 @@ def put(url, **kw): if len(calls) == 1: if failure == "tls-eof": raise requests.exceptions.SSLError(MaxRetryError(None, url, SSLError(ssl.SSLEOFError("TLS EOF")))) + if failure.startswith("proxy-"): + reason = ssl.SSLEOFError("TLS EOF") if failure == "proxy-tls-eof" else OSError("proxy unavailable") + raise requests.exceptions.ProxyError(MaxRetryError(None, url, ProxyError("proxy failed", reason))) return SimpleNamespace(status_code=408, headers={}, close=lambda: None) if len(calls) == 2: assert no_wait[0] == 1 @@ -2883,14 +2886,14 @@ def put(url, **kw): @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) -@pytest.mark.parametrize("failure", ["certificate", "setup"]) +@pytest.mark.parametrize("failure", ["certificate", "proxy-certificate", "setup"]) def test_normalized_tls_error_never_exposes_session_credentials( cloud_target, monkeypatch, capsys, model_file, no_wait, model, failure ): import ssl import requests - from urllib3.exceptions import MaxRetryError, SSLError + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError routes = _resume_routes() start = {**_resumable_started(), "upload_url": "https://bucket.example/?upload_id=%53ECRETSIG"} @@ -2901,8 +2904,10 @@ def put(url, **kw): prepared = requests.Request("PUT", url).prepare().url attempts.append(prepared) assert "upload_id=SECRETSIG" in prepared - if failure == "certificate": + if failure in ("certificate", "proxy-certificate"): reason = ssl.SSLCertVerificationError("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed") + if failure == "proxy-certificate": + raise requests.exceptions.ProxyError(MaxRetryError(None, prepared, ProxyError("proxy failed", reason))) raise requests.exceptions.SSLError(MaxRetryError(None, prepared, SSLError(reason))) raise OSError(f"TLS setup failed for {prepared}") @@ -2918,14 +2923,15 @@ def put(url, **kw): @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) @pytest.mark.parametrize("saved", [False, True], ids=["piece", "status-query"]) +@pytest.mark.parametrize("proxy", [False, True], ids=["direct", "proxy"]) def test_invalid_ca_contents_stop_without_backoff( - cloud_target, monkeypatch, capsys, model_file, no_wait, tmp_path, model, saved + cloud_target, monkeypatch, capsys, model_file, no_wait, tmp_path, model, saved, proxy ): """Preserve a real OpenSSL loading error through the transport's wrappers.""" import ssl import requests - from urllib3.exceptions import MaxRetryError, SSLError + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError run = _model_upload if model else _upload args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) @@ -2942,6 +2948,8 @@ def test_invalid_ca_contents_stop_without_backoff( def put(url, **kw): attempts.append(kw["headers"]["Content-Range"]) + if proxy: + raise requests.exceptions.ProxyError(MaxRetryError(None, url, ProxyError("proxy failed", native.value))) raise requests.exceptions.SSLError(MaxRetryError(None, url, SSLError(native.value))) env, rc, cloud, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=put) @@ -2954,13 +2962,14 @@ def put(url, **kw): @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) @pytest.mark.parametrize("failure", ["certificate", "bundle", "flattened"]) +@pytest.mark.parametrize("proxy", [False, True], ids=["direct", "proxy"]) def test_legacy_tls_certificate_failure_stops_without_a_fresh_upload( - cloud_target, monkeypatch, capsys, model_file, no_wait, tmp_path, model, failure + cloud_target, monkeypatch, capsys, model_file, no_wait, tmp_path, model, failure, proxy ): import ssl import requests - from urllib3.exceptions import MaxRetryError, SSLError + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError if failure == "bundle": bundle = tmp_path / "invalid-ca.pem" @@ -2977,8 +2986,9 @@ def put(url, **kw): attempts.append(prepared) assert "signature=SECRETSIG" in prepared assert "Content-Range" not in kw["headers"] - wrapped = MaxRetryError(None, prepared, SSLError(reason)) - raise requests.exceptions.SSLError(str(wrapped) if failure == "flattened" else wrapped) + wrapped = MaxRetryError(None, prepared, ProxyError("proxy failed", reason) if proxy else SSLError(reason)) + error_type = requests.exceptions.ProxyError if proxy else requests.exceptions.SSLError + raise error_type(str(wrapped) if failure == "flattened" else wrapped) routes = dict(_UPLOAD_ROUTES) routes[("POST", "/api/assets/uploads")] = { @@ -2998,15 +3008,19 @@ def put(url, **kw): @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("failure_kind", ["tls-eof", "proxy-tls-eof", "proxy-connect"]) def test_legacy_tls_eof_still_retries_with_a_fresh_upload( - cloud_target, monkeypatch, capsys, model_file, no_wait, model + cloud_target, monkeypatch, capsys, model_file, no_wait, model, failure_kind ): import ssl import requests - from urllib3.exceptions import MaxRetryError, SSLError + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError failure = requests.exceptions.SSLError(MaxRetryError(None, "SECRETSIG", SSLError(ssl.SSLEOFError("TLS EOF")))) + if failure_kind.startswith("proxy-"): + reason = ssl.SSLEOFError("TLS EOF") if failure_kind == "proxy-tls-eof" else OSError("proxy unavailable") + failure = requests.exceptions.ProxyError(MaxRetryError(None, "SECRETSIG", ProxyError("proxy failed", reason))) run = _model_upload if model else _upload args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) env, rc, cloud, storage, everything = run(monkeypatch, capsys, args, dict(_UPLOAD_ROUTES), [failure, 200]) diff --git a/tests/comfy_cli/command/test_model_upload_resume.py b/tests/comfy_cli/command/test_model_upload_resume.py index 687d25d3a..da7d57709 100644 --- a/tests/comfy_cli/command/test_model_upload_resume.py +++ b/tests/comfy_cli/command/test_model_upload_resume.py @@ -399,3 +399,28 @@ def test_existing_invalid_ca_bundle_is_fatal_with_real_requests(tmp_path, monkey with pytest.raises(resume.TlsSetupFailed) as error: resume._request(start, b"", "bytes */16") assert "SESSION" not in str(error.value) + + +@pytest.mark.parametrize("certificate", [False, True], ids=["transient-eof", "certificate"]) +def test_https_proxy_error_uses_real_requests_adapter(session, monkeypatch, certificate): + """Keep Requests' proxy exception translation; replace only the network call.""" + import ssl + + from urllib3.connectionpool import HTTPSConnectionPool + from urllib3.exceptions import MaxRetryError, ProxyError + + calls = [] + + def fail(pool, method, url, **kwargs): + calls.append((method, url)) + reason = ssl.SSLCertVerificationError("certificate verify failed") if certificate else ssl.SSLEOFError("EOF") + raise MaxRetryError(pool, url, ProxyError("proxy failed", reason)) + + monkeypatch.setattr(HTTPSConnectionPool, "urlopen", fail) + if certificate: + with pytest.raises(resume.TlsSetupFailed) as error: + resume._request(session, b"", "bytes */16") + assert "SESSION" not in str(error.value) + else: + assert resume._request(session, b"", "bytes */16") == (None, None) + assert len(calls) == 1