diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 0f5da884c..c9c2e2f9f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -62,6 +62,12 @@ jobs: run: | uv run --locked --extra dev pytest tests/comfy_cli/command/test_jobs_pid_alive.py -v + - name: Test model upload and resume + env: + PYTHONPATH: ${{ github.workspace }} + run: | + uv run --locked --extra dev pytest tests/comfy_cli/command/test_assets_library.py tests/comfy_cli/command/test_model_upload_resume.py + - name: Test e2e env: PYTHONPATH: ${{ github.workspace }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f505eb04..4cba8e143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,32 @@ history. ## [Unreleased] +### Changed + +- Both `comfy assets library upload` and `comfy model upload` request resumable + model uploads up to 25 GB. Dropped connections retry from storage's held + bytes. Running the command again continues a saved upload for the same file, + account, workspace and folder while its session remains valid. Sessions expire + no later than seven days after creation; the server can expire them earlier. + A paused upload reports whether its session was saved for a later run. Older + servers keep the single-PUT upload. File approval, block checks and the final stored-hash + check still apply. The final storage check no longer stops after fifteen + minutes. The retired `asset_upload_timeout` error code stays discoverable + for compatibility with older versions. The task id is reported before the + storage wait. Refused resumable sessions are discarded and restarted once; + certificate verification and invalid CA-bundle failures stop immediately + without transport exception text. + HTTP timeouts and transient TLS interruptions query the held bytes and retry. + A closed progress pipe or stream does not stop task polling or the final + stored-hash check, including when Rich converts the pipe failure to an exit. + Legacy single-PUT storage addresses + retain HTTP compatibility. Certificate verification and invalid CA-bundle + errors on that path also stop immediately with fixed TLS guidance. + Both paths recognize certificate failures wrapped by an HTTPS proxy while + retaining retries for temporary proxy failures. +- `comfy build push` shares the upload progress reporter. A broken pipe or + closed progress stream now mutes its progress instead of ending the push. + ### Added - The `comfy-run-on-cloud` agent skill, installed with the other bundled diff --git a/README.md b/README.md index 11d64ca35..1f680289c 100644 --- a/README.md +++ b/README.md @@ -444,7 +444,9 @@ the bisect tool can help you pinpoint the custom node that causes the issue. `comfy model upload --type --where cloud ?[--dry-run] ?[--workspace ] ?[--expected-sha256 ] ?[--yes]` - - Puts one local `.safetensors` file (up to 2 GB) into your Comfy Cloud model library, so a workflow run on Comfy Cloud can load it. `--type` is the model folder, e.g. `loras` or `checkpoints`; `comfy model list-folders --where cloud` lists them. + - Puts one local `.safetensors` file (up to 25 GB on servers with resumable uploads) into your Comfy Cloud model library, so a workflow run on Comfy Cloud can load it. `--type` is the model folder, e.g. `loras` or `checkpoints`; `comfy model list-folders --where cloud` lists them. + - Uploads use 64 MiB pieces when the server supports them. A dropped connection retries from the bytes storage holds. If the command stops, run it again with the same file, model folder, account and workspace to continue while its saved session remains valid. Sessions expire no later than seven days after creation; the server can expire them earlier. A paused upload tells you whether its session was saved. Without saved state, a rerun starts a new upload. The CLI hashes the file again before resuming. Older servers keep using one PUT and enforce their own size limit. Upload sessions are credentials, kept in a private `model-uploads` directory under the CLI config folder and removed after completion. + - After 30 minutes without progress, the command exits with the held byte count. The final check on Comfy Cloud can take longer than fifteen minutes; the command waits until it ends. It reports the task id before waiting. In JSON mode, the `upload_storing` event goes to stderr; in streaming mode it goes to stdout. If the wait is interrupted, the id identifies the task at Cloud's `GET /api/tasks/` endpoint. - Only the file you name is uploaded, and only the bytes hashed before the upload: each block is checked against its hash before it is sent. The upload succeeds only once Comfy Cloud's hash of the stored file matches your file's. - `--dry-run` uploads nothing. It reports the file's `sha256` and whether Comfy Cloud already has the file. - The output's `loader_path` is the value a loader node takes for the model on Comfy Cloud, as Comfy Cloud's record of the model gives it. It is null for a dry run that would upload, or when Comfy Cloud gives none. diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index 7ee9bf5e0..6ed002828 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -24,6 +24,7 @@ handle_cloud_http_error, http_request, ) +from comfy_cli.command.model_upload_resume import TlsSetupFailed as _TlsSetupFailed from comfy_cli.output.renderer import get_renderer app = typer.Typer(help="Browse your Comfy Cloud asset library (list, borrow, upload).") @@ -346,9 +347,9 @@ def ensure_cmd( # What Comfy Cloud takes from `upload` (POST /api/assets/uploads): one -# `.safetensors` file of 1 byte to 2 GiB under a bare name of at most 255 bytes. +# `.safetensors` file under a bare name of at most 255 bytes. Start enforces +# the size limit (up to 25 GiB when the server supports resumable uploads). # A pickle format (.ckpt, .pt, .bin) is refused: it can run code when loaded. -_MAX_UPLOAD_BYTES = 2 * 1024**3 _MAX_FILENAME_BYTES = 255 # The first upload plus one fresh one after a PUT that got no answer or a 5xx. # Fresh, not a second PUT to the same URL: a signed URL lasts 15 minutes, which @@ -358,9 +359,8 @@ def ensure_cmd( # id names a fresh object, so it can only be this upload's: the bytes landed. _ALREADY_WRITTEN = 412 _PUT_TIMEOUT = (10, 600) -# Storing the file (hashing up to 2 GiB) normally takes a minute or two. +# Large uploads can take more than fifteen minutes to check and store. _TASK_POLL_SECONDS = 2.0 -_TASK_WAIT_SECONDS = 15 * 60.0 # A task read that gets a 5xx or no answer is retried for this long from the # first failure in a row; a 4xx ends the wait at once. _UNANSWERED_WINDOW_SECONDS = 60.0 @@ -370,7 +370,7 @@ def ensure_cmd( "MODEL_STORAGE_LIMIT_REACHED": "delete models you no longer need, or move to a plan with more model storage", "UNSUPPORTED_FILE_TYPE": "convert the model to .safetensors", "UNSUPPORTED_MODEL_TYPE": "pass a --type from `model_upload_folders` in GET /api/features", - "FILE_TOO_LARGE": "files can be up to 2 GB", + "FILE_TOO_LARGE": "use a file within the size limit in the server message", } # Any form Comfy Cloud reports a model's content hash in: bare hex, canonical # ``blake3:`` or the stored ``.``, any case. @@ -401,10 +401,6 @@ class _FileUnreadable(Exception): """The file could not be opened or read while it was being sent; ``args[0]`` is the ``OSError``.""" -class _TlsSetupFailed(Exception): - """This machine's TLS settings stopped the PUT before it connected; ``args[0]`` is the message to show.""" - - class _FileChanged(Exception): """A block of the file read for the PUT is not the block hashed before it; ``args[0]`` is the block's index.""" @@ -423,11 +419,14 @@ class _HashedBytes: since are not sent. """ - def __init__(self, raw, digests) -> None: + def __init__(self, raw, digests, offset: int = 0) -> None: self._raw, self._digests = raw, digests self._pending = b"" - self._next_block = 0 - self._given = 0 + self._next_block = offset // digests.block_size + self._skip = offset % digests.block_size + self._given = offset + if offset: + raw.seek(self._next_block * digests.block_size) def read(self, size: int = -1) -> bytes: from blake3 import blake3 @@ -439,7 +438,8 @@ def read(self, size: int = -1) -> bytes: block = self._raw.read(min(d.block_size, d.size - self._next_block * d.block_size)) if blake3(block).digest() != d.blocks[self._next_block]: raise _BlockChanged(f"block {self._next_block} of the file is not the one hashed") - self._pending, self._next_block = block, self._next_block + 1 + self._pending, self._next_block = block[self._skip :], self._next_block + 1 + self._skip = 0 n = len(self._pending) if size is None or size < 0 else size out, self._pending = self._pending[:n], self._pending[n:] self._given += len(out) @@ -498,8 +498,6 @@ def _local_refusal(path: Path) -> tuple[str, str] | None: return _file_problem(path, e) if size < 1: return "empty_file", f"{path} is empty" - if size > _MAX_UPLOAD_BYTES: - return "file_too_large", f"{name} is {size} bytes; files can be up to 2 GB ({_MAX_UPLOAD_BYTES} bytes)" return None @@ -650,16 +648,15 @@ def _put(upload_url: str, headers: dict, path: Path, progress, digests) -> None: A transport exception is dropped, never shown: its text quotes the signed URL, whose query string is a live credential. A file that cannot be opened or read raises ``_FileUnreadable`` instead: a fresh upload would fail - the same way, so it is not a transport failure to retry. Any other - ``OSError`` comes from setting up TLS (a CA bundle path that no longer - exists) and raises ``_TlsSetupFailed`` with its message; a retry would - fail the same way too. + the same way, so it is not a transport failure to retry. Certificate + verification/loading errors and other TLS setup ``OSError`` failures + raise ``_TlsSetupFailed`` with fixed TLS guidance; a retry would fail the + same way too. Transient TLS errors still permit a fresh upload. """ - import urllib.parse - import requests from comfy_cli.builder_api import _CountingReader + from comfy_cli.command.model_upload_resume import _tls_setup_failure class _Reader(_CountingReader): # urllib3 reports a body that fails to read as a dropped connection, @@ -704,21 +701,83 @@ def read(self, size: int = -1) -> bytes: raise _FileChanged(str(reader.failed)) from None if reader.failed is not None: raise _FileUnreadable(reader.failed) from None + if isinstance(e, (requests.exceptions.SSLError, requests.exceptions.ProxyError)) and _tls_setup_failure(e): + raise _TlsSetupFailed() from None if isinstance(e, requests.RequestException): raise _PutFailed(None) from None # Neither the file nor the transport (Requests wraps those): the - # TLS setup, e.g. REQUESTS_CA_BUNDLE naming a removed file. Its - # text names a certificate path, but is never shown if it quotes - # the signed URL. - text, query = str(e) or type(e).__name__, urllib.parse.urlsplit(upload_url).query - if upload_url in text or (query and query in text): - text = type(e).__name__ - raise _TlsSetupFailed(text) from None + # TLS setup, e.g. REQUESTS_CA_BUNDLE naming a removed file. Never + # retain its text: Requests may normalize a credential in the URL. + raise _TlsSetupFailed() from None if not (200 <= resp.status_code < 300 or resp.status_code == _ALREADY_WRITTEN): raise _PutFailed(resp.status_code) -def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: +def _read_range(path, digests, offset: int, length: int) -> bytes: + try: + with path.open("rb") as raw: + reader = _HashedBytes(raw, digests, offset) + blocks = [] + while length: + block = reader.read(length) + if not block: + raise _FileChanged() + blocks.append(block) + length -= len(block) + return b"".join(blocks) + except _BlockChanged: + raise _FileChanged() from None + except OSError as e: + raise _FileUnreadable(e) from None + + +def _upload_identity(target, sign_in) -> dict | None: + """Only server-confirmed account and workspace ids permit saved-session reuse.""" + import http.client + import urllib.error + + try: + _, user = _send(target, sign_in, target.url("user"), strict_json=True) + workspace = _current_workspace(target, sign_in) + except (urllib.error.URLError, OSError, http.client.HTTPException, ResponseUnparseable): + return None + account = user.get("id") if isinstance(user, dict) else None + if not isinstance(account, str) or not account or not workspace: + return None + return {"account": account, "workspace": workspace} + + +def _complete_upload(renderer, target, sign_in, upload_id: str, body: dict, *, resumable: bool) -> dict: + import http.client + import urllib.error + import urllib.parse + + from comfy_cli.command.model_upload_resume import STALL_SECONDS, SessionExpired + + parts = ("assets", "uploads", urllib.parse.quote(upload_id, safe=""), "complete") + if not resumable: + return _call(renderer, target, "complete", parts, body, sign_in=sign_in) + started, delay = _clock(), 1 + while True: + try: + _, parsed = _send(target, sign_in, target.url(*parts), method="POST", body=body, strict_json=True) + except (urllib.error.URLError, OSError, http.client.HTTPException) as e: + code = e.code if isinstance(e, urllib.error.HTTPError) else None + if code == 404: + raise SessionExpired() from None + if (code is not None and code != 429 and code < 500) or _clock() - started >= STALL_SECONDS: + raise _http_failure(renderer, e, "complete", sign_in) from None + _sleep(min(delay, STALL_SECONDS - (_clock() - started))) + delay = min(delay * 2, 60) + except ResponseUnparseable: + raise _unexpected(renderer, "/api/assets/uploads/complete", "body was not valid JSON") from None + else: + if not isinstance(parsed, dict): + raise _unexpected(renderer, "/api/assets/uploads/complete", "expected a JSON object") + return parsed + + +def _wait_for_task(renderer, target, task_id: str, sign_in=None, *, reporter) -> dict: """Read the task until it ends, riding out a brief 5xx, 429 or dropped connection.""" import http.client import urllib.error @@ -727,7 +786,7 @@ def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: from comfy_cli.command._cloud_errors import retry_after_from_headers url = target.url("tasks", urllib.parse.quote(task_id, safe="")) - deadline = _clock() + _TASK_WAIT_SECONDS + started, last_notice = _clock(), _clock() unanswered_since: float | None = None status = None while True: @@ -744,8 +803,8 @@ def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: raise _http_failure(renderer, e, "task", sign_in) from e retry_after = retry_after_from_headers(getattr(e, "headers", None)) if code == 429 else None if retry_after is not None: - # Never past the end of the window or of the whole wait. - left = min(unanswered_since + _UNANSWERED_WINDOW_SECONDS, deadline) - now + # Never past the end of the unanswered window. + left = unanswered_since + _UNANSWERED_WINDOW_SECONDS - now pause = max(_TASK_POLL_SECONDS, min(retry_after, left)) except ResponseUnparseable as e: raise _unexpected(renderer, "/api/tasks", "body was not valid JSON", task_id=task_id) from e @@ -756,14 +815,9 @@ def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: status = task["status"] if status in _TASK_DONE: return task - if _clock() >= deadline: - renderer.error( - code="asset_upload_timeout", - message=f"Comfy Cloud was still storing the file after {int(_TASK_WAIT_SECONDS)}s", - hint="storing goes on without this command; check `comfy assets library ls --tags models` later", - details={"task_id": task_id, "status": status}, - ) - raise typer.Exit(code=1) + if renderer.is_pretty() and _clock() - last_notice >= 30: + reporter.waiting(int(_clock() - started)) + last_notice = _clock() _sleep(pause) @@ -859,7 +913,7 @@ def _loader_path(asset: dict | None) -> str | None: @app.command( "upload", - help="Upload one named .safetensors model file (up to 2 GB) into your Comfy Cloud model library. " + help="Upload one named .safetensors model file (up to 25 GB) into your Comfy Cloud model library. Run the same command again to resume a stopped upload. " "Succeeds only once Comfy Cloud's hash of the stored file matches this file's; --dry-run checks without uploading.", ) @tracking.track_command("assets") @@ -906,9 +960,16 @@ def run_upload( the folder; without it the file is uploaded under that name, as a new version of it. """ - import urllib.parse - from comfy_cli.command.build_upload_progress import UploadProgressReporter + from comfy_cli.command.model_upload_resume import ( + SessionExpired, + SessionStore, + TransferFailed, + UploadPaused, + transfer, + valid_start, + ) + from comfy_cli.config_manager import ConfigManager from comfy_cli.credentials import DataPlaneToken from comfy_cli.hashing import file_digests from comfy_cli.interaction import confirm @@ -987,6 +1048,7 @@ def refuse(reason: str, message: str, **details: Any) -> typer.Exit: # One read gives the hashes, the size and each block's digest; the PUT # sends only bytes that match these blocks (`_HashedBytes`). digests = file_digests(file) + modified_ns = file.stat().st_mtime_ns except OSError as e: # removed or locked since it was checked; nothing is sent yet raise refuse(*_file_problem(file, e)) from None local_hash, sha256, size = digests.blake3, digests.sha256, digests.size @@ -1063,31 +1125,85 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None tags = ["models", folder] reporter = UploadProgressReporter(renderer) reporter.plan([item], already_held=0) + store = SessionStore( + Path(ConfigManager().get_config_path()) / "model-uploads", + { + "server": target.url(), + "path": str(file.resolve()), + "size": size, + "mtime_ns": modified_ns, + "hash": local_hash, + "folder": folder, + }, + ) + identity = _upload_identity(target, sign_in) if store.has_saved() else None + saved = store.load(identity) for attempt in range(1, _UPLOAD_STARTS + 1): - start = _call( + resumed = saved is not None + session_saved = resumed + start = saved or _call( renderer, target, "start", ("assets", "uploads"), - {"filename": item.filename, "size_bytes": item.size_bytes, "tags": tags}, + {"filename": item.filename, "size_bytes": item.size_bytes, "tags": tags, "resumable": True}, sign_in=sign_in, ) - upload_id, upload_url, headers = start.get("upload_id"), start.get("upload_url"), start.get("headers") - if not ( - isinstance(upload_id, str) - and upload_id - and isinstance(upload_url, str) - and upload_url - and start.get("method") == "PUT" - and isinstance(headers, dict) - and all(isinstance(k, str) and isinstance(v, str) for k, v in headers.items()) - ): - # Never the URL itself in the error: it is a credential. + saved = None + if not valid_start(start): raise _unexpected(renderer, "/api/assets/uploads", "expected upload_id, upload_url, method PUT and headers") + upload_id, upload_url, headers = start["upload_id"], start["upload_url"], start["headers"] + resumable = start.get("upload_mode") == "resumable" + if resumable and not resumed: + identity = identity or _upload_identity(target, sign_in) + session_saved = store.save(start, identity) + if not session_saved: + renderer.warn("This upload cannot be saved for a later run; connection retries still work.") try: with reporter.uploading(item, 1, 1) as progress: - _put(upload_url, headers, file, progress, digests) + if resumable: + transfer( + start, + size, + lambda offset, length: _read_range(file, digests, offset, length), + progress, + resumed=resumed, + clock=_clock, + sleep=_sleep, + ) + else: + _put(upload_url, headers, file, progress, digests) + task_id = _complete_upload( + renderer, + target, + sign_in, + upload_id, + {"filename": item.filename, "tags": tags}, + resumable=resumable, + ).get("task_id") + if not isinstance(task_id, str) or not task_id: + raise _unexpected(renderer, "/api/assets/uploads/complete", "no task_id", upload_id=upload_id) + store.clear() break + except UploadPaused as e: + from comfy_cli.output.progress import human_bytes + + next_run = ( + "Run the same command again to continue while the saved session remains valid." + if session_saved + else "This session was not saved. Run the same command again to start a new upload." + ) + renderer.error( + code="asset_upload_failed", + message=f"Upload paused at {human_bytes(e.args[0])} of {human_bytes(size)}. {next_run}", + details={ + "reason": "upload_paused", + "upload_id": upload_id, + "bytes_held": e.args[0], + "session_saved": session_saved, + }, + ) + raise typer.Exit(code=1) from None except _FileUnreadable as e: renderer.error( code="asset_upload_failed", @@ -1098,6 +1214,7 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None ) raise typer.Exit(code=1) from None except _FileChanged: + store.clear() # Only the blocks before the changed one were sent, and the # upload is never completed, so the staged bytes never become an asset. renderer.error( @@ -1111,14 +1228,27 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None renderer.error( code="asset_upload_failed", message=f"could not set up a secure connection to storage ({e.args[0]})", - hint="point REQUESTS_CA_BUNDLE (or CURL_CA_BUNDLE) at a certificate bundle that exists, or unset it, " - "then upload again; nothing was added to the library", + hint="check the storage certificate and proxy, and point REQUESTS_CA_BUNDLE (or CURL_CA_BUNDLE) " + "at a trusted certificate bundle that exists, or unset it; then upload again; " + "nothing was added to the library", details={"reason": "tls_setup", "upload_id": upload_id, "attempts": attempt}, ) raise typer.Exit(code=1) from None - except _PutFailed as e: - status = e.args[0] - if (status is None or status >= 500) and attempt < _UPLOAD_STARTS: + except (_PutFailed, TransferFailed, SessionExpired) as e: + expired = isinstance(e, SessionExpired) + status = 404 if expired else e.args[0] + refused = ( + resumable + and isinstance(e, TransferFailed) + and status is not None + and 400 <= status < 500 + and status not in (408, 429) + ) + if expired or refused: + store.clear() + if ( + expired or refused or (not resumable and (status is None or status >= 500)) + ) and attempt < _UPLOAD_STARTS: renderer.warn("The upload to storage did not go through; starting a fresh upload.") continue answer = "no response" if status is None else f"HTTP {status}" @@ -1130,15 +1260,8 @@ def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None ) raise typer.Exit(code=1) from None - complete = ("assets", "uploads", urllib.parse.quote(upload_id, safe=""), "complete") - task_id = _call( - renderer, target, "complete", complete, {"filename": item.filename, "tags": tags}, sign_in=sign_in - ).get("task_id") - if not isinstance(task_id, str) or not task_id: - raise _unexpected(renderer, "/api/assets/uploads/complete", "no task_id", upload_id=upload_id) - if renderer.is_pretty(): - renderer.info("Uploaded; Comfy Cloud is checking and storing the file.") - task = _wait_for_task(renderer, target, task_id, sign_in) + reporter.storing(task_id) + task = _wait_for_task(renderer, target, task_id, sign_in, reporter=reporter) result = task.get("result") if isinstance(task.get("result"), dict) else {} if task["status"] != "completed" or result.get("success") is False: diff --git a/comfy_cli/command/build_upload_progress.py b/comfy_cli/command/build_upload_progress.py index b00ab4ca8..9f5f66f84 100644 --- a/comfy_cli/command/build_upload_progress.py +++ b/comfy_cli/command/build_upload_progress.py @@ -44,6 +44,7 @@ EVENT_PLAN: Final = "upload_plan" EVENT_PROGRESS: Final = "upload_progress" EVENT_COMPLETE: Final = "upload_complete" +EVENT_STORING: Final = "upload_storing" class Clock(Protocol): @@ -152,6 +153,17 @@ def plan(self, uploads: tuple[UploadItem, ...] | list[UploadItem], *, already_he else: self._say(plan_line(len(uploads), self._overall_total, already_held)) + def storing(self, task_id: str) -> None: + """Report the model's storage task without letting a closed pipe stop verification.""" + if self._renderer.is_pretty(): + self._say(f"Uploaded; Comfy Cloud is checking and storing the file. Task id: {task_id}") + self._event(EVENT_STORING, task_id=task_id) + + def waiting(self, seconds: int) -> None: + """Keep storage polling notices inside the same muted-output boundary.""" + if self._renderer.is_pretty(): + self._say(f"Comfy Cloud is still checking and storing the file ({seconds}s so far).") + # ----- one file ----- @contextmanager @@ -276,15 +288,18 @@ def _event(self, type: str, **fields: Any) -> None: return try: self._renderer.progress_event(type, **fields) - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True def _say(self, message: str) -> None: if self._muted: return + # Rich converts a broken pipe to SystemExit and a closed stream to + # ValueError. Suppress those only at this progress-output boundary; + # task polling and byte verification must still finish. try: self._renderer.info(message) - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True def _open_live(self, transfer: _Transfer) -> None: @@ -316,18 +331,18 @@ def _open_live(self, transfer: _Transfer) -> None: rate="", left="", ) - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True try: live.stop() - except OSError: + except (OSError, ValueError, SystemExit): pass return self._live = live self._live_task = task def _update_live(self, numbers: dict[str, Any]) -> None: - if self._live is None: + if self._live is None or self._muted: return rate = numbers["bytes_per_second"] left = numbers["overall_eta_seconds"] @@ -338,7 +353,7 @@ def _update_live(self, numbers: dict[str, Any]) -> None: rate="" if rate is None else f"{human_bytes(rate)}/s", left="" if left is None else f"{human_seconds(left)} left", ) - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True def _close_live(self) -> None: @@ -347,7 +362,7 @@ def _close_live(self) -> None: return try: live.stop() - except OSError: + except (OSError, ValueError, SystemExit): self._muted = True diff --git a/comfy_cli/command/model_upload_resume.py b/comfy_cli/command/model_upload_resume.py new file mode 100644 index 000000000..1d4686d57 --- /dev/null +++ b/comfy_cli/command/model_upload_resume.py @@ -0,0 +1,304 @@ +"""Private, expiring model-upload sessions and the bucket's acknowledged byte range.""" + +from __future__ import annotations + +import json +import math +import os +import re +import ssl +import stat +import tempfile +import time +from datetime import datetime +from pathlib import Path +from urllib.parse import urlsplit +from uuid import uuid4 + +import requests + +CHUNK_BYTES = 64 * 1024**2 +LIFETIME = 7 * 24 * 60 * 60 +STALL_SECONDS = 30 * 60 +_MAX_ENTRIES = 32 +_RANGE = re.compile(r"bytes=0-([0-9]+)$") + + +class SessionExpired(Exception): + """Storage or complete no longer knows this upload; start again.""" + + +class TransferFailed(Exception): + """Only a status is safe to expose; never keep a transport exception.""" + + +class TlsSetupFailed(Exception): + """A permanent TLS failure; never retain transport exception text.""" + + def __init__(self) -> None: + super().__init__("certificate verification or CA certificate bundle setup failed") + + +class UploadPaused(Exception): + """The argument is the last byte count acknowledged by storage.""" + + +def _expiry(start: dict) -> float: + try: + return datetime.fromisoformat(start["expires_at"].replace("Z", "+00:00")).timestamp() + except (KeyError, TypeError, ValueError, OverflowError, AttributeError): + return 0.0 + + +def chunk_size(start: dict) -> int: + value = start.get("chunk_size_bytes") + if type(value) is not int or value < 256 * 1024 or value > CHUNK_BYTES or value % (256 * 1024): + raise TransferFailed(None) + return value + + +class SessionStore: + """At most 32 bounded JSON files, written atomically with user-only access. + + No credential is used to identify the account: the caller supplies the ids + returned by the authenticated user and workspace endpoints. Unknown identity + disables persistence. Failure to secure or write the directory stops saving, + but does not stop the current transfer. + """ + + def __init__(self, directory: Path, binding: dict) -> None: + self.directory, self.binding = directory, binding + self.path: Path | None = None + + def _entries(self) -> list[Path]: + if self.directory.is_symlink(): + return [] + try: + entries = list(self.directory.glob("*.json")) + temporary = sorted(self.directory.glob(".upload-*"), key=lambda p: p.lstat().st_mtime) + for index, path in enumerate(temporary): + if index < len(temporary) - _MAX_ENTRIES or time.time() - path.lstat().st_mtime >= LIFETIME: + path.unlink(missing_ok=True) + valid = [] + now = time.time() + for path in entries: + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + continue + if info.st_size > 16384 or now - info.st_mtime >= LIFETIME: + path.unlink(missing_ok=True) + else: + valid.append(path) + return sorted(valid, key=lambda p: p.stat().st_mtime) + except OSError: + return [] + + def has_saved(self) -> bool: + return bool(self._entries()) + + def _secure(self) -> None: + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + if self.directory.is_symlink(): + raise OSError("session directory is a symlink") + self.directory.chmod(0o700) + + def load(self, identity: dict | None) -> dict | None: + if identity is None: + return None + try: + self._secure() + except OSError: + return None + for path in reversed(self._entries()): + try: + info = path.lstat() + if os.name != "nt" and (info.st_uid != os.getuid() or info.st_mode & 0o077): + continue + with path.open(encoding="utf-8") as stream: + saved = json.load(stream) + if not isinstance(saved, dict) or not isinstance(saved.get("start"), dict): + continue + created = saved.get("created_at") + if not isinstance(created, (float, int)) or not math.isfinite(created) or created > time.time(): + continue + start = saved["start"] + if min(created + LIFETIME, _expiry(start)) <= time.time(): + path.unlink(missing_ok=True) + continue + if saved.get("binding") == self.binding and saved.get("identity") == identity: + # Validate before any request to a saved credential address. + if start.get("upload_mode") != "resumable" or not valid_start(start): + continue + chunk_size(start) + self.path = path + return start + except (OSError, ValueError, KeyError, TypeError, TransferFailed): + continue + return None + + def save(self, start: dict, identity: dict | None) -> bool: + if identity is None or _expiry(start) <= time.time(): + return False + temporary = None + try: + self._secure() + # Two commands may both start before either has saved a session. + # Each owns a separate record, including when their bindings match. + path = self.directory / f"{uuid4().hex}.json" + entries = self._entries() + for old in entries[: max(0, len(entries) - _MAX_ENTRIES + 1)]: + old.unlink(missing_ok=True) + fd, temporary = tempfile.mkstemp(prefix=".upload-", dir=self.directory) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump( + {"binding": self.binding, "identity": identity, "created_at": time.time(), "start": start}, stream + ) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + self.path = path + return True + except (OSError, ValueError): + return False + finally: + if temporary is not None: + try: + Path(temporary).unlink(missing_ok=True) + except OSError: + pass + + def clear(self) -> None: + if self.path is not None: + try: + self.path.unlink(missing_ok=True) + except OSError: + pass + self.path = None + + +def valid_start(start: dict) -> bool: + if not isinstance(start.get("upload_url"), str): + return False + try: + url = urlsplit(start.get("upload_url", "")) + return ( + isinstance(start.get("upload_id"), str) + and bool(start["upload_id"]) + and bool(start["upload_url"]) + and ( + start.get("upload_mode") is None + or (url.scheme == "https" and bool(url.hostname) and not url.username and not url.password) + ) + and start.get("method") == "PUT" + and isinstance(start.get("headers"), dict) + and all(isinstance(k, str) and isinstance(v, str) for k, v in start["headers"].items()) + and start.get("upload_mode") in (None, "resumable") + ) + except (TypeError, ValueError): + return False + + +def _tls_setup_failure(error: BaseException) -> bool: + """Recognize certificate verification/loading errors through transport wrappers.""" + pending, seen = [error], set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, ssl.SSLCertVerificationError) or ( + isinstance(current, ssl.SSLError) and getattr(current, "library", None) in {"X509", "PEM", "ASN1"} + ): + return True + # Some adapters flatten the native error into a message. + text = str(current).lower() + if any( + marker in text + for marker in ( + "certificate_verify_failed", + "certificate verify failed", + "no_certificate_or_crl_found", + "no certificate or crl found", + "pem lib", + "[pem:", + "[x509:", + "[asn1:", + ) + ): + return True + pending.extend( + value + for value in (*current.args, getattr(current, "reason", None), current.__cause__, current.__context__) + if isinstance(value, BaseException) + ) + return False + + +def _request(start: dict, data: bytes, content_range: str) -> tuple[int | None, str | None]: + try: + response = requests.put( + start["upload_url"], + data=data, + headers={**start["headers"], "Content-Length": str(len(data)), "Content-Range": content_range}, + timeout=(10, 60), + allow_redirects=False, + ) + except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as error: + if _tls_setup_failure(error): + raise TlsSetupFailed() from None + return None, None + except requests.RequestException: + # Transport exceptions may quote the session credential. + return None, None + except OSError: + raise TlsSetupFailed() from None + try: + return response.status_code, response.headers.get("Range") + finally: + response.close() + + +def transfer( + start: dict, size: int, read_range, progress, *, resumed: bool, clock=time.monotonic, sleep=time.sleep +) -> None: + """Query after every uncertain PUT; only acknowledged bytes advance progress. + + ``read_range`` checks each whole hashing block before returning a piece. + It may start inside a block, because a connection can drop at any byte. + """ + piece_size = chunk_size(start) + offset, query, delay, last_progress = 0, resumed, 1, clock() + while True: + if clock() - last_progress >= STALL_SECONDS: + raise UploadPaused(offset) + end = offset if query else min(offset + piece_size, size) + body = b"" if query else read_range(offset, end - offset) + content_range = f"bytes */{size}" if query else f"bytes {offset}-{end - 1}/{size}" + status, held = _request(start, body, content_range) + if status in (200, 201, 412): + progress(size - offset) + return + if status in (404, 410): + raise SessionExpired() + if status == 308: + match = _RANGE.fullmatch(held) if isinstance(held, str) else None + if held is not None and match is None: + raise TransferFailed(308) + acknowledged = int(match.group(1)) + 1 if match else 0 + if acknowledged < offset or acknowledged > size or (not query and acknowledged > end): + raise TransferFailed(308) + if acknowledged > offset: + progress(acknowledged - offset) + offset, last_progress, delay = acknowledged, clock(), 1 + if offset == size: + return + query = False + continue + # A fresh empty status is normal; a piece with no progress backs off. + if query: + query = False + continue + elif status is not None and status not in (408, 429) and status < 500: + raise TransferFailed(status) + sleep(min(delay, max(0, STALL_SECONDS - (clock() - last_progress)))) + delay, query = min(delay * 2, 60), True diff --git a/comfy_cli/command/models/upload.py b/comfy_cli/command/models/upload.py index 965e2d4fd..42bb7b2e6 100644 --- a/comfy_cli/command/models/upload.py +++ b/comfy_cli/command/models/upload.py @@ -21,9 +21,9 @@ @app.command( "upload", - help="Upload one named .safetensors model file (up to 2 GB) to your Comfy Cloud model library, so a " + help="Upload one named .safetensors model file (up to 25 GB) to your Comfy Cloud model library, so a " "workflow run on Comfy Cloud can load it. Uploads only the file named; succeeds only once Comfy Cloud's " - "hash of the stored file matches this file's. Run with --dry-run first to see its sha256 and whether " + "hash of the stored file matches this file's. Dropped uploads resume when you run the same command again. Run with --dry-run first to see its sha256 and whether " "Comfy Cloud already has it.", ) @tracking.track_command("model") diff --git a/comfy_cli/error_codes.py b/comfy_cli/error_codes.py index 162466b12..d633978cd 100644 --- a/comfy_cli/error_codes.py +++ b/comfy_cli/error_codes.py @@ -5,11 +5,11 @@ - ``tests/comfy_cli/output/test_error_code_registry.py``: every raised code is registered - every registered code is raised somewhere + every registered code is raised somewhere or explicitly retired That makes this module the canonical contract for agents. Agents fetch the -list via ``comfy discover`` and branch on the codes; if you rename, deprecate, -or remove one, you're breaking the contract and the tests fail before merge. +list via ``comfy discover`` and branch on the codes. Retired codes stay +registered for compatibility; never rename, repurpose or remove one. Codes are snake_case and match ``^[a-z][a-z0-9_]*$``. @@ -498,7 +498,7 @@ class ErrorCode: "asset_upload_invalid", "`model upload` or `assets library upload` refused the file or `--type` before uploading anything: " "`details.reason` is unsupported_file_type (not .safetensors), bad_filename, file_not_found, " - "file_unreadable (it could not be opened or its size read), empty_file, file_too_large (over 2 GB), " + "file_unreadable (it could not be opened or its size read), empty_file, " "unsupported_model_type (`details.valid_types` lists the folders the workspace takes), " "bad_expected_sha256 (`--expected-sha256` is not 64 hex), file_changed (the file's sha256, " "`details.sha256`, is not `details.expected_sha256`: it changed since the dry run that was approved), " @@ -529,13 +529,16 @@ class ErrorCode: "Comfy Cloud refused a model upload; the message is the server's. `details.server_code` is its " "code, e.g. FEATURE_DISABLED (uploads are off for this workspace), MODEL_STORAGE_LIMIT_REACHED " "(the message names the cap), UNSUPPORTED_FILE_TYPE, UNSUPPORTED_MODEL_TYPE or FILE_TOO_LARGE. " - "Retrying the same upload gets the same answer.", + "Cloud decides the size limit. Retrying the same upload gets the same answer.", "act on the server's message; do not retry unchanged", ), ErrorCode( "asset_upload_failed", "A model upload did not end in a stored asset: sending the bytes to storage failed after a fresh " - "retry (`details.status`, null when no answer came), the local file could not be read while it " + "retry (`details.status`, null when no answer came), a resumable transfer paused after thirty minutes " + "without acknowledged progress (`details.reason` upload_paused; `details.bytes_held` counts the bytes " + "storage holds; `details.session_saved` says whether the session was saved for a later run), " + "the local file could not be read while it " "was being sent (`details.reason` file_unreadable; not retried), this machine's TLS settings stopped " "the PUT before it connected (`details.reason` tls_setup, e.g. REQUESTS_CA_BUNDLE naming a missing " "file; not retried), a block of the file read for sending was not the block hashed before it " @@ -543,13 +546,16 @@ class ErrorCode: "none of the changed bytes were, and the upload was not completed, so no asset was added), or " "Comfy Cloud's storing task failed or was cancelled (the message carries the task's error, e.g. not a " "valid .safetensors file).", - "retry for a storage failure; for an unreadable file or a task failure fix the file the message names; " + "retry for a storage failure; for upload_paused run the same command again to continue only when " + "session_saved is true and the session remains valid; otherwise start a new upload; " + "for an unreadable file or a task failure fix the file the message names; " "for tls_setup fix the certificate setting the message names; for file_changed wait until nothing " "writes the file, then run the dry run again", ), ErrorCode( "asset_upload_timeout", - "The model file was uploaded, but Comfy Cloud had not finished storing it when " + "Retired: current versions wait until storing ends. Older versions raised this when " + "the model file was uploaded, but Comfy Cloud had not finished storing it when " "`model upload` or `assets library upload` stopped waiting. `details.task_id` is the storing task, " "which goes on.", "check `comfy assets library ls --tags models` later instead of uploading again", diff --git a/comfy_cli/schemas/build_push_event.json b/comfy_cli/schemas/build_push_event.json index 031341494..56bd85dd7 100644 --- a/comfy_cli/schemas/build_push_event.json +++ b/comfy_cli/schemas/build_push_event.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://comfy.org/schemas/build_push_event.json", "title": "comfy build push upload progress event", - "description": "One JSON object per line while `comfy build push` uploads model files and packaged custom nodes, or `comfy model upload` / `comfy assets library upload` uploads one model file. Under `--json-stream` the lines are on stdout, ahead of the final envelope (build_push.json schema in .data; model_upload.json for `model upload`, assets_library.json for `assets library upload`). Under `--json` they are on stderr, so stdout stays the single envelope. A push opens with one `upload_plan`; each file that transfers gets an `upload_progress` when it starts and about every two seconds after, then one `upload_complete`. A file the builder already held gets only an `upload_complete` with `deduplicated: true`. A file whose upload fails gets no `upload_complete`: the error envelope follows.", + "description": "One JSON object per line while `comfy build push` uploads model files and packaged custom nodes, or `comfy model upload` / `comfy assets library upload` uploads one model file. Under `--json-stream` the lines are on stdout, ahead of the final envelope (build_push.json schema in .data; model_upload.json for `model upload`, assets_library.json for `assets library upload`). Under `--json` they are on stderr, so stdout stays the single envelope. A push opens with one `upload_plan`; each file that transfers gets an `upload_progress` when it starts and about every two seconds after, then one `upload_complete`. A file the builder already held gets only an `upload_complete` with `deduplicated: true`. A file whose upload fails gets no `upload_complete`: the error envelope follows. Model uploads then emit one `upload_storing` with `task_id` before waiting for the storage task.", "type": "object", "required": ["schema", "type"], "additionalProperties": true, @@ -13,8 +13,9 @@ }, "type": { "type": "string", - "description": "Open-ended: this comfy-cli version emits upload_plan, upload_progress and upload_complete (the shapes below), and adding a type is an additive change that does NOT bump `event/1`. Ignore types you do not recognise rather than treat them as a validation failure." + "description": "Open-ended: this comfy-cli version emits upload_plan, upload_progress, upload_complete and upload_storing (the shapes below), and adding a type is an additive change that does NOT bump `event/1`. Ignore types you do not recognise rather than treat them as a validation failure." }, + "task_id": {"type": "string", "minLength": 1, "description": "upload_storing: the model storage task to look up if the wait is interrupted."}, "files": {"type": "integer", "minimum": 0, "description": "upload_plan: files this push still has to send."}, "already_held": {"type": "integer", "minimum": 0, "description": "upload_plan: local models and nodes that need no upload, because an earlier push stored them or a public source link stands in for the bytes."}, "file": {"type": "string", "description": "The model's filename, or `.zip` for a packaged custom node."}, @@ -32,6 +33,10 @@ "deduplicated": {"type": "boolean", "description": "upload_complete: true when the builder already held these bytes and nothing was sent."} }, "allOf": [ + { + "if": {"required": ["type"], "properties": {"type": {"const": "upload_storing"}}}, + "then": {"required": ["task_id"]} + }, { "if": {"required": ["type"], "properties": {"type": {"const": "upload_plan"}}}, "then": {"required": ["files", "bytes_total", "already_held"]} diff --git a/comfy_cli/schemas/error_codes.md b/comfy_cli/schemas/error_codes.md index abe22e53c..ee3cedac7 100644 --- a/comfy_cli/schemas/error_codes.md +++ b/comfy_cli/schemas/error_codes.md @@ -27,7 +27,10 @@ comfy --json discover | jq '.data.error_codes[]' Three tests pin the contract: - Every `renderer.error(code="…")` site is registered. -- Every registered code is raised somewhere (no dead codes). +- Every registered code is raised somewhere or explicitly retired in the + registry test. Retired codes stay discoverable for compatibility and must + no longer be emitted. `asset_upload_timeout` is retired because model + uploads now wait until storing ends instead of stopping after fifteen minutes. - A code emitted only as a warning (an object in a `warnings` list of a successful result) is not registered. Registered codes are failure categories; a warning keeps its `code` string in `warnings[]`. Warning-only diff --git a/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md new file mode 100644 index 000000000..d2a5fad38 --- /dev/null +++ b/docs/adr/TRANSFER-0012-resume-model-uploads-only-for-the-same-file-and-account.md @@ -0,0 +1,98 @@ +# ADR-TRANSFER-0012: Resume Model Uploads Only for the Same File and Account + +Date: 2026-11-06 + +## Status + +Proposed + +Relation: Builds on [ADR-TRANSFER-0011](TRANSFER-0011-upload-only-the-named-model-and-prove-cloud-stored-its-bytes.md). +Approval, name-reuse checks, checking each hashing block before sending it, and +proving the stored hash all remain in effect. The success envelope is unchanged. + +## Context + +A user uploads a large checkpoint and their connection drops. Sending one PUT +again loses the bytes already sent. A storage session can continue that upload, +but its address is a credential and must not follow a later sign-in or changed +file into another upload. + +## Decision + +Both model upload commands request resumable uploads. A response without an +upload mode keeps the existing single-PUT path. The server decides the size +limit; the CLI does not keep a separate upper limit. + +The CLI owns saved sessions in a private directory under its configuration +folder. An atomic replacement writes each session with user-only permissions. +At most 32 session records and 32 interrupted-save temporary files remain. +Sessions expire at the earlier of the server's expiry or seven days after +creation. Oversized, malformed, expired and insecure records are never reused. +On Windows, file access follows the user's configuration-directory ACL; POSIX +also checks ownership and mode 0600. + +Reuse requires authenticated user and workspace ids from Cloud, the same server, +resolved path, folder, hashed size, modified time and freshly computed content +hash. Unknown identity disables persistence. A failure to save is reported and +still permits retries within that process. + +Storage's status query decides where to resume. Each piece is at most 64 MiB. +A resumed offset inside a hashing block still checks that whole block before +sending any part of it. Only acknowledged bytes advance progress. Transport +errors keep no exception text because it can contain the session credential. +Expired sessions and permanent storage refusals discard the saved record and +restart once; thirty minutes without acknowledged progress pauses. Certificate +verification and certificate-bundle loading failures stop immediately with +credential-safe TLS guidance. +This includes certificate failures wrapped by an HTTPS proxy; temporary proxy +failures remain retryable. +The legacy single-PUT path uses the same certificate classifier and guidance; +transient TLS failures there retain the existing bounded fresh-upload retry. +HTTP 408 and transient TLS failures query held bytes and retry within the same +session. TLS setup diagnostics use fixed text, including for normalized URLs. +Completion deletes the session after a valid task response, and the command +waits for the task without a total deadline. Repeated unanswered task polls +still have a bounded retry window. The task id is emitted before waiting, so +a caller can look it up if the command stops. HTTPS with a host and no userinfo +is required for resumable and saved sessions; legacy single-PUT addresses retain +the existing validation for compatibility with storage emulators. +Progress output failures, including Rich converting broken pipes to process-exit +exceptions, mute reporting without stopping task polling or the stored-hash check. +Later storage-wait notices use the same reporting boundary. +The shared reporter also keeps `comfy build push` running after a broken pipe +or closed progress stream; its upload protocol is unchanged. + +## Alternatives considered + +- Keep one PUT and raise only the size limit. A drop still restarts the file. +- Reuse state by path alone. This could send bytes for a different file, + account, workspace or folder. +- Trust size and modified time alone. Files can change without either changing. +- Save session addresses in ordinary config or output. That exposes credentials. + +## Consequences + +Reruns hash the whole file again. Storage receives only newly needed pieces, +and no temporary copy of a multi-GB model is created. State is local to one +machine; simultaneous invocations are not coordinated and may start separate +uploads. Each new upload has a separate record, so completing one does not +delete another upload's saved state. Losing saved state loses the ability to +resume that session, but a new upload still works. A task may continue after +the command loses its sign-in. + +## Validation + +The model-upload tests use a fake bucket for mid-piece drops, process reruns, +final-piece recovery, refusal, expiry, transient errors and byte verification. +State tests cover each identity field, expiration, permissions and bounds. +Real Requests and OpenSSL check invalid local certificate bundles. Real Rich +rendering checks closed streams and broken pipes during storage notices and +polling. Tests isolate Rich's process-wide stdout redirection. Existing command +tests retain approval and stored-hash assertions. CI runs +these tests on Linux, macOS and Windows. Real multi-GB storage is not exercised +by the fake bucket. + +## Revisit triggers + +Revisit when storage changes its resumable protocol or lifetime, when users +need to resume across machines, or when concurrent uploads must share a session. diff --git a/docs/json-output.md b/docs/json-output.md index 0c9c8ddc2..3021c2ed0 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -487,7 +487,7 @@ time. If the callback never arrives, the terminal envelope is an {"schema": "event/1", "type": "login_url", "url": "https://api.comfy.org/oauth/authorize?...", "timeout_s": 300} ``` -### `upload_plan`, `upload_progress`, `upload_complete` +### `upload_plan`, `upload_progress`, `upload_complete`, `upload_storing` Emitted by `comfy build push` while it uploads model files and packaged custom nodes, and by `comfy model upload` and `comfy assets library upload` while they @@ -513,6 +513,19 @@ when there is nothing to upload. Each file that transfers gets an {"schema": "event/1", "type": "upload_complete", "file": "model.safetensors", "kind": "model", "index": 1, "of": 3, "bytes_total": 23761671782, "seconds": 498.2, "bytes_per_second": 47695045, "deduplicated": false, "overall_bytes_done": 23761671782, "overall_bytes_total": 56908316672} ``` +After transferring the bytes, `comfy model upload` and `comfy assets library +upload` emit `upload_storing` as soon as Cloud returns a storage task id. It +arrives before task polling and follows the same stdout/stderr rules above. +If the wait is interrupted, look up that id at Cloud's +`GET /api/tasks/` endpoint. +The final success envelope still requires a completed task and a matching +stored hash. A closed progress pipe mutes progress reporting; task polling and +the stored-hash check still run. + +```json +{"schema": "event/1", "type": "upload_storing", "task_id": "task-1"} +``` + `bytes_per_second` on a progress event is measured over the last ten seconds and is sampled on a timer, not on bytes moving, so a stalled connection keeps reporting and its rate falls to `0` with `eta_seconds: null`. That is how to diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index b9ae2bbd4..8d63f60ce 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -44,7 +44,12 @@ @pytest.fixture(autouse=True) -def reset_singleton(): +def reset_singleton(monkeypatch, tmp_path): + from comfy_cli import constants + + monkeypatch.setattr( + constants, "DEFAULT_CONFIG", {key: str(tmp_path / "config") for key in constants.DEFAULT_CONFIG} + ) reset_renderer_for_testing() yield reset_renderer_for_testing() @@ -969,7 +974,12 @@ def test_uploads_completes_and_reports_the_stored_asset( "/api/tasks/task-1", "/api/assets/asset-9", ] - assert cloud.calls[2][2] == {"filename": "style.safetensors", "size_bytes": 16, "tags": ["models", "loras"]} + assert cloud.calls[2][2] == { + "filename": "style.safetensors", + "size_bytes": 16, + "tags": ["models", "loras"], + "resumable": True, + } assert cloud.calls[3][2] == {"filename": "style.safetensors", "tags": ["models", "loras"]} (put,) = storage.puts assert put["url"] == f"{_SIGNED}1" and put["headers"] == _PUT_HEADERS and put["allow_redirects"] is False @@ -986,7 +996,6 @@ def test_uploads_completes_and_reports_the_stored_asset( ("x" * 250 + ".safetensors", None, "bad_filename"), ("style.safetensors", None, "file_not_found"), ("style.safetensors", 0, "empty_file"), - ("style.safetensors", 2 * 1024**3 + 1, "file_too_large"), ], ) def test_a_file_comfy_cloud_would_refuse_is_refused_before_any_request( @@ -995,7 +1004,7 @@ def test_a_file_comfy_cloud_would_refuse_is_refused_before_any_request( path = tmp_path / name if size is not None: with path.open("wb") as f: - f.truncate(size) # sparse: no 2 GB written to disk + f.truncate(size) env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, [str(path), "--type", "loras"], {}, []) @@ -1003,10 +1012,20 @@ def test_a_file_comfy_cloud_would_refuse_is_refused_before_any_request( assert env["error"]["details"]["reason"] == reason assert cloud.calls == [] and storage.puts == [] - def test_exactly_2_gib_is_accepted_locally(self, tmp_path): + @pytest.mark.parametrize("size", [2 * 1024**3 + 1, 25 * 1024**3, 25 * 1024**3 + 1]) + def test_size_limit_is_left_to_the_server(self, tmp_path, monkeypatch, size): path = tmp_path / "big.SAFETENSORS" - with path.open("wb") as f: - f.truncate(2 * 1024**3) + path.write_bytes(b"x") + real_stat = Path.stat + + def stat_with_large_size(candidate, *args, **kwargs): + info = real_stat(candidate, *args, **kwargs) + if candidate == path: + return os.stat_result((*info[:6], size, *info[7:])) + return info + + # Extending a file can allocate its full size on Windows. + monkeypatch.setattr(Path, "stat", stat_with_large_size) assert assets_library._local_refusal(path) is None def test_unknown_type_is_refused_listing_the_valid_ones_before_the_upload( @@ -1137,13 +1156,13 @@ def test_a_task_4xx_ends_the_wait_at_once(self, cloud_target, monkeypatch, capsy assert rc == 1 and cloud.paths().count("/api/tasks/task-1") == 1 - def test_a_task_that_never_ends_times_out(self, cloud_target, monkeypatch, capsys, model_file, no_wait): + def test_a_task_can_take_longer_than_fifteen_minutes(self, cloud_target, monkeypatch, capsys, model_file, no_wait): routes = dict(_UPLOAD_ROUTES) - routes[("GET", "/api/tasks/task-1")] = {"id": "task-1", "status": "running"} + routes[("GET", "/api/tasks/task-1")] = [{"id": "task-1", "status": "running"}] * 500 + [_DONE] env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) - assert rc == 1 and env["error"]["code"] == "asset_upload_timeout" - assert env["error"]["details"] == {"task_id": "task-1", "status": "running"} + assert rc == 0 and env["data"]["id"] == "asset-9" + assert no_wait[0] > 15 * 60 @pytest.mark.parametrize( ("route", "answer"), @@ -1737,8 +1756,9 @@ def __call__(self, url, data=None, headers=None, **kw): assert "Transfer-Encoding" not in storage.prepared assert storage.puts[0]["body"] == model_file.read_bytes() + @pytest.mark.parametrize("resumable", [False, True], ids=["single-put", "resumable"]) def test_a_missing_ca_bundle_ends_the_upload_with_a_tls_setup_error( - self, cloud_target, monkeypatch, model_file, tmp_path, no_wait + self, cloud_target, monkeypatch, model_file, tmp_path, no_wait, resumable ): # The real Requests adapter: with REQUESTS_CA_BUNDLE naming a removed # file, it raises a plain OSError while setting up TLS, before any @@ -1759,7 +1779,7 @@ def __call__(self, url, **kw): return real_put(url, **kw) monkeypatch.setenv("REQUESTS_CA_BUNDLE", str(tmp_path / "removed-ca-bundle.pem")) - cloud, storage = _Cloud(dict(_UPLOAD_ROUTES)), _RealPut([]) + cloud, storage = _Cloud(_resume_routes() if resumable else dict(_UPLOAD_ROUTES)), _RealPut([]) monkeypatch.setattr("urllib.request.urlopen", cloud) monkeypatch.setattr("requests.put", storage) result = CliRunner().invoke( @@ -2348,3 +2368,735 @@ def test_it_is_registered_under_the_model_noun_with_a_schema(self): assert "upload" in {c.name for c in models.app.registered_commands} assert COMMAND_SCHEMAS["comfy model upload"] == "model_upload" assert STREAM_EVENT_SCHEMAS["comfy model upload"] == "build_push_event" + + +# --- Resumable uploads through both public commands ---------------------------- + + +def _resumable_started(n=1): + import time + from datetime import datetime, timezone + + return { + **_started(n), + "upload_mode": "resumable", + "chunk_size_bytes": 64 * 1024**2, + "expires_at": datetime.fromtimestamp(time.time() + 7 * 24 * 3600, timezone.utc).isoformat(), + } + + +def _resume_routes(): + return { + **_UPLOAD_ROUTES, + ("POST", "/api/assets/uploads"): [_resumable_started(1), _resumable_started(2)], + ("GET", "/api/user"): {"id": "user-1"}, + ("GET", "/api/workspaces/current"): {"id": "ws-1"}, + } + + +class _ResumableBucket: + def __init__(self, *, crash=False, held=b"", status=None): + self.crash, self.held, self.status = crash, held, status + self.puts = [] + + def __call__(self, url, *, data, headers, **kw): + self.puts.append({"url": url, "headers": headers, "body": data, **kw}) + assert kw["allow_redirects"] is False + if self.crash: + self.held = data[:5] + self.crash = False + raise KeyboardInterrupt() + if self.status is not None: + return SimpleNamespace(status_code=self.status, headers={}, close=lambda: None) + if not headers["Content-Range"].startswith("bytes */"): + offset = int(headers["Content-Range"].split()[1].split("-")[0]) + assert offset == len(self.held) + self.held += data + code = self.status or (200 if len(self.held) == 16 else 308) + return SimpleNamespace( + status_code=code, + headers={"Range": f"bytes=0-{len(self.held) - 1}"} if self.held else {}, + close=lambda: None, + ) + + +class TestResumableCommands: + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + def test_killed_upload_resumes_the_same_session_on_rerun( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model + ): + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + ( + ["--yes", "--workspace", "ws-1", "--expected-sha256", _MODEL_SHA256] if model else [] + ) + bucket = _ResumableBucket(crash=True) + # CliRunner converts KeyboardInterrupt to an exit; the session has already been saved. + monkeypatch.setattr("requests.put", bucket) + cloud = _Cloud(_resume_routes()) + monkeypatch.setattr("urllib.request.urlopen", cloud) + from comfy_cli.command.models import models + + result = CliRunner().invoke(models.app if model else assets_library.app, ["upload", *args, "--where", "cloud"]) + assert result.exit_code != 0 + assert bucket.held == _MODEL_BYTES[:5] + capsys.readouterr() + bucket.puts.clear() + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=bucket) + assert rc == 0 and env["data"]["uploaded"] is True + assert "/api/assets/uploads" not in cloud.paths() + assert bucket.puts[0]["headers"]["Content-Range"] == "bytes */16" + assert bucket.puts[1]["headers"]["Content-Range"] == "bytes 5-15/16" + assert bucket.held == _MODEL_BYTES + assert "SECRETSIG" not in everything + from comfy_cli.config_manager import ConfigManager + + assert list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) == [] + + def test_crash_after_final_piece_completes_without_resending( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _cloud_error(403, "NO", "try again") + bucket = _ResumableBucket() + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=bucket) + assert rc == 1 and bucket.held == _MODEL_BYTES + bucket.puts.clear() + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], _resume_routes(), [], storage=bucket + ) + assert rc == 0 and "/api/assets/uploads" not in cloud.paths() + assert [p["headers"]["Content-Range"] for p in bucket.puts] == ["bytes */16"] + + @pytest.mark.parametrize("expired_at", ["bucket", "complete"]) + def test_expired_session_starts_fresh(self, cloud_target, monkeypatch, capsys, model_file, no_wait, expired_at): + routes = _resume_routes() + if expired_at == "complete": + routes[("POST", _COMPLETE_1)] = _http_error(404) + bucket = _ResumableBucket(status=404 if expired_at == "bucket" else None) + original = bucket.__call__ + + def put(url, **kw): + if url.endswith("2"): + bucket.status = None + bucket.held = b"" + return original(url, **kw) + + monkeypatch.setattr("requests.put", put) + # _upload accepts a callable in place of its storage fake. + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=put + ) + assert rc == 0 and cloud.paths("POST").count("/api/assets/uploads") == 2 + + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + @pytest.mark.parametrize("status", [400, 403, 422]) + @pytest.mark.parametrize("saved", [False, True], ids=["fresh", "saved"]) + def test_refused_session_is_discarded_and_restarted_once( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model, status, saved + ): + from comfy_cli.config_manager import ConfigManager + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + directory = Path(ConfigManager().get_config_path()) / "model-uploads" + if saved: + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + env, rc, *_ = run(monkeypatch, capsys, args, routes, [], storage=_ResumableBucket()) + assert rc == 1 and len(list(directory.glob("*.json"))) == 1 + bucket = _ResumableBucket() + refused = [] + + def put(url, **kw): + if url.endswith("1"): + refused.append(kw["headers"]["Content-Range"]) + return SimpleNamespace(status_code=status, headers={}, close=lambda: None) + return bucket(url, **kw) + + routes = _resume_routes() + if saved: + routes[("POST", "/api/assets/uploads")] = _resumable_started(2) + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, routes, [], storage=put) + assert rc == 0 and env["data"]["uploaded"] is True + assert cloud.paths("POST").count("/api/assets/uploads") == (1 if saved else 2) + assert refused == ["bytes */16" if saved else "bytes 0-15/16"] + assert bucket.held == _MODEL_BYTES + assert not list(directory.glob("*.json")) + assert "SECRETSIG" not in everything + + def test_repeated_storage_refusal_is_bounded_and_not_saved_for_the_next_run( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + from comfy_cli.config_manager import ConfigManager + + def refused(*args, **kw): + return SimpleNamespace(status_code=403, headers={}, close=lambda: None) + + for _ in range(2): + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], _resume_routes(), [], storage=refused + ) + assert rc == 1 and env["error"]["details"]["status"] == 403 + assert cloud.paths("POST").count("/api/assets/uploads") == 2 + assert not list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) + + @pytest.mark.parametrize("change", ["workspace", "account", "same-size-bytes", "folder", "mtime", "size", "server"]) + def test_rerun_with_a_changed_file_or_identity_starts_fresh( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, change + ): + from comfy_cli.config_manager import ConfigManager + + bucket = _ResumableBucket() + args = [str(model_file), "--type", "loras"] + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + env, rc, *_ = _upload(monkeypatch, capsys, args, routes, [], storage=bucket) + assert rc == 1 + assert len(list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json"))) == 1 + bucket.status = 403 + routes = _resume_routes() + if change == "workspace": + routes[("GET", "/api/workspaces/current")] = {"id": "ws-2"} + elif change == "account": + routes[("GET", "/api/user")] = {"id": "user-2"} + elif change == "same-size-bytes": + info = model_file.stat() + model_file.write_bytes(b"X" + _MODEL_BYTES[1:]) + os.utime(model_file, ns=(info.st_atime_ns, info.st_mtime_ns)) + elif change == "mtime": + info = model_file.stat() + os.utime(model_file, ns=(info.st_atime_ns, info.st_mtime_ns + 1000000000)) + elif change == "size": + model_file.write_bytes(_MODEL_BYTES + b"x") + elif change == "folder": + args[-1] = "checkpoints" + else: + from dataclasses import replace + + monkeypatch.setattr( + "comfy_cli.target.resolve_target", lambda **kw: replace(cloud_target, base_url="https://other.example") + ) + # A refusal at storage is enough to see whether start was called, regardless of stored hash. + bucket.held = b"" + env, rc, cloud, _, _ = _upload(monkeypatch, capsys, args, routes, [], storage=bucket) + assert rc == 1 and "/api/assets/uploads" in cloud.paths() + assert not bucket.puts[-1]["headers"]["Content-Range"].startswith("bytes */") + + def test_resumable_still_checks_changed_blocks_and_stored_hash( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + routes = _resume_routes() + routes[("POST", "/api/assets/uploads")] = _then( + lambda: model_file.write_bytes(b"X" + _MODEL_BYTES[1:]), _resumable_started() + ) + bucket = _ResumableBucket() + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=bucket + ) + assert rc == 1 and env["error"]["details"]["reason"] == "file_changed" + assert bucket.puts == [] and _COMPLETE_1 not in cloud.paths() + model_file.write_bytes(_MODEL_BYTES) + routes = _resume_routes() + routes[("GET", "/api/tasks/task-1")] = {**_DONE, "result": {**_DONE["result"], "hash": "b" * 64}} + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=bucket) + assert rc == 1 and env["error"]["code"] == "asset_upload_unverified" + + def test_complete_retries_transient_errors(self, cloud_target, monkeypatch, capsys, model_file, no_wait): + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = [_http_error(429), _http_error(503), {"task_id": "task-1"}] + env, rc, cloud, _, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=_ResumableBucket() + ) + assert rc == 0 and cloud.paths().count(_COMPLETE_1) == 3 + + +class TestResumableRefusals: + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + def test_legacy_single_put_keeps_http_storage_compatibility( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model + ): + routes = dict(_UPLOAD_ROUTES) + address = "http://127.0.0.1:9023/bucket/model?signature=SECRETSIG" + routes[("POST", "/api/assets/uploads")] = {**_started(), "upload_url": address} + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, _, storage, everything = run(monkeypatch, capsys, args, routes, [200]) + assert rc == 0 and env["data"]["uploaded"] is True + assert storage.puts[0]["url"] == address + assert storage.puts[0]["body"] == _MODEL_BYTES + assert "SECRETSIG" not in everything + + @pytest.mark.parametrize("quote", ["url", "query"]) + @pytest.mark.parametrize("saved", [False, True], ids=["piece", "status-query"]) + def test_certificate_failure_stops_at_once_without_session_credentials( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, quote, saved + ): + from urllib.parse import urlsplit + + import requests + + args = [str(model_file), "--type", "loras"] + if saved: + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + env, rc, *_ = _upload(monkeypatch, capsys, args, routes, [], storage=_ResumableBucket()) + assert rc == 1 + attempts = [] + + def failed(url, **kw): + attempts.append(kw["headers"]["Content-Range"]) + address = url if quote == "url" else urlsplit(url).query + raise requests.exceptions.SSLError(f"[SSL: CERTIFICATE_VERIFY_FAILED] {address}") + + env, rc, cloud, _, everything = _upload(monkeypatch, capsys, args, _resume_routes(), [], storage=failed) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert attempts == ["bytes */16" if saved else "bytes 0-15/16"] + assert no_wait[0] == 0 + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything + + @pytest.mark.parametrize("url", [1, True, {}, []]) + @pytest.mark.parametrize("resumable", [False, True]) + def test_invalid_start_url_returns_the_cloud_error( + self, cloud_target, monkeypatch, capsys, model_file, url, resumable + ): + routes = _resume_routes() + start = _resumable_started() if resumable else _started() + routes[("POST", "/api/assets/uploads")] = {**start, "upload_url": url} + env, rc, _, storage, everything = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, []) + assert rc == 1 and env["error"]["code"] == "cloud_http_error" + assert storage.puts == [] + assert "SECRETSIG" not in everything + + @pytest.mark.parametrize("quote", ["url", "query"]) + def test_tls_setup_error_never_displays_session_credentials( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, quote + ): + from urllib.parse import urlsplit + + attempts = [] + + def failed(url, **kw): + attempts.append(url) + raise OSError(url if quote == "url" else urlsplit(url).query) + + env, rc, cloud, _, everything = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], _resume_routes(), [], storage=failed + ) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert len(attempts) == 1 + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything + + def test_server_size_refusal_prints_its_message_before_sending(self, cloud_target, monkeypatch, capsys, model_file): + routes = _resume_routes() + routes[("POST", "/api/assets/uploads")] = _cloud_error(400, "FILE_TOO_LARGE", "Model files can be up to 25 GB.") + env, rc, _, storage, _ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, []) + assert rc == 1 and env["error"]["message"] == "Model files can be up to 25 GB." + assert storage.puts == [] + + @pytest.mark.parametrize("endpoint", ["user", "workspaces/current"]) + def test_unknown_identity_does_not_save_a_session( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, endpoint + ): + routes = _resume_routes() + routes[("GET", "/api/" + endpoint)] = {} + bucket = _ResumableBucket() + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=bucket) + assert rc == 0 + from comfy_cli.config_manager import ConfigManager + + assert list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) == [] + + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + def test_paused_upload_redacts_the_credential_and_keeps_the_state( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model + ): + import requests + + def lost(*a, **kw): + raise requests.ConnectionError("SECRETSIG") + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, _, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=lost) + assert rc == 1 and env["error"]["details"]["reason"] == "upload_paused" + assert env["error"]["details"]["session_saved"] is True + assert "SECRETSIG" not in everything + assert "Run the same command again" in env["error"]["message"] + from comfy_cli.config_manager import ConfigManager + + assert len(list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json"))) == 1 + + env, rc, cloud, _, _ = run(monkeypatch, capsys, args, _resume_routes(), [], storage=lost) + assert rc == 1 and env["error"]["details"]["session_saved"] is True + assert "/api/assets/uploads" not in cloud.paths() + + @pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) + @pytest.mark.parametrize("unsaved", ["user", "workspaces/current", "write-failed"]) + def test_paused_upload_without_saved_state_explains_it_must_start_again( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, model, unsaved + ): + import requests + + from comfy_cli.command.model_upload_resume import SessionStore + from comfy_cli.config_manager import ConfigManager + + def lost(*a, **kw): + raise requests.ConnectionError("SECRETSIG") + + def cannot_save(self): + raise OSError("permission denied") + + routes = _resume_routes() + if unsaved == "write-failed": + monkeypatch.setattr(SessionStore, "_secure", cannot_save) + else: + routes[("GET", "/api/" + unsaved)] = {} + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, routes, [], storage=lost) + assert rc == 1 and env["error"]["details"]["reason"] == "upload_paused" + assert env["error"]["details"]["session_saved"] is False + assert "not saved" in env["error"]["message"] + assert "start a new upload" in env["error"]["message"] + assert "to continue" not in env["error"]["message"] + assert "SECRETSIG" not in everything + assert _COMPLETE_1 not in cloud.paths() + assert list((Path(ConfigManager().get_config_path()) / "model-uploads").glob("*.json")) == [] + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("mode", [OutputMode.JSON, OutputMode.NDJSON]) +def test_storing_task_is_reported_before_waiting(cloud_target, monkeypatch, model_file, model, mode): + import jsonschema + + from comfy_cli.command.models import models + + monkeypatch.setattr("urllib.request.urlopen", _Cloud(_resume_routes())) + monkeypatch.setattr("requests.put", _ResumableBucket()) + + def interrupted(*args, **kw): + raise KeyboardInterrupt() + + monkeypatch.setattr(assets_library, "_wait_for_task", interrupted) + set_renderer(Renderer(mode=mode)) + result = CliRunner().invoke( + models.app if model else assets_library.app, + ["upload", str(model_file), "--type", "loras", "--where", "cloud", *(["--yes"] if model else [])], + ) + assert result.exit_code != 0 + stream = result.stdout if mode is OutputMode.NDJSON else result.stderr + events = [json.loads(line) for line in stream.splitlines() if line.startswith("{")] + storing = [event for event in events if event["type"] == "upload_storing"] + assert storing == [{"schema": "event/1", "type": "upload_storing", "task_id": "task-1"}] + schema = json.loads((Path(assets_library.__file__).parents[1] / "schemas" / "build_push_event.json").read_text()) + jsonschema.validate(storing[0], schema) + if mode is OutputMode.JSON: + assert result.stdout == "" + assert "SECRETSIG" not in result.output + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("mismatch", [False, True], ids=["verified", "wrong-hash"]) +@pytest.mark.parametrize("mode", [OutputMode.JSON, OutputMode.NDJSON, OutputMode.PRETTY]) +def test_closed_progress_pipe_still_polls_and_verifies(cloud_target, monkeypatch, model_file, model, mismatch, mode): + from comfy_cli.command.models import models + + routes = _resume_routes() + if mismatch: + routes[("GET", "/api/tasks/task-1")] = {**_DONE, "result": {**_DONE["result"], "hash": "b" * 64}} + cloud = _Cloud(routes) + monkeypatch.setattr("urllib.request.urlopen", cloud) + monkeypatch.setattr("requests.put", _ResumableBucket()) + refused = [] + original_event = Renderer.progress_event + original_info = Renderer.info + + def event(self, type, **fields): + if type != "upload_plan": + refused.append(type) + raise BrokenPipeError() + return original_event(self, type, **fields) + + def info(self, message, **kw): + if "Task id:" in message: + refused.append("storing") + raise BrokenPipeError() + return original_info(self, message, **kw) + + monkeypatch.setattr(Renderer, "progress_event", event) + monkeypatch.setattr(Renderer, "info", info) + set_renderer(Renderer(mode=mode)) + result = CliRunner().invoke( + models.app if model else assets_library.app, + ["upload", str(model_file), "--type", "loras", "--where", "cloud", *(["--yes"] if model else [])], + ) + assert "/api/tasks/task-1" in cloud.paths(), repr(result.exception) + assert result.exit_code == (1 if mismatch else 0), repr(result.exception) + assert refused + if mode is not OutputMode.PRETTY: + env = json.loads(result.stdout.strip().splitlines()[-1]) + assert env["ok"] is (not mismatch) + if mismatch: + assert env["error"]["code"] == "asset_upload_unverified" + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("failure", ["tls-eof", "proxy-tls-eof", "proxy-connect", "408"]) +@pytest.mark.parametrize("saved", [False, True], ids=["piece", "status-query"]) +def test_transient_storage_failure_keeps_session_and_queries_offset( + cloud_target, monkeypatch, capsys, model_file, no_wait, model, failure, saved +): + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + bucket = _ResumableBucket() + if saved: + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + env, rc, *_ = run(monkeypatch, capsys, args, routes, [], storage=bucket) + assert rc == 1 + calls = [] + + def put(url, **kw): + calls.append((url, kw["headers"]["Content-Range"])) + if len(calls) == 1: + if failure == "tls-eof": + raise requests.exceptions.SSLError(MaxRetryError(None, url, SSLError(ssl.SSLEOFError("TLS EOF")))) + if failure.startswith("proxy-"): + reason = ssl.SSLEOFError("TLS EOF") if failure == "proxy-tls-eof" else OSError("proxy unavailable") + raise requests.exceptions.ProxyError(MaxRetryError(None, url, ProxyError("proxy failed", reason))) + return SimpleNamespace(status_code=408, headers={}, close=lambda: None) + if len(calls) == 2: + assert no_wait[0] == 1 + return bucket(url, **kw) + + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=put) + assert rc == 0 and env["ok"] is True + assert calls[1][1] == "bytes */16" + assert len({url for url, _ in calls}) == 1 + assert cloud.paths("POST").count("/api/assets/uploads") == (0 if saved else 1) + assert bucket.held == _MODEL_BYTES + assert "SECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("failure", ["certificate", "proxy-certificate", "setup"]) +def test_normalized_tls_error_never_exposes_session_credentials( + cloud_target, monkeypatch, capsys, model_file, no_wait, model, failure +): + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError + + routes = _resume_routes() + start = {**_resumable_started(), "upload_url": "https://bucket.example/?upload_id=%53ECRETSIG"} + routes[("POST", "/api/assets/uploads")] = start + attempts = [] + + def put(url, **kw): + prepared = requests.Request("PUT", url).prepare().url + attempts.append(prepared) + assert "upload_id=SECRETSIG" in prepared + if failure in ("certificate", "proxy-certificate"): + reason = ssl.SSLCertVerificationError("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed") + if failure == "proxy-certificate": + raise requests.exceptions.ProxyError(MaxRetryError(None, prepared, ProxyError("proxy failed", reason))) + raise requests.exceptions.SSLError(MaxRetryError(None, prepared, SSLError(reason))) + raise OSError(f"TLS setup failed for {prepared}") + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, routes, [], storage=put) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert len(attempts) == 1 and no_wait[0] == 0 + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything + assert "%53ECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("saved", [False, True], ids=["piece", "status-query"]) +@pytest.mark.parametrize("proxy", [False, True], ids=["direct", "proxy"]) +def test_invalid_ca_contents_stop_without_backoff( + cloud_target, monkeypatch, capsys, model_file, no_wait, tmp_path, model, saved, proxy +): + """Preserve a real OpenSSL loading error through the transport's wrappers.""" + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError + + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + if saved: + routes = _resume_routes() + routes[("POST", _COMPLETE_1)] = _http_error(403) + _, rc, *_ = run(monkeypatch, capsys, args, routes, [], storage=_ResumableBucket()) + assert rc == 1 + bundle = tmp_path / "invalid-ca.pem" + bundle.write_text("not a certificate\n") + with pytest.raises(ssl.SSLError) as native: + ssl.create_default_context(cafile=str(bundle)) + attempts = [] + + def put(url, **kw): + attempts.append(kw["headers"]["Content-Range"]) + if proxy: + raise requests.exceptions.ProxyError(MaxRetryError(None, url, ProxyError("proxy failed", native.value))) + raise requests.exceptions.SSLError(MaxRetryError(None, url, SSLError(native.value))) + + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, _resume_routes(), [], storage=put) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert len(attempts) == 1 and no_wait[0] == 0 + assert attempts[0].startswith("bytes */") is saved + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("failure", ["certificate", "bundle", "flattened"]) +@pytest.mark.parametrize("proxy", [False, True], ids=["direct", "proxy"]) +def test_legacy_tls_certificate_failure_stops_without_a_fresh_upload( + cloud_target, monkeypatch, capsys, model_file, no_wait, tmp_path, model, failure, proxy +): + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError + + if failure == "bundle": + bundle = tmp_path / "invalid-ca.pem" + bundle.write_text("not a certificate\n") + with pytest.raises(ssl.SSLError) as native: + ssl.create_default_context(cafile=str(bundle)) + reason = native.value + else: + reason = ssl.SSLCertVerificationError("certificate verify failed") + attempts = [] + + def put(url, **kw): + prepared = requests.Request("PUT", url).prepare().url + attempts.append(prepared) + assert "signature=SECRETSIG" in prepared + assert "Content-Range" not in kw["headers"] + wrapped = MaxRetryError(None, prepared, ProxyError("proxy failed", reason) if proxy else SSLError(reason)) + error_type = requests.exceptions.ProxyError if proxy else requests.exceptions.SSLError + raise error_type(str(wrapped) if failure == "flattened" else wrapped) + + routes = dict(_UPLOAD_ROUTES) + routes[("POST", "/api/assets/uploads")] = { + **_started(), + "upload_url": "https://bucket.example/?signature=%53ECRETSIG", + } + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, cloud, _, everything = run(monkeypatch, capsys, args, routes, [], storage=put) + assert rc == 1 and env["error"]["details"]["reason"] == "tls_setup" + assert "CA certificate bundle" in env["error"]["message"] + assert env["error"]["details"]["attempts"] == 1 + assert len(attempts) == 1 and no_wait[0] == 0 + assert cloud.paths("POST").count("/api/assets/uploads") == 1 + assert _COMPLETE_1 not in cloud.paths() + assert "SECRETSIG" not in everything and "%53ECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("failure_kind", ["tls-eof", "proxy-tls-eof", "proxy-connect"]) +def test_legacy_tls_eof_still_retries_with_a_fresh_upload( + cloud_target, monkeypatch, capsys, model_file, no_wait, model, failure_kind +): + import ssl + + import requests + from urllib3.exceptions import MaxRetryError, ProxyError, SSLError + + failure = requests.exceptions.SSLError(MaxRetryError(None, "SECRETSIG", SSLError(ssl.SSLEOFError("TLS EOF")))) + if failure_kind.startswith("proxy-"): + reason = ssl.SSLEOFError("TLS EOF") if failure_kind == "proxy-tls-eof" else OSError("proxy unavailable") + failure = requests.exceptions.ProxyError(MaxRetryError(None, "SECRETSIG", ProxyError("proxy failed", reason))) + run = _model_upload if model else _upload + args = [str(model_file), "--type", "loras"] + (["--yes"] if model else []) + env, rc, cloud, storage, everything = run(monkeypatch, capsys, args, dict(_UPLOAD_ROUTES), [failure, 200]) + assert rc == 0 and env["ok"] is True + assert len(storage.puts) == 2 + assert cloud.paths("POST").count("/api/assets/uploads") == 2 + assert "/api/tasks/task-1" in cloud.paths() + assert "SECRETSIG" not in everything + + +@pytest.mark.parametrize("model", [False, True], ids=["assets-library", "model"]) +@pytest.mark.parametrize("mismatch", [False, True], ids=["verified", "wrong-hash"]) +@pytest.mark.parametrize("failure", ["broken-pipe", "closed-stream"]) +@pytest.mark.parametrize("notice", ["storing", "polling"]) +def test_real_pretty_output_failure_preserves_polling_and_hash_check( + cloud_target, monkeypatch, model_file, no_wait, model, mismatch, failure, notice +): + """Run real Rich rendering; isolate only its process-wide stdout redirection.""" + import rich.console + + from comfy_cli.command.models import models + + # Keep Rich's real BrokenPipeError -> SystemExit conversion, but do not + # let on_broken_pipe redirect pytest's stdout file descriptor to /dev/null. + redirected = [] + + def redirect(fd, target): + redirected.append((fd, target)) + os.close(fd) + + monkeypatch.setattr(rich.console, "os", SimpleNamespace(**{**vars(os), "dup2": redirect})) + monkeypatch.setattr( + rich.console, "sys", SimpleNamespace(**{**vars(sys), "stdout": SimpleNamespace(fileno=lambda: 1)}) + ) + failed_writes = [] + + class Output(io.StringIO): + def write(self, text): + marker = "Task id:" if notice == "storing" else "still checking" + if failed_writes or marker in text: + failed_writes.append(text) + if failure == "broken-pipe": + raise BrokenPipeError() + raise ValueError("I/O operation on closed file.") + return super().write(text) + + renderer = Renderer(mode=OutputMode.PRETTY) + renderer.pretty_stream = Output() + routes = _resume_routes() + routes[("GET", "/api/tasks/task-1")] = [{"id": "task-1", "status": "running"}] * 40 + [ + {**_DONE, "result": {"success": True, "asset_id": "asset-9"}} + ] + verified = [] + original_stored = assets_library._stored_asset + routes[("GET", "/api/assets/asset-9")] = {"id": "asset-9", "hash": "b" * 64 if mismatch else _MODEL_HASH} + + def stored(*args): + verified.append(True) + # The final result has its own output boundary. Restore that output + # only once the task completed and its hash is about to be checked. + renderer.pretty_stream = io.StringIO() + return original_stored(*args) + + monkeypatch.setattr(assets_library, "_stored_asset", stored) + cloud = _Cloud(routes) + monkeypatch.setattr("urllib.request.urlopen", cloud) + monkeypatch.setattr("requests.put", _ResumableBucket()) + set_renderer(renderer) + result = CliRunner().invoke( + models.app if model else assets_library.app, + ["upload", str(model_file), "--type", "loras", "--where", "cloud", *(["--yes"] if model else [])], + ) + assert verified, repr(result.exception) + assert cloud.paths().count("/api/tasks/task-1") == 41 + assert result.exit_code == (1 if mismatch else 0), repr(result.exception) + assert len(failed_writes) == 1 + assert bool(redirected) is (failure == "broken-pipe") + if mismatch: + assert "asset_upload_unverified" in renderer.pretty_stream.getvalue() diff --git a/tests/comfy_cli/command/test_model_upload_resume.py b/tests/comfy_cli/command/test_model_upload_resume.py new file mode 100644 index 000000000..da7d57709 --- /dev/null +++ b/tests/comfy_cli/command/test_model_upload_resume.py @@ -0,0 +1,426 @@ +"""A fake bucket holds bytes across connection drops and process reruns.""" + +import json +import os +import time +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest +import requests + +from comfy_cli.command import assets_library as uploads +from comfy_cli.command import model_upload_resume as resume +from comfy_cli.hashing import file_digests + + +@pytest.fixture +def session(): + return { + "upload_id": "upload-1", + "upload_url": "https://bucket.example/upload?secret=SESSION", + "upload_mode": "resumable", + "method": "PUT", + "headers": {}, + "chunk_size_bytes": 256 * 1024, + "expires_at": datetime.fromtimestamp(time.time() + resume.LIFETIME, timezone.utc).isoformat(), + } + + +class Bucket: + def __init__(self, *, held=b"", drop=None, lose_final=False): + self.held, self.drop, self.lose_final = held, drop, lose_final + self.calls = [] + + def __call__(self, url, *, data, headers, timeout, allow_redirects): + assert allow_redirects is False + assert headers["Content-Length"] == str(len(data)) + content_range = headers["Content-Range"] + self.calls.append(content_range) + size = int(content_range.split("/")[1]) + if not content_range.startswith("bytes */"): + offset = int(content_range.split()[1].split("-")[0]) + assert offset == len(self.held) + if self.drop is not None: + self.held += data[: self.drop] + self.drop = None + raise requests.ConnectionError(url) + self.held += data + if self.lose_final and len(self.held) == size: + self.lose_final = False + raise requests.ConnectionError(url) + code = 200 if len(self.held) == size else 308 + response_headers = {"Range": f"bytes=0-{len(self.held) - 1}"} if self.held else {} + return SimpleNamespace(status_code=code, headers=response_headers, close=lambda: None) + + +@pytest.fixture +def file(tmp_path): + path = tmp_path / "checkpoint.safetensors" + path.write_bytes(bytes(range(256)) * 2500) + return path + + +@pytest.fixture +def clock(): + now = [0.0] + return lambda: now[0], lambda s: now.__setitem__(0, now[0] + s) + + +def send(session, file, clock, *, resumed=False): + digests = file_digests(file, block_size=4096) + progress = [] + resume.transfer( + session, + digests.size, + lambda o, n: uploads._read_range(file, digests, o, n), + progress.append, + resumed=resumed, + clock=clock[0], + sleep=clock[1], + ) + return progress + + +def test_a_drop_mid_piece_queries_the_held_range_before_sending(session, file, clock, monkeypatch): + bucket = Bucket(drop=12345) + monkeypatch.setattr(requests, "put", bucket) + assert sum(send(session, file, clock)) == file.stat().st_size + assert bucket.held == file.read_bytes() + assert bucket.calls[:3] == [ + f"bytes 0-262143/{file.stat().st_size}", + f"bytes */{file.stat().st_size}", + f"bytes 12345-274488/{file.stat().st_size}", + ] + + +def test_a_rerun_queries_before_sending_and_checks_the_partial_hash_block(session, file, clock, monkeypatch): + bucket = Bucket(held=file.read_bytes()[:12345]) + monkeypatch.setattr(requests, "put", bucket) + assert sum(send(session, file, clock, resumed=True)) == file.stat().st_size + assert bucket.held == file.read_bytes() + assert bucket.calls[0] == f"bytes */{file.stat().st_size}" + + +@pytest.mark.parametrize("rerun", [False, True]) +def test_last_piece_without_an_answer_goes_to_complete(session, file, clock, monkeypatch, rerun): + bucket = Bucket(held=file.read_bytes() if rerun else b"", lose_final=not rerun) + monkeypatch.setattr(requests, "put", bucket) + send(session, file, clock, resumed=rerun) + assert bucket.calls[-1] == f"bytes */{file.stat().st_size}" + if rerun: + assert len(bucket.calls) == 1 + + +@pytest.mark.parametrize("status", [408, 429, 500, 503, None]) +def test_transient_errors_back_off_and_query(session, file, clock, monkeypatch, status): + bucket = Bucket() + calls = [] + + def put(*args, **kw): + calls.append(kw["headers"]["Content-Range"]) + if len(calls) == 1: + if status is None: + raise requests.ConnectionError("SESSION") + return SimpleNamespace(status_code=status, headers={}, close=lambda: None) + return bucket(*args, **kw) + + monkeypatch.setattr(requests, "put", put) + send(session, file, clock) + assert calls[1].startswith("bytes */") + assert clock[0]() == 1 + + +@pytest.mark.parametrize( + "code,exception", + [ + (404, resume.SessionExpired), + (410, resume.SessionExpired), + (403, resume.TransferFailed), + (307, resume.TransferFailed), + ], +) +def test_expiry_or_refusal_never_follows_a_redirect(session, file, clock, monkeypatch, code, exception): + monkeypatch.setattr( + requests, "put", lambda *a, **k: SimpleNamespace(status_code=code, headers={}, close=lambda: None) + ) + with pytest.raises(exception): + send(session, file, clock) + + +def test_412_goes_to_complete(session, file, clock, monkeypatch): + monkeypatch.setattr( + requests, "put", lambda *a, **k: SimpleNamespace(status_code=412, headers={}, close=lambda: None) + ) + assert sum(send(session, file, clock)) == file.stat().st_size + + +@pytest.mark.parametrize("held", ["bad", "bytes=1-20", "bytes=0-9999999"]) +def test_bad_ranges_are_refused(session, file, clock, monkeypatch, held): + monkeypatch.setattr( + requests, "put", lambda *a, **k: SimpleNamespace(status_code=308, headers={"Range": held}, close=lambda: None) + ) + with pytest.raises(resume.TransferFailed): + send(session, file, clock) + + +def test_thirty_minutes_without_acknowledged_bytes_pauses_without_credentials(session, file, clock, monkeypatch): + def lost(*a, **kw): + raise requests.ConnectionError(session["upload_url"]) + + monkeypatch.setattr(requests, "put", lost) + with pytest.raises(resume.UploadPaused) as error: + send(session, file, clock) + assert error.value.args == (0,) + assert "SESSION" not in str(error.value) + assert clock[0]() == resume.STALL_SECONDS + + +def test_a_changed_partial_block_sends_none_of_its_bytes(file): + digests = file_digests(file, block_size=4096) + original = file.read_bytes() + file.write_bytes(b"X" + original[1:]) + with pytest.raises(uploads._FileChanged): + uploads._read_range(file, digests, 123, 10) + + +def test_shortened_file_is_refused(file): + digests = file_digests(file, block_size=4096) + file.write_bytes(b"x") + with pytest.raises(uploads._FileChanged): + uploads._read_range(file, digests, 123, 10) + + +def test_state_is_private_atomic_and_round_trips(tmp_path, session): + store = resume.SessionStore(tmp_path / "sessions", {"path": "model", "hash": "hash"}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + assert store.load(identity) == session + if os.name != "nt": + assert store.path.stat().st_mode & 0o777 == 0o600 + assert store.directory.stat().st_mode & 0o777 == 0o700 + assert not list(store.directory.glob(".upload-*")) + store.clear() + assert store.load(identity) is None + + +@pytest.mark.parametrize("field", ["server", "path", "size", "mtime_ns", "hash", "folder"]) +def test_changed_file_or_destination_never_reuses_state(tmp_path, session, field): + binding = { + "server": "https://example/api", + "path": "model", + "size": 10, + "mtime_ns": 123, + "hash": "a", + "folder": "loras", + } + identity = {"account": "user", "workspace": "ws"} + store = resume.SessionStore(tmp_path / "sessions", binding) + assert store.save(session, identity) + changed = {**binding, field: "different"} + assert resume.SessionStore(store.directory, changed).load(identity) is None + + +@pytest.mark.parametrize( + "identity", [None, {"account": "other", "workspace": "ws"}, {"account": "user", "workspace": "other"}] +) +def test_unknown_or_different_identity_never_reuses_state(tmp_path, session, identity): + store = resume.SessionStore(tmp_path / "sessions", {}) + assert store.save(session, {"account": "user", "workspace": "ws"}) + assert store.load(identity) is None + + +def test_expired_and_old_state_is_pruned(tmp_path, session, monkeypatch): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + now = time.time() + monkeypatch.setattr(resume.time, "time", lambda: now + resume.LIFETIME + 1) + assert store.load(identity) is None + assert list(store.directory.glob("*.json")) == [] + + +def test_directory_is_bounded_and_invalid_json_is_ignored(tmp_path, session): + identity = {"account": "user", "workspace": "ws"} + for i in range(resume._MAX_ENTRIES + 5): + store = resume.SessionStore(tmp_path / "sessions", {"path": str(i)}) + assert store.save(session, identity) + assert len(list(store.directory.glob("*.json"))) == resume._MAX_ENTRIES + store.path.write_text("{broken") + assert store.load(identity) is None + + +def test_state_never_follows_a_symlink(tmp_path, session): + actual = tmp_path / "actual" + actual.mkdir() + link = tmp_path / "link" + try: + link.symlink_to(actual, target_is_directory=True) + except OSError: + pytest.skip("symlink permission unavailable") + store = resume.SessionStore(link, {}) + assert not store.save(session, {"account": "user", "workspace": "ws"}) + assert list(actual.iterdir()) == [] + + +def test_corrupt_newer_session_does_not_hide_valid_session(tmp_path, session): + identity = {"account": "user", "workspace": "ws"} + first = resume.SessionStore(tmp_path / "sessions", {"path": "model"}) + second = resume.SessionStore(first.directory, first.binding) + assert first.save(session, identity) + assert second.save(session, identity) + second.path.write_text("{broken") + now = time.time() + os.utime(first.path, (now - 10, now - 10)) + os.utime(second.path, (now, now)) + loaded = resume.SessionStore(first.directory, first.binding) + assert loaded.load(identity) == session + assert loaded.path == first.path + + +def test_unknown_identity_is_never_persisted(tmp_path, session): + store = resume.SessionStore(tmp_path / "sessions", {}) + assert not store.save(session, None) + assert not store.directory.exists() + + +def test_state_expiry_cannot_be_extended_by_a_future_server_expiry(tmp_path, session, monkeypatch): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save({**session, "expires_at": "2099-01-01T00:00:00Z"}, identity) + saved = json.loads(store.path.read_text()) + saved["created_at"] = time.time() - resume.LIFETIME - 1 + store.path.write_text(json.dumps(saved)) + assert store.load(identity) is None + + +@pytest.mark.parametrize("piece", [None, 0, True, 123, 128 * 1024**2]) +def test_invalid_piece_size_is_refused_before_sending(session, file, clock, monkeypatch, piece): + bucket = Bucket() + monkeypatch.setattr(requests, "put", bucket) + with pytest.raises(resume.TransferFailed): + send({**session, "chunk_size_bytes": piece}, file, clock) + assert bucket.calls == [] + + +def test_range_cannot_go_backwards(session, file, clock, monkeypatch): + responses = iter(["bytes=0-262143", "bytes=0-5"]) + monkeypatch.setattr( + requests, + "put", + lambda *a, **k: SimpleNamespace(status_code=308, headers={"Range": next(responses)}, close=lambda: None), + ) + with pytest.raises(resume.TransferFailed): + send(session, file, clock) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX permission bits") +def test_world_readable_state_is_not_reused(tmp_path, session): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + store.path.chmod(0o644) + assert store.load(identity) is None + + +def test_failed_atomic_save_keeps_the_previous_session(tmp_path, session, monkeypatch): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + + def fail(*a): + raise OSError("disk full") + + monkeypatch.setattr(resume.os, "replace", fail) + assert not store.save({**session, "upload_id": "other"}, identity) + assert store.load(identity) == session + assert not list(store.directory.glob(".upload-*")) + + +@pytest.mark.parametrize("url", [1, True, {}, [], b"https://bucket.example/upload"]) +def test_invalid_url_types_are_refused(session, url): + assert not resume.valid_start({**session, "upload_url": url}) + + +@pytest.mark.parametrize("url", [1, True, {}, [], "http://bucket.example/upload"]) +def test_malformed_saved_url_is_ignored(tmp_path, session, url): + store = resume.SessionStore(tmp_path / "sessions", {}) + identity = {"account": "user", "workspace": "ws"} + assert store.save(session, identity) + saved = json.loads(store.path.read_text()) + saved["start"]["upload_url"] = url + store.path.write_text(json.dumps(saved)) + assert resume.SessionStore(store.directory, {}).load(identity) is None + + +def test_overlapping_uploads_keep_separate_sessions(tmp_path, session): + identity = {"account": "user", "workspace": "ws"} + first = resume.SessionStore(tmp_path / "sessions", {"path": "model", "hash": "hash"}) + second = resume.SessionStore(first.directory, first.binding) + assert first.load(identity) is None + assert second.load(identity) is None + other_session = {**session, "upload_id": "upload-2", "upload_url": "https://bucket.example/other"} + assert first.save(session, identity) + assert second.save(other_session, identity) + assert first.path != second.path + first.clear() + rerun = resume.SessionStore(first.directory, first.binding) + assert rerun.load(identity) == other_session + assert rerun.path == second.path + rerun.clear() + assert not list(first.directory.glob("*.json")) + + +@pytest.mark.parametrize( + "url", ["http://bucket.example/upload", "https:///missing-host", "https://user:pass@bucket.example/upload"] +) +def test_resumable_addresses_still_require_https_without_userinfo(session, url): + assert not resume.valid_start({**session, "upload_url": url}) + + +@pytest.mark.parametrize("setting", ["REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"]) +@pytest.mark.parametrize( + "contents", ["not a certificate\n", "-----BEGIN CERTIFICATE-----\ninvalid\n-----END CERTIFICATE-----\n"] +) +def test_existing_invalid_ca_bundle_is_fatal_with_real_requests(tmp_path, monkeypatch, setting, contents): + """Exercise Requests, urllib3 and OpenSSL without contacting a remote service.""" + import socket + + bundle = tmp_path / "invalid-ca.pem" + bundle.write_text(contents) + monkeypatch.delenv("REQUESTS_CA_BUNDLE", raising=False) + monkeypatch.setenv(setting, str(bundle)) + monkeypatch.setenv("NO_PROXY", "127.0.0.1") + monkeypatch.setenv("no_proxy", "127.0.0.1") + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + start = {"upload_url": f"https://127.0.0.1:{listener.getsockname()[1]}/?secret=SESSION", "headers": {}} + with pytest.raises(resume.TlsSetupFailed) as error: + resume._request(start, b"", "bytes */16") + assert "SESSION" not in str(error.value) + + +@pytest.mark.parametrize("certificate", [False, True], ids=["transient-eof", "certificate"]) +def test_https_proxy_error_uses_real_requests_adapter(session, monkeypatch, certificate): + """Keep Requests' proxy exception translation; replace only the network call.""" + import ssl + + from urllib3.connectionpool import HTTPSConnectionPool + from urllib3.exceptions import MaxRetryError, ProxyError + + calls = [] + + def fail(pool, method, url, **kwargs): + calls.append((method, url)) + reason = ssl.SSLCertVerificationError("certificate verify failed") if certificate else ssl.SSLEOFError("EOF") + raise MaxRetryError(pool, url, ProxyError("proxy failed", reason)) + + monkeypatch.setattr(HTTPSConnectionPool, "urlopen", fail) + if certificate: + with pytest.raises(resume.TlsSetupFailed) as error: + resume._request(session, b"", "bytes */16") + assert "SESSION" not in str(error.value) + else: + assert resume._request(session, b"", "bytes */16") == (None, None) + assert len(calls) == 1 diff --git a/tests/comfy_cli/output/test_discovery.py b/tests/comfy_cli/output/test_discovery.py index 4e682b9c9..f7f4b3ae4 100644 --- a/tests/comfy_cli/output/test_discovery.py +++ b/tests/comfy_cli/output/test_discovery.py @@ -104,6 +104,20 @@ def test_discover_includes_error_codes_from_markdown(): assert "cql_no_graph" in codes # the loader's "no source available" survives +def test_model_upload_guidance_leaves_size_limits_to_cloud_and_explains_resume(): + from comfy_cli.discovery import load_error_codes + + codes = {row["code"]: row for row in load_error_codes()} + assert "file_too_large" not in codes["asset_upload_invalid"]["meaning"] + assert "FILE_TOO_LARGE" in codes["asset_upload_refused"]["meaning"] + paused = codes["asset_upload_failed"] + assert "upload_paused" in paused["meaning"] + assert "bytes_held" in paused["meaning"] + assert "session_saved" in paused["meaning"] + assert "same command" in paused["hint"] + assert "start a new upload" in paused["hint"] + + def test_discover_capabilities_flags(): envelope = _run_cli(["--json", "discover"]) caps = envelope["data"]["capabilities"] diff --git a/tests/comfy_cli/output/test_error_code_registry.py b/tests/comfy_cli/output/test_error_code_registry.py index ffbb58536..eee678c45 100644 --- a/tests/comfy_cli/output/test_error_code_registry.py +++ b/tests/comfy_cli/output/test_error_code_registry.py @@ -9,9 +9,9 @@ 1. Every code raised in source is in the registry. A typo or a fresh code added without registry update fails the test and surfaces the typo. -2. Every code in the registry is raised somewhere. A code that's - deprecated or removed but left in the registry fails the test, forcing - the dead entry to be deleted. +2. Every code in the registry is raised somewhere or explicitly retired. + Retired codes stay registered under ADR-AUTOMATION-0003's append-only + contract; an accidental unused entry still fails the test. Warning objects (``{"code": …, "message": …}`` placed in a ``warnings`` list) are collected separately from failures. Registered codes are failure @@ -256,6 +256,11 @@ def _collect_codes() -> tuple[dict[str, list[Path]], dict[str, list[Path]]]: ) +# The storage wait no longer has a total deadline. Keep the published code +# discoverable for clients that also handle output from older CLI versions. +RETIRED_ERROR_REGISTRATIONS = frozenset({"asset_upload_timeout"}) + + @pytest.fixture(scope="module") def collected_codes() -> tuple[dict[str, list[Path]], dict[str, list[Path]]]: return _collect_codes() @@ -343,18 +348,28 @@ def test_every_registered_code_is_raised(raised_codes, warned_codes): """If this fails: a code in the registry is no longer raised anywhere. A legacy warning registration (:data:`LEGACY_WARNING_REGISTRATIONS`) - counts as used while its warning is still emitted. + counts as used while its warning is still emitted. Explicitly retired + codes stay registered for compatibility but must no longer be emitted. - Fix: delete the dead entry from the registry, or wire the code up. + Fix: wire up an accidental unused code, or explicitly retire a published + code whose behavior was removed. Never delete a published registration. """ dead = [ c for c in error_codes.all_codes() - if c not in raised_codes and not (c in LEGACY_WARNING_REGISTRATIONS and c in warned_codes) + if c not in raised_codes + and c not in RETIRED_ERROR_REGISTRATIONS + and not (c in LEGACY_WARNING_REGISTRATIONS and c in warned_codes) ] assert not dead, ( - f"Registered but never raised:\n{dead}\nEither delete these from comfy_cli/error_codes.REGISTRY or use them." + f"Registered but never raised:\n{dead}\nUse these codes or explicitly retire their published registrations." + ) + invalid_retired = sorted( + c + for c in RETIRED_ERROR_REGISTRATIONS + if not error_codes.is_registered(c) or c in raised_codes or c in warned_codes ) + assert not invalid_retired, f"Retired codes must stay registered and never be emitted: {invalid_retired}" def test_every_build_code_has_a_first_call_site(raised_codes):