diff --git a/CHANGELOG.md b/CHANGELOG.md index 8929a14ca..4f3d950e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,41 @@ history. `asset_upload_refused`; new codes `asset_upload_invalid`, `asset_upload_failed` and `asset_upload_timeout` cover the other ways it can end. +- `comfy model upload --type --where cloud` uploads one local + model file into your Comfy Cloud model library, so a workflow run on Comfy + Cloud can load it, e.g. `comfy model upload my_style.safetensors --type loras + --where cloud`. Run it with `--dry-run` first: it checks the file, the folder + and the library, uploads nothing, and prints the file's sha256. Check that + sha256 with `shasum -a 256` before you approve. The real upload asks before it + starts. An agent (or any `--json` caller the CLI takes for one) must pass + `--yes`, or it gets `model_upload_needs_confirm` and nothing is sent. + `--expected-sha256 ` uploads nothing unless the file still has the bytes + the dry run reported. `--workspace ` uploads nothing unless this sign-in + uploads into that workspace (`asset_upload_wrong_workspace`). The `--json` + output (the `model_upload` schema) carries `loader_path`, the value a + workflow's loader node takes to load the file on Comfy Cloud. It is Comfy + Cloud's own value from its record of the model, and null when Comfy Cloud + gives none or when a dry run would upload. Builds on #1008. +- `comfy model upload` and `comfy assets library upload` upload only the file + named, and succeed only once Comfy Cloud's hash of the stored file equals the + local file's. Before anything is sent they look the name up in that folder of + the workspace's model library: every member's models, the ones a run on Comfy + Cloud loads, and no public ones. + The same bytes under that name are reported as `already_present` and not sent + again (for `comfy assets library upload`, only when they are the only model of + that name). `comfy model upload` refuses other bytes under that name + (`asset_upload_invalid`, `details.reason` `name_in_use`), because a second + file would change what that name loads for every workflow in the workspace. + `comfy assets library upload` still uploads them, as a new version of the + name, so a script that replaces a model that way keeps working. The upload sends only bytes + that were hashed: each 1 MiB block is checked against its hash before any of + it is sent. A file that changed after it was hashed stops the upload before + the changed bytes are sent, and is never completed (`asset_upload_failed`, + `file_changed`). A stored + hash that differs, or that Comfy Cloud does not report, ends as + `asset_upload_unverified`. `comfy assets library upload` gains `--dry-run`, + and its `--json` output gains `loader_path`, `sha256`, `workspace_id`, + `uploaded`, `already_present` and `dry_run`. - `comfy knowledge resolve` suggests canonical model ids for an incomplete family/version name such as `Qwen 2.1`. The command still fails with `knowledge_unknown_model` and exit 1, but `error.details.candidate_ids` lists diff --git a/README.md b/README.md index 614e7683a..11d64ca35 100644 --- a/README.md +++ b/README.md @@ -440,6 +440,18 @@ the bisect tool can help you pinpoint the custom node that causes the issue. `comfy model list ?[--relative-path ]` +- Model upload to Comfy Cloud + + `comfy model upload --type --where cloud ?[--dry-run] ?[--workspace ] ?[--expected-sha256 ] ?[--yes]` + + - Puts one local `.safetensors` file (up to 2 GB) into your Comfy Cloud model library, so a workflow run on Comfy Cloud can load it. `--type` is the model folder, e.g. `loras` or `checkpoints`; `comfy model list-folders --where cloud` lists them. + - Only the file you name is uploaded, and only the bytes hashed before the upload: each block is checked against its hash before it is sent. The upload succeeds only once Comfy Cloud's hash of the stored file matches your file's. + - `--dry-run` uploads nothing. It reports the file's `sha256` and whether Comfy Cloud already has the file. + - The output's `loader_path` is the value a loader node takes for the model on Comfy Cloud, as Comfy Cloud's record of the model gives it. It is null for a dry run that would upload, or when Comfy Cloud gives none. + - `--workspace` and `--expected-sha256` tie an approval to one workspace and one set of bytes. If you are signed in to another workspace, or the file changed since the dry run, nothing is uploaded. + - It asks before uploading, unless you pass `--yes`. + - It refuses a name that the folder already holds with different bytes in your workspace's model library (any member's upload), since a second file under that name would change what the name loads in every workflow. + ### Running on Comfy Cloud (`--where cloud`) Comfy Cloud runs your workflow on Comfy's GPUs — no local ComfyUI install and no GPU required. The same verbs you use locally take `--where cloud`; the only extra step is signing in once. diff --git a/comfy_cli/command/assets_library.py b/comfy_cli/command/assets_library.py index f522bf70b..7ee9bf5e0 100644 --- a/comfy_cli/command/assets_library.py +++ b/comfy_cli/command/assets_library.py @@ -9,7 +9,6 @@ from __future__ import annotations -import os import re import time from dataclasses import dataclass, replace @@ -373,6 +372,14 @@ def ensure_cmd( "UNSUPPORTED_MODEL_TYPE": "pass a --type from `model_upload_folders` in GET /api/features", "FILE_TOO_LARGE": "files can be up to 2 GB", } +# Any form Comfy Cloud reports a model's content hash in: bare hex, canonical +# ``blake3:`` or the stored ``.``, any case. +_ANY_HASH_RE = re.compile(r"^(?:blake3:)?([0-9a-fA-F]{64})(?:\.[A-Za-z0-9]+)?$") +# One page at the API's maximum when looking up the models a name already +# stands for; a name matching more than this is refused, not paged. +_LOOKUP_LIMIT = 500 +# An approval's SHA-256: 64 hex, bare or ``sha256:``-prefixed, any case. +_SHA256_RE = re.compile(r"^(?:sha256:)?([0-9a-fA-F]{64})$") _sleep = time.sleep _clock = time.monotonic @@ -391,13 +398,64 @@ class _PutFailed(Exception): class _FileUnreadable(Exception): - """The file could not be opened, sized or read while it was being sent; ``args[0]`` is the ``OSError``.""" + """The file could not be opened or read while it was being sent; ``args[0]`` is the ``OSError``.""" class _TlsSetupFailed(Exception): """This machine's TLS settings stopped the PUT before it connected; ``args[0]`` is the message to show.""" +class _FileChanged(Exception): + """A block of the file read for the PUT is not the block hashed before it; ``args[0]`` is the block's index.""" + + +class _BlockChanged(OSError): + """Raised from inside the PUT's body read, which Requests and urllib3 let through only as an ``OSError``.""" + + +class _HashedBytes: + """The file, read for the PUT, giving out only the bytes that were hashed. + + Each block is read whole and checked against its digest from the hash + taken before the upload (``FileDigests.blocks``) before any of it is + handed on, so a byte that was not hashed, and approved by its hash, never + reaches storage. Reading stops at the hashed size: bytes added to the end + since are not sent. + """ + + def __init__(self, raw, digests) -> None: + self._raw, self._digests = raw, digests + self._pending = b"" + self._next_block = 0 + self._given = 0 + + def read(self, size: int = -1) -> bytes: + from blake3 import blake3 + + if not self._pending: + d = self._digests + if self._next_block >= len(d.blocks): + return b"" + block = self._raw.read(min(d.block_size, d.size - self._next_block * d.block_size)) + if blake3(block).digest() != d.blocks[self._next_block]: + raise _BlockChanged(f"block {self._next_block} of the file is not the one hashed") + self._pending, self._next_block = block, self._next_block + 1 + n = len(self._pending) if size is None or size < 0 else size + out, self._pending = self._pending[:n], self._pending[n:] + self._given += len(out) + return out + + @property + def mode(self) -> str: + return self._raw.mode + + def fileno(self) -> int: + return self._raw.fileno() + + def tell(self) -> int: + return self._given + + def _file_problem(path: Path, e: OSError) -> tuple[str, str]: """``(reason, message)`` for an ``OSError`` met while checking the file.""" if isinstance(e, FileNotFoundError): @@ -552,15 +610,26 @@ def _send(target, sign_in, url: str, **kwargs) -> tuple[int, dict | None]: return http_request(url, replace(target, auth_token=sign_in.current()), **kwargs) -def _call(renderer, target, operation: str, parts: tuple[str, ...], body: dict | None = None, *, sign_in=None) -> dict: +def _call( + renderer, + target, + operation: str, + parts: tuple[str, ...], + body: dict | None = None, + *, + sign_in=None, + query: list[tuple[str, Any]] | None = None, +) -> dict: """One JSON call to the cloud API; anything but a JSON object is a mapped error.""" import http.client import urllib.error + import urllib.parse route = "/api/" + "/".join(parts) method = "GET" if body is None else "POST" + url = target.url(*parts) + ("?" + urllib.parse.urlencode(query) if query else "") try: - _, parsed = _send(target, sign_in, target.url(*parts), method=method, body=body, strict_json=True) + _, parsed = _send(target, sign_in, url, method=method, body=body, strict_json=True) # HTTPException: a connection closed partway through the body (IncompleteRead). except (urllib.error.URLError, OSError, http.client.HTTPException) as e: raise _http_failure(renderer, e, operation, sign_in) from e @@ -571,12 +640,16 @@ def _call(renderer, target, operation: str, parts: tuple[str, ...], body: dict | return parsed -def _put(upload_url: str, headers: dict, path: Path, progress) -> None: - """Stream the file to its signed URL, or raise ``_PutFailed``. +def _put(upload_url: str, headers: dict, path: Path, progress, digests) -> None: + """Stream the bytes ``digests`` was taken from to the signed URL, or raise ``_PutFailed``. + + Each block is checked against ``digests`` before any of it is sent + (``_HashedBytes``); a block that changed since raises ``_FileChanged``, + with only the earlier, unchanged blocks sent. A transport exception is dropped, never shown: its text quotes the signed - URL, whose query string is a live credential. A file that cannot be opened, - sized or read raises ``_FileUnreadable`` instead: a fresh upload would fail + URL, whose query string is a live credential. A file that cannot be opened + or read raises ``_FileUnreadable`` instead: a fresh upload would fail the same way, so it is not a transport failure to retry. Any other ``OSError`` comes from setting up TLS (a CA bundle path that no longer exists) and raises ``_TlsSetupFailed`` with its message; a retry would @@ -598,9 +671,8 @@ def __init__(self, raw, progress, size: int) -> None: self._size = size def __len__(self) -> int: - # Requests takes the length from here rather than its own - # `os.fstat`, so the file is sized only below, where an error is - # known to be the file's. Same Content-Length either way. + # Requests takes the length from here: the hashed size, the only + # bytes `_HashedBytes` gives out. return self._size def read(self, size: int = -1) -> bytes: @@ -615,11 +687,10 @@ def read(self, size: int = -1) -> bytes: except OSError as e: raise _FileUnreadable(e) from None with f: - try: - size = os.fstat(f.fileno()).st_size - except OSError as e: # e.g. a stale handle on a network filesystem - raise _FileUnreadable(e) from None - reader = _Reader(f, progress, size) + # The size hashed, not the file's size now: only the hashed bytes are + # sent, and a file cut short since fails its last block's check. + size = digests.size + reader = _Reader(_HashedBytes(f, digests), progress, size) try: resp = requests.put( upload_url, @@ -629,6 +700,8 @@ def read(self, size: int = -1) -> bytes: allow_redirects=False, # the URL names one object; never divert the file ) except OSError as e: # `requests.RequestException` is an OSError too + if isinstance(reader.failed, _BlockChanged): + raise _FileChanged(str(reader.failed)) from None if reader.failed is not None: raise _FileUnreadable(reader.failed) from None if isinstance(e, requests.RequestException): @@ -694,28 +767,168 @@ def _wait_for_task(renderer, target, task_id: str, sign_in=None) -> dict: _sleep(pause) -@app.command("upload", help="Upload a .safetensors model file (up to 2 GB) into your Comfy Cloud model library.") +def _content_hash(value: Any) -> str | None: + """``blake3:`` for any form Comfy Cloud reports a content hash in + (bare hex, canonical ``blake3:``, or the stored ``.``); + ``None`` for anything else, so an absent or garbled hash never matches.""" + m = _ANY_HASH_RE.match(value) if isinstance(value, str) else None + return f"blake3:{m.group(1).lower()}" if m else None + + +def _models_named(renderer, target, name: str, folder: str, sign_in, *, strict: bool) -> list[dict] | None: + """The models in ``folder`` named exactly ``name`` in this workspace's model + library: every member's, and never the public catalog. + + These are the rows a run on Comfy Cloud loads a model from, by folder and + name (the cloud's ``runtimemodels.Lookup`` reads the workspace without a + user and without public models), so they are the rows the name already + stands for. ``workspace_library=true`` asks for exactly that set; without + it Comfy Cloud lists only the caller's own models plus public ones. A + listing that does not fit on one page cannot show that none of them holds + other bytes: with ``strict`` it refuses rather than guess, and without it + the answer is ``None``, not known. + """ + body = _call( + renderer, + target, + "lookup", + ("assets",), + sign_in=sign_in, + query=[ + # Comma-separated: Comfy Cloud refuses a repeated include_tags with a 400. + ("include_tags", f"models,{folder}"), + ("name_contains", name), + ("workspace_library", "true"), + # Comfy Cloud drops public models for a workspace library anyway; + # said here too, so a server that ignores workspace_library never + # answers with public models the run does not load. + ("include_public", "false"), + ("limit", _LOOKUP_LIMIT), + ], + ) + rows = body.get("assets") + if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows): + raise _unexpected(renderer, "/api/assets", "assets must be a list of objects", operation="lookup") + if body.get("has_more") is True: + if not strict: + return None + renderer.error( + code="asset_upload_invalid", + message=f"more than {_LOOKUP_LIMIT} models in {folder} have names containing {name!r}, " + "so it cannot be checked that none of them is already named that with other bytes", + hint="rename the file to a more distinctive name and upload it again", + details={"reason": "name_unverified", "file": name, "type": folder}, + ) + raise typer.Exit(code=1) + return [r for r in rows if r.get("name") == name] + + +def _stored_asset(target, sign_in, asset_id: str) -> dict | None: + """Comfy Cloud's record of ``asset_id``, or ``None`` when it cannot be read.""" + import http.client + import urllib.error + import urllib.parse + + try: + _, asset = _send(target, sign_in, target.url("assets", urllib.parse.quote(asset_id, safe="")), strict_json=True) + except (urllib.error.URLError, OSError, http.client.HTTPException, ResponseUnparseable): + return None + return asset if isinstance(asset, dict) else None + + +def _current_workspace(target, sign_in) -> str | None: + """The id of the workspace this sign-in uploads into, or ``None`` when Comfy Cloud does not say.""" + import http.client + import urllib.error + + try: + _, body = _send(target, sign_in, target.url("workspaces", "current"), strict_json=True) + except (urllib.error.URLError, OSError, http.client.HTTPException, ResponseUnparseable): + return None + ws = body.get("id") if isinstance(body, dict) else None + return ws if isinstance(ws, str) and ws else None + + +def _loader_path(asset: dict | None) -> str | None: + """The value a loader widget takes for the model, as Comfy Cloud's record + gives it; ``None`` when there is no record or it gives none (Comfy Cloud + could not derive one from the name).""" + value = asset.get("loader_path") if isinstance(asset, dict) else None + return value if isinstance(value, str) and value else None + + +@app.command( + "upload", + help="Upload one named .safetensors model file (up to 2 GB) into your Comfy Cloud model library. " + "Succeeds only once Comfy Cloud's hash of the stored file matches this file's; --dry-run checks without uploading.", +) @tracking.track_command("assets") def upload_cmd( file: Annotated[Path, typer.Argument(help="The .safetensors file to upload.", show_default=False)], folder: Annotated[ str, typer.Option("--type", help="Model folder to file it under (e.g. loras, checkpoints, vae).") ], + dry_run: Annotated[ + bool, + typer.Option( + "--dry-run", + help="Check the file, the folder and the library, and report the file's hashes and whether " + "Comfy Cloud already has it; upload nothing.", + ), + ] = False, where: Annotated[str | None, typer.Option("--where", show_default=False)] = None, ): + run_upload(file, folder, where=where, dry_run=dry_run, command="assets library upload") + + +def run_upload( + file: Path, + folder: str, + *, + where: str | None, + dry_run: bool, + command: str, + workspace: str | None = None, + expected_sha256: str | None = None, + yes: bool = True, + ctx: typer.Context | None = None, + refuse_name_reuse: bool = False, +) -> None: + """Upload ``file``, and only ``file``, into ``folder`` of the Comfy Cloud model library. + + Shared by ``comfy assets library upload`` and ``comfy model upload``. It + succeeds only once Comfy Cloud's hash of the stored bytes is the file's + own. ``workspace`` and ``expected_sha256`` pin an approval: when given, a + sign-in to another workspace, or a file whose bytes are not the ones + approved, uploads nothing. ``yes=False`` asks before the upload starts. + The same bytes already under the name are never sent again. + ``refuse_name_reuse`` refuses a name that already holds other bytes in + the folder; without it the file is uploaded under that name, as a new + version of it. + """ import urllib.parse from comfy_cli.command.build_upload_progress import UploadProgressReporter from comfy_cli.credentials import DataPlaneToken + from comfy_cli.hashing import file_digests + from comfy_cli.interaction import confirm - def refuse(reason: str, message: str) -> typer.Exit: - renderer.error(code="asset_upload_invalid", message=message, details={"reason": reason, "file": str(file)}) + def refuse(reason: str, message: str, **details: Any) -> typer.Exit: + renderer.error( + code="asset_upload_invalid", message=message, details={"reason": reason, "file": str(file), **details} + ) return typer.Exit(code=1) renderer = get_renderer() refusal = _local_refusal(file) if refusal is not None: raise refuse(*refusal) + approved = None + if expected_sha256 is not None: + m = _SHA256_RE.match(expected_sha256) + if m is None: + raise refuse("bad_expected_sha256", f"--expected-sha256 {expected_sha256!r} is not a 64-hex SHA-256") + approved = m.group(1).lower() target = cloud_target_or_local_error(where, renderer) # A stored sign-in's token can expire during the PUT, so completion and # polling keep it fresh, but only from the same sign-in: Comfy Cloud files the @@ -729,6 +942,30 @@ def refuse(reason: str, message: str) -> typer.Exit: else None ) + # The person approved an upload into one workspace; a sign-in that has + # since moved to another must not put the file there instead. + workspace_id = None + if workspace is not None: + workspace_id = _current_workspace(target, sign_in) + if workspace_id != workspace: + renderer.error( + code="asset_upload_wrong_workspace", + message=( + f"this sign-in uploads into workspace {workspace_id}, not {workspace}" + if workspace_id is not None + else f"Comfy Cloud did not say which workspace this sign-in uploads into, " + f"so it cannot be checked that it is {workspace}" + ), + hint="check the workspace with `comfy cloud status`; sign in to the approved one with " + "`comfy cloud login`, then run the upload again", + details={ + "reason": "workspace_mismatch" if workspace_id is not None else "workspace_unknown", + "workspace": workspace, + "current_workspace": workspace_id, + }, + ) + raise typer.Exit(code=1) + # Absent while uploads are off for the workspace, or when the server could # not evaluate its flags. Either way the start call decides, and it refuses # before any byte is sent. @@ -745,9 +982,83 @@ def refuse(reason: str, message: str) -> typer.Exit: raise typer.Exit(code=1) try: - size = file.stat().st_size + if renderer.is_pretty(): + renderer.info(f"Hashing {file.name}...") + # One read gives the hashes, the size and each block's digest; the PUT + # sends only bytes that match these blocks (`_HashedBytes`). + digests = file_digests(file) except OSError as e: # removed or locked since it was checked; nothing is sent yet raise refuse(*_file_problem(file, e)) from None + local_hash, sha256, size = digests.blake3, digests.sha256, digests.size + if approved is not None and approved != sha256: + raise refuse( + "file_changed", + f"{file.name}'s bytes are not the ones --expected-sha256 names; the file changed since it was checked", + expected_sha256=approved, + sha256=sha256, + ) + + # A run loads the newest model of a name in a folder, so a second file + # under a name already in use changes what that name loads for every + # workflow in the workspace: `comfy model upload` refuses it, while + # `comfy assets library upload` keeps uploading it as a new version. The + # same bytes under the name are already there and need no upload. + named = _models_named(renderer, target, file.name, folder, sign_in, strict=refuse_name_reuse) or [] + same = [r for r in named if _content_hash(r.get("hash")) == local_hash] + others = [r for r in named if _content_hash(r.get("hash")) != local_hash] + if others and refuse_name_reuse: + raise refuse( + "name_in_use", + f"{folder}/{file.name} is already in your Comfy Cloud library with different bytes; " + "uploading would change what that name loads", + hash=local_hash, + existing=[{"id": r.get("id"), "hash": r.get("hash")} for r in others], + ) + # Uploading again is what makes these bytes the newest under the name, + # so a name that also holds other bytes, or a lookup too long to read, + # is not "already present" for the older command. + present = same[0] if same and not others else None + + def report(asset_id, *, task_id=None, uploaded=False, name=file.name, asset=None) -> dict: + return { + "id": asset_id, + "name": name, + "folder": folder, + "loader_path": _loader_path(asset), + "hash": local_hash, + "sha256": sha256, + "size": size, + "workspace_id": workspace_id, + "task_id": task_id, + "uploaded": uploaded, + "already_present": present is not None, + "dry_run": dry_run, + } + + if dry_run or present is not None: + payload = report(present.get("id") if present is not None else None, asset=present) + if renderer.is_pretty(): + if present is not None: + renderer.success(f"{folder}/{file.name} is already in your library with these bytes; nothing to upload") + else: + renderer.info(f"Would upload {file.name} ({size} bytes) to {folder}") + renderer.info(f"sha256: {sha256}\nhash: {local_hash}") + renderer.emit(payload, command=command, where="cloud", changed=False) + return + + into = f" in workspace {workspace_id}" if workspace_id else "" + if not confirm( + f"Upload {file} ({size} bytes, sha256 {sha256}) to {folder} on Comfy Cloud{into}?", + yes=yes, + error_code="model_upload_needs_confirm", + details={"file": str(file), "type": folder, "sha256": sha256, "workspace_id": workspace_id}, + ctx=ctx, + ): + # Only a prompt can decline, and one is allowed only in pretty mode, + # where `emit` writes nothing; a machine caller is refused above. + renderer.info("Nothing uploaded.") + return + item = _UploadItem(filename=file.name, size_bytes=size) tags = ["models", folder] reporter = UploadProgressReporter(renderer) @@ -775,7 +1086,7 @@ def refuse(reason: str, message: str) -> typer.Exit: raise _unexpected(renderer, "/api/assets/uploads", "expected upload_id, upload_url, method PUT and headers") try: with reporter.uploading(item, 1, 1) as progress: - _put(upload_url, headers, file, progress) + _put(upload_url, headers, file, progress, digests) break except _FileUnreadable as e: renderer.error( @@ -786,6 +1097,16 @@ def refuse(reason: str, message: str) -> typer.Exit: details={"reason": "file_unreadable", "upload_id": upload_id, "attempts": attempt}, ) raise typer.Exit(code=1) from None + except _FileChanged: + # Only the blocks before the changed one were sent, and the + # upload is never completed, so the staged bytes never become an asset. + renderer.error( + code="asset_upload_failed", + message=f"{file.name} changed after it was hashed; none of the changed bytes were sent", + hint="wait until nothing is writing the file, then upload it again; nothing was added to the library", + details={"reason": "file_changed", "upload_id": upload_id, "hash": local_hash, "attempts": attempt}, + ) + raise typer.Exit(code=1) from None except _TlsSetupFailed as e: renderer.error( code="asset_upload_failed", @@ -832,15 +1153,47 @@ def refuse(reason: str, message: str) -> typer.Exit: asset_id = result.get("asset_id") if not isinstance(asset_id, str) or not asset_id: raise _unexpected(renderer, "/api/tasks", "a completed task with no asset_id", task_id=task_id) - payload = { - "id": asset_id, - "name": result["filename"] if isinstance(result.get("filename"), str) else item.filename, - "folder": folder, - "hash": result["hash"] if isinstance(result.get("hash"), str) else None, - "size": item.size_bytes, - "task_id": task_id, - } + + # The upload counts only when Comfy Cloud's own hash of what it stored is + # this file's. The task reports it; the asset itself is read when not. + stored = _content_hash(result.get("hash")) + asset = None + if stored is None: + asset = _stored_asset(target, sign_in, asset_id) + stored = _content_hash(asset.get("hash")) if asset is not None else None + if stored != local_hash: + mismatch = stored is not None + renderer.error( + code="asset_upload_unverified", + message=( + f"Comfy Cloud stored asset {asset_id} with hash {stored}, not {item.filename}'s {local_hash}" + if mismatch + else f"Comfy Cloud stored asset {asset_id} but its hash could not be read to check it" + ), + hint=( + f"delete asset {asset_id} from your Comfy Cloud model library before running a workflow that " + f"loads {item.filename}, then upload it again" + if mismatch + else f"check asset {asset_id}'s hash with `comfy assets library ls --name {item.filename} " + "--tags models` before running a workflow that loads it" + ), + details={ + "reason": "hash_mismatch" if mismatch else "hash_unavailable", + "asset_id": asset_id, + "task_id": task_id, + "hash": local_hash, + "stored_hash": stored, + }, + ) + raise typer.Exit(code=1) + + # The bytes are verified; the record is read only for its loader_path, + # which stays null when it cannot be read. + if asset is None: + asset = _stored_asset(target, sign_in, asset_id) + name = result["filename"] if isinstance(result.get("filename"), str) else item.filename + payload = report(asset_id, task_id=task_id, uploaded=True, name=name, asset=asset) if renderer.is_pretty(): - renderer.success(f"Added {payload['name']} to {folder}") - renderer.info(f"asset id: {asset_id}\nhash: {payload['hash'] or '-'}") - renderer.emit(payload, command="assets library upload", where="cloud", changed=True) + renderer.success(f"Added {name} to {folder}") + renderer.info(f"asset id: {asset_id}\nsha256: {sha256}\nhash: {local_hash} (matches the stored file)") + renderer.emit(payload, command=command, where="cloud", changed=True) diff --git a/comfy_cli/command/models/models.py b/comfy_cli/command/models/models.py index be55c1e09..e5e02f051 100644 --- a/comfy_cli/command/models/models.py +++ b/comfy_cli/command/models/models.py @@ -18,6 +18,7 @@ from comfy_cli import constants, download_state, tracking, ui from comfy_cli.command.models import search as models_search_command +from comfy_cli.command.models import upload as models_upload_command from comfy_cli.config_manager import ConfigManager from comfy_cli.constants import DEFAULT_COMFY_MODEL_PATH from comfy_cli.file_utils import ( @@ -2459,3 +2460,7 @@ def list_command( # setup on the whole noun would leak it onto those. app.registered_commands.extend(models_search_command.app.registered_commands) app.registered_groups.extend(models_search_command.app.registered_groups) +# `comfy model upload` (the Comfy Cloud model library) lives in its own module +# and is carried over the same way. It is not on the `comfy models` alias, +# which covers the discovery leaves only. +app.registered_commands.extend(models_upload_command.app.registered_commands) diff --git a/comfy_cli/command/models/upload.py b/comfy_cli/command/models/upload.py new file mode 100644 index 000000000..965e2d4fd --- /dev/null +++ b/comfy_cli/command/models/upload.py @@ -0,0 +1,82 @@ +"""``comfy model upload`` — put one local model file into the Comfy Cloud model library. + +The upload itself is ``assets_library.run_upload``, shared with ``comfy assets +library upload``. This command adds what an agent needs to act on a person's +approval: ``--workspace`` and ``--expected-sha256`` pin the approval to one +workspace and one set of bytes, and the upload starts only with ``--yes`` or +an answered prompt. A dry run reports what the person is asked to approve. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated + +import typer + +from comfy_cli import tracking + +app = typer.Typer() + + +@app.command( + "upload", + help="Upload one named .safetensors model file (up to 2 GB) to your Comfy Cloud model library, so a " + "workflow run on Comfy Cloud can load it. Uploads only the file named; succeeds only once Comfy Cloud's " + "hash of the stored file matches this file's. Run with --dry-run first to see its sha256 and whether " + "Comfy Cloud already has it.", +) +@tracking.track_command("model") +def upload_command( + ctx: typer.Context, + file: Annotated[Path, typer.Argument(help="The .safetensors file to upload.", show_default=False)], + folder: Annotated[ + str, + typer.Option( + "--type", + help="Model folder to file it under (e.g. loras, checkpoints, vae); " + "`comfy model list-folders --where cloud` lists them.", + ), + ], + workspace_id: Annotated[ + str | None, + typer.Option( + "--workspace", + show_default=False, + help="The Comfy Cloud workspace id the upload was approved for (`comfy cloud status`). " + "If this sign-in uploads into another, nothing is uploaded.", + ), + ] = None, + expected_sha256: Annotated[ + str | None, + typer.Option( + "--expected-sha256", + show_default=False, + help="The sha256 a --dry-run reported. If the file's bytes no longer have it, nothing is uploaded.", + ), + ] = None, + dry_run: Annotated[ + bool, + typer.Option( + "--dry-run", + help="Check the file, the folder, the workspace and the library, and report the file's sha256 and " + "whether Comfy Cloud already has it; upload nothing.", + ), + ] = False, + yes: Annotated[bool, typer.Option("--yes", "-y", help="Upload without asking.")] = False, + where: Annotated[str | None, typer.Option("--where", show_default=False)] = None, +): + from comfy_cli.command.assets_library import run_upload + + run_upload( + file, + folder, + where=where, + dry_run=dry_run, + command="model upload", + workspace=workspace_id, + expected_sha256=expected_sha256, + yes=yes, + ctx=ctx, + refuse_name_reuse=True, + ) diff --git a/comfy_cli/discovery.py b/comfy_cli/discovery.py index 7898860e8..cefab74bd 100644 --- a/comfy_cli/discovery.py +++ b/comfy_cli/discovery.py @@ -166,6 +166,8 @@ "comfy model download-status": "download_status", "comfy model download-cancel": "download_status", "comfy model downloads": "downloads", + # one local model file into the Comfy Cloud model library + "comfy model upload": "model_upload", # partner-model discovery (`comfy generate`'s two read-only sub-actions — # they are argv-tail actions, not Typer subcommands, so they have no node in # the help tree; agents resolve them through `command_schemas`). @@ -231,6 +233,7 @@ # upload progress; under plain --json the same lines go to stderr "comfy build push": "build_push_event", "comfy assets library upload": "build_push_event", + "comfy model upload": "build_push_event", # the deployment coming up; same rule about stderr "comfy deploy up": "deploy_progress_event", "comfy deploy status": "deploy_progress_event", diff --git a/comfy_cli/error_codes.py b/comfy_cli/error_codes.py index 387e467c1..1e2590361 100644 --- a/comfy_cli/error_codes.py +++ b/comfy_cli/error_codes.py @@ -496,11 +496,33 @@ class ErrorCode: ), ErrorCode( "asset_upload_invalid", - "`assets library upload` refused the file or `--type` before uploading anything: " + "`model upload` or `assets library upload` refused the file or `--type` before uploading anything: " "`details.reason` is unsupported_file_type (not .safetensors), bad_filename, file_not_found, " - "file_unreadable (it could not be opened or its size read), empty_file, file_too_large (over 2 GB) " - "or unsupported_model_type (`details.valid_types` lists the folders the workspace takes).", - "fix the file or `--type` named in the message; nothing was sent", + "file_unreadable (it could not be opened or its size read), empty_file, file_too_large (over 2 GB), " + "unsupported_model_type (`details.valid_types` lists the folders the workspace takes), " + "bad_expected_sha256 (`--expected-sha256` is not 64 hex), file_changed (the file's sha256, " + "`details.sha256`, is not `details.expected_sha256`: it changed since the dry run that was approved), " + "name_in_use (`model upload` only: the folder of the workspace's model library, every member's models, " + "already holds a model of this name with other bytes, `details.existing`; a second one would change " + "what the name loads for every workflow in the workspace) or " + "name_unverified (`model upload` only: too many models with similar names to check that none holds other bytes).", + "fix the file or `--type` named in the message; for file_changed run the dry run again and get the " + "new sha256 approved; for name_in_use or name_unverified rename the file; nothing was sent", + ), + ErrorCode( + "asset_upload_wrong_workspace", + "`model upload --workspace` named a workspace this sign-in does not upload into " + "(`details.reason` workspace_mismatch; `details.current_workspace` is the one it does), or Comfy " + "Cloud did not say which workspace that is (`details.reason` workspace_unknown). Nothing was sent.", + "check `comfy cloud status`; sign in to the approved workspace with `comfy cloud login`, or get the " + "upload approved for the current one", + ), + ErrorCode( + "model_upload_needs_confirm", + "`model upload` was run without `--yes` where nothing can answer a prompt. Nothing was sent. " + "`details.file`, `details.type`, `details.sha256` and `details.workspace_id` name what would be " + "uploaded; `details.question` is the confirmation nothing could answer.", + "ask the person to approve uploading the file named, then pass `--yes --expected-sha256 `", ), ErrorCode( "asset_upload_refused", @@ -516,17 +538,31 @@ class ErrorCode: "retry (`details.status`, null when no answer came), the local file could not be read while it " "was being sent (`details.reason` file_unreadable; not retried), this machine's TLS settings stopped " "the PUT before it connected (`details.reason` tls_setup, e.g. REQUESTS_CA_BUNDLE naming a missing " - "file; not retried), or Comfy Cloud's storing task failed or was cancelled (the message carries the " - "task's error, e.g. not a valid .safetensors file).", + "file; not retried), a block of the file read for sending was not the block hashed before it " + "(`details.reason` file_changed: something wrote the file meanwhile; the blocks before it were sent, " + "none of the changed bytes were, and the upload was not completed, so no asset was added), or " + "Comfy Cloud's storing task failed or was cancelled (the message carries the task's error, e.g. not a " + "valid .safetensors file).", "retry for a storage failure; for an unreadable file or a task failure fix the file the message names; " - "for tls_setup fix the certificate setting the message names", + "for tls_setup fix the certificate setting the message names; for file_changed wait until nothing " + "writes the file, then run the dry run again", ), ErrorCode( "asset_upload_timeout", "The model file was uploaded, but Comfy Cloud had not finished storing it when " - "`assets library upload` stopped waiting. `details.task_id` is the storing task, which goes on.", + "`model upload` or `assets library upload` stopped waiting. `details.task_id` is the storing task, " + "which goes on.", "check `comfy assets library ls --tags models` later instead of uploading again", ), + ErrorCode( + "asset_upload_unverified", + "Comfy Cloud stored the uploaded file as an asset (`details.asset_id`), but its hash of the stored " + "bytes is not the local file's (`details.reason` hash_mismatch; `details.stored_hash` against " + "`details.hash`), or it could not be read to check (`details.reason` hash_unavailable). The asset " + "exists; do not run a workflow that loads it until it is checked.", + "for hash_mismatch delete the asset and upload again; for hash_unavailable compare the asset's hash " + "in `comfy assets library ls --name --tags models` with `details.hash`", + ), ErrorCode( "template_fetch_failed", "Fetching the per-template workflow JSON from `Comfy-Org/workflow_templates` failed.", diff --git a/comfy_cli/hashing.py b/comfy_cli/hashing.py index 235d3979d..6b40cbde2 100644 --- a/comfy_cli/hashing.py +++ b/comfy_cli/hashing.py @@ -1,5 +1,6 @@ +import hashlib from pathlib import Path -from typing import Final +from typing import Final, NamedTuple from blake3 import blake3 @@ -12,3 +13,32 @@ def blake3_file(path: Path) -> str: for chunk in iter(lambda: f.read(_CHUNK_SIZE), b""): h.update(chunk) return f"blake3:{h.hexdigest()}" + + +class FileDigests(NamedTuple): + """A file's hashes and size from one read, and the BLAKE3 digest of each block.""" + + blake3: str + sha256: str + size: int + block_size: int + blocks: tuple[bytes, ...] + + +def file_digests(path: Path, block_size: int | None = None) -> FileDigests: + """Hash the file once: ``blake3`` (``"blake3:"``), ``sha256`` (hex) and each block's BLAKE3 digest. + + BLAKE3 is the hash Comfy Cloud stores a model under; SHA-256 is the one a + person can check on any machine (``shasum -a 256``, ``certutil``). The + block digests let a later read of the same path be checked block by block, + before each block is used. + """ + size_of_block = block_size or _CHUNK_SIZE + b, s, size, blocks = blake3(), hashlib.sha256(), 0, [] + with path.open("rb") as f: + for chunk in iter(lambda: f.read(size_of_block), b""): + b.update(chunk) + s.update(chunk) + size += len(chunk) + blocks.append(blake3(chunk).digest()) + return FileDigests(f"blake3:{b.hexdigest()}", s.hexdigest(), size, size_of_block, tuple(blocks)) diff --git a/comfy_cli/schemas/assets_library.json b/comfy_cli/schemas/assets_library.json index 9dcf04015..67976a726 100644 --- a/comfy_cli/schemas/assets_library.json +++ b/comfy_cli/schemas/assets_library.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://comfy.org/schemas/assets_library.json", "title": "comfy assets library --json data payload", - "description": "Union schema for `assets library ls`, `assets library ensure` and `assets library upload`. `upload` returns the stored asset: `id`, `name`, `folder` (its model folder), `hash`, `size` (bytes) and `task_id` (the server task that stored it).", + "description": "Union schema for `assets library ls`, `assets library ensure` and `assets library upload`. `upload` returns the same fields as `comfy model upload` (see model_upload.json): `id`, `name`, `folder` (its model folder), `loader_path`, `hash` (BLAKE3, checked against Comfy Cloud's hash of the stored file), `sha256`, `size` (bytes), `workspace_id` (always null here), `task_id` (the server task that stored it; null when nothing was uploaded), `uploaded`, `already_present` and `dry_run`.", "type": "object", "additionalProperties": true, "properties": { @@ -33,6 +33,12 @@ "name": { "type": "string" }, "folder": { "type": "string" }, "size": { "type": "integer", "minimum": 1 }, - "task_id": { "type": "string" } + "task_id": { "type": ["string", "null"] }, + "loader_path": { "type": ["string", "null"] }, + "sha256": { "type": "string" }, + "workspace_id": { "type": ["string", "null"] }, + "uploaded": { "type": "boolean" }, + "already_present": { "type": "boolean" }, + "dry_run": { "type": "boolean" } } } diff --git a/comfy_cli/schemas/build_push_event.json b/comfy_cli/schemas/build_push_event.json index 052807ffe..031341494 100644 --- a/comfy_cli/schemas/build_push_event.json +++ b/comfy_cli/schemas/build_push_event.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://comfy.org/schemas/build_push_event.json", "title": "comfy build push upload progress event", - "description": "One JSON object per line while `comfy build push` uploads model files and packaged custom nodes, or `comfy assets library upload` uploads one model file. Under `--json-stream` the lines are on stdout, ahead of the final envelope (build_push.json schema in .data; assets_library.json for `assets library upload`). Under `--json` they are on stderr, so stdout stays the single envelope. A push opens with one `upload_plan`; each file that transfers gets an `upload_progress` when it starts and about every two seconds after, then one `upload_complete`. A file the builder already held gets only an `upload_complete` with `deduplicated: true`. A file whose upload fails gets no `upload_complete`: the error envelope follows.", + "description": "One JSON object per line while `comfy build push` uploads model files and packaged custom nodes, or `comfy model upload` / `comfy assets library upload` uploads one model file. Under `--json-stream` the lines are on stdout, ahead of the final envelope (build_push.json schema in .data; model_upload.json for `model upload`, assets_library.json for `assets library upload`). Under `--json` they are on stderr, so stdout stays the single envelope. A push opens with one `upload_plan`; each file that transfers gets an `upload_progress` when it starts and about every two seconds after, then one `upload_complete`. A file the builder already held gets only an `upload_complete` with `deduplicated: true`. A file whose upload fails gets no `upload_complete`: the error envelope follows.", "type": "object", "required": ["schema", "type"], "additionalProperties": true, diff --git a/comfy_cli/schemas/model_upload.json b/comfy_cli/schemas/model_upload.json new file mode 100644 index 000000000..107e0c392 --- /dev/null +++ b/comfy_cli/schemas/model_upload.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comfy.org/schemas/model_upload.json", + "title": "comfy model upload --json data payload", + "description": "One local .safetensors file and its place in the Comfy Cloud model library. A real upload succeeds only once Comfy Cloud's hash of the stored file equals `hash`. `uploaded` is true when this run added it; `already_present` is true when the library already held these bytes under this name and folder (nothing was uploaded); a `dry_run` uploads nothing and says which of the two a real run would do. `loader_path` is the value a loader widget in a workflow takes for the model on Comfy Cloud, as Comfy Cloud's own record of the model gives it; null for a dry run that would upload, or when Comfy Cloud gives none. `sha256` is the hash to show a person and pass back as `--expected-sha256`.", + "type": "object", + "additionalProperties": true, + "required": [ + "id", + "name", + "folder", + "loader_path", + "hash", + "sha256", + "size", + "workspace_id", + "task_id", + "uploaded", + "already_present", + "dry_run" + ], + "properties": { + "id": { "type": ["string", "null"], "description": "The asset id; null for a dry run of a file Comfy Cloud does not have yet." }, + "name": { "type": "string" }, + "folder": { "type": "string", "description": "The model folder, from --type." }, + "loader_path": { "type": ["string", "null"] }, + "hash": { "type": "string", "pattern": "^blake3:[0-9a-f]{64}$", "description": "The file's BLAKE3 hash, the one Comfy Cloud stores it under." }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "size": { "type": "integer", "minimum": 1 }, + "workspace_id": { "type": ["string", "null"], "description": "The workspace checked against --workspace; null when --workspace was not given." }, + "task_id": { "type": ["string", "null"], "description": "The server task that stored the file; null when nothing was uploaded." }, + "uploaded": { "type": "boolean" }, + "already_present": { "type": "boolean" }, + "dry_run": { "type": "boolean" } + } +} diff --git a/comfy_cli/tracking.py b/comfy_cli/tracking.py index 310af3154..5cbf83743 100644 --- a/comfy_cli/tracking.py +++ b/comfy_cli/tracking.py @@ -125,6 +125,12 @@ def _resolve_posthog_token() -> str: # `workflow add-node --text`: a note body is verbatim user content, like # `--prompt` — analytics only learns that the option was supplied. "text", + # `model upload --workspace` names a Comfy Cloud workspace, which can + # identify a customer; `--expected-sha256` names one of their files. + # The kwarg is `workspace_id`, not `workspace`, so `deploy --workspace` + # (a bool) stays measurable. + "workspace_id", + "expected_sha256", # `--models-dir` (repeatable, so a list: the whole list is redacted) and # `--out` are local paths, typed `str` like `--workflow`. "models_dir", diff --git a/docs/adr/TRANSFER-0011-upload-only-the-named-model-and-prove-cloud-stored-its-bytes.md b/docs/adr/TRANSFER-0011-upload-only-the-named-model-and-prove-cloud-stored-its-bytes.md new file mode 100644 index 000000000..d86dd9f9e --- /dev/null +++ b/docs/adr/TRANSFER-0011-upload-only-the-named-model-and-prove-cloud-stored-its-bytes.md @@ -0,0 +1,139 @@ +# ADR-TRANSFER-0011: Upload Only the Named Model and Prove Comfy Cloud Stored Its Bytes + +Date: 2026-11-05 + +## Status + +Proposed + +Relation: Builds on ADR-AUTOMATION-0003. `comfy model upload` and +`comfy assets library upload` keep the `envelope/1` output and add only fields, +a schema and registered error codes. + +## Context + +A person has a ComfyUI workflow that runs on their machine and wants it to run +on Comfy Cloud. The workflow loads `my_style.safetensors` from `loras`, and +Comfy Cloud has no file by that name. An agent helping them finds the gap and +asks: "Upload my_style.safetensors (151 MB) to loras on Comfy Cloud?" The person +says yes. The agent then runs `comfy model upload` for that one file. + +Several things can go wrong between that yes and a model Comfy Cloud loads: + +- The agent uploads a file the person did not name, such as a whole folder or a + file found by a search. +- The file changes between the question and the upload, so Comfy Cloud gets + bytes the person never approved. +- The sign-in moves to another workspace, so the file lands in a library the + person did not pick. +- The bytes are cut or altered on the way, and Comfy Cloud stores something + other than the local file. +- The folder already holds a different file of the same name. A Comfy Cloud + run loads a model by folder and name, so a second file would change what + that name loads for every workflow in the workspace. + +Draft PR #1008 added an upload command with an approval step. This decision +keeps its approval idea and builds on the presigned upload already in +`comfy assets library upload`. + +## Decision + +- The command uploads exactly one file, the path given as its argument. It never + expands a folder, a glob or a search into more files. +- The CLI reads the file once before anything is sent and computes its blake3 + (Comfy Cloud's hash), its sha256 (what `shasum -a 256` prints on any + machine) and the blake3 of each 1 MiB block. A dry run reports the two file + hashes and uploads nothing. +- An approval is pinned to bytes and a workspace. `--expected-sha256` uploads + nothing unless the file still has that sha256. `--workspace` uploads nothing + unless `GET /api/workspaces/current` says this sign-in uploads into that + workspace; no answer counts as a mismatch. +- `comfy model upload` asks before it starts. A machine caller must pass + `--yes`, through the shared `confirm` helper, or it is refused with + `model_upload_needs_confirm`. +- Before sending, the CLI lists the models with that exact name in that folder + of the workspace's model library (`workspace_library=true`): every member's + models, and no public ones. A run on Comfy Cloud loads from that same set. + The same blake3 there means nothing is sent, and the output says + `already_present`. For `comfy model upload`, any other hash, or no hash, + refuses with `name_in_use`. `comfy assets library upload` uploads the file + anyway, as before this decision, so a script that replaces a model by + uploading a new version under its name keeps working, and reports + `already_present` only when the same bytes are the only model of that name. + For `comfy model upload`, a listing that may be cut short refuses with + `name_unverified`; `comfy assets library upload` uploads. +- The upload sends only bytes that were hashed. It reads the file again one + block at a time and checks each block against its blake3 from the first read + before any of that block is sent. A changed block stops the upload, so none + of the changed bytes reach storage, and complete is never called, so Comfy + Cloud never files the earlier blocks. Reading stops at the hashed size. +- The upload succeeds only when the hash Comfy Cloud reports for the stored + asset equals the local blake3. The task result is read first, then the asset. + A different hash, or none, ends as `asset_upload_unverified`. +- The output's `loader_path` is the value a loader node takes to load the file + on Comfy Cloud. The CLI reports only Comfy Cloud's own value: from the + matching row when the bytes are already present, and from the asset's record, + read after the hash check, when they were uploaded. It is null when Comfy + Cloud gives none, when that read fails, and for a dry run that would upload. + Comfy Cloud gives null when the name does not form a valid model path, and + its own model list then leaves the model out, so the CLI does not put the + file name in its place. A failed read does not fail an upload whose hash + already matched. + +## Alternatives considered + +- Upload with a multipart `POST` through the API server, as #1008 does. + Rejected because the presigned upload sends bytes straight to storage and + already has retries, progress and token refresh. +- Copy the file to a temporary snapshot before uploading, as #1008 does. + Rejected because a model can be several GB. The block check sends only + hashed bytes with no second copy. +- Hash the bytes as they stream and compare once the PUT ends. Rejected + because the changed bytes are already in storage by then. +- Take `--name` to upload under another name. Left out because a workflow + names the file it loads, and a different name would not match it. +- Trust the upload once storage accepts it. Rejected because only Comfy + Cloud's hash of the stored asset shows the right bytes landed. + +## What we reject + +- Any upload of a file the caller did not name on the command line. +- A second file under a folder and name already in use in the library, from + `comfy model upload`. +- Reporting success before Comfy Cloud's stored hash equals the local hash. + +## Consequences + +### Positive + +- An agent can ask a person about one file by name, show its sha256, and + upload exactly those bytes into the workspace they approved. +- Running the upload again for a file Comfy Cloud already has sends nothing. + +### Negative + +- Every upload reads the file twice: once to hash it and once to send it. The + second read hashes each block again. +- A Comfy Cloud server without `/api/workspaces/current` refuses every + `--workspace` upload. + +### Known limits + +- Two uploads of different bytes under one name can both pass the name check + if each lists the folder before the other completes. Only a Comfy Cloud write + that refuses a name already in use can stop that. + +## Validation + +`tests/comfy_cli/command/test_assets_library.py` covers each refusal above, the +lookup's workspace-library query, the skip for bytes already present, a file +changed after it was hashed sending none of the changed bytes and never +completing, a stored hash that differs or is missing, the `--yes` refusal for an +agent caller, and a `loader_path` that is null when Comfy Cloud gives none or +its record cannot be read after the upload. + +## Revisit triggers + +- Comfy Cloud loads models by hash instead of by folder and name. +- Comfy Cloud stops sending `loader_path` on asset records, so every upload + reports null. diff --git a/docs/json-output.md b/docs/json-output.md index ca5fa4afb..0c9c8ddc2 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -490,8 +490,9 @@ time. If the callback never arrives, the terminal envelope is an ### `upload_plan`, `upload_progress`, `upload_complete` Emitted by `comfy build push` while it uploads model files and packaged custom -nodes. They are part of the push, not the `run` stream, and validate against -`build_push_event.json`. +nodes, and by `comfy model upload` and `comfy assets library upload` while they +upload one model file. They are part of the upload, not the `run` stream, and +validate against `build_push_event.json`. Where they go depends on the mode. Under `--json-stream` they are on stdout, ahead of the envelope, like every other event. Under plain `--json` (which is diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 68e9306e3..b9ae2bbd4 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -22,6 +22,7 @@ from __future__ import annotations import base64 +import hashlib import http.client import io import json @@ -33,6 +34,7 @@ from typing import Any import pytest +from blake3 import blake3 from typer.testing import CliRunner from comfy_cli import error_codes @@ -761,11 +763,17 @@ def test_bare_uppercase_digest_is_sent_unchanged(self, cloud_target, monkeypatch "x-goog-if-generation-match": "0", } _FOLDERS = {"model_upload_folders": ["checkpoints", "embeddings", "loras", "vae"]} +# What the `model_file` fixture holds, and its hashes. +_MODEL_BYTES = b"0123456789abcdef" +_MODEL_HASH = "blake3:" + blake3(_MODEL_BYTES).hexdigest() +_MODEL_SHA256 = hashlib.sha256(_MODEL_BYTES).hexdigest() _DONE = { "id": "task-1", "status": "completed", - "result": {"success": True, "asset_id": "asset-9", "hash": "blake3:" + "cd" * 32, "filename": "style.safetensors"}, + "result": {"success": True, "asset_id": "asset-9", "hash": _MODEL_HASH, "filename": "style.safetensors"}, } +# No model in the folder has the file's name yet. +_NONE_NAMED = {"assets": [], "has_more": False} def _started(n: int = 1) -> dict: @@ -839,6 +847,25 @@ class _Resp: return _Resp() +class _SentSoFar(_Storage): + """A ``_Storage`` that records each block as it reads it, so a PUT whose body read + raises still shows the bytes a socket would already have sent.""" + + def __call__(self, url, data=None, headers=None, timeout=None, allow_redirects=True): + put = {"url": url, "headers": headers, "body": b"", "allow_redirects": allow_redirects} + self.puts.append(put) + while block := data.read(4): + put["body"] += block + outcome = self.outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + + class _Resp: + status_code = outcome + + return _Resp() + + def _cloud_error(code: int, server_code: str, message: str): body = json.dumps({"code": server_code, "message": message}).encode() return urllib.error.HTTPError("https://cloud.example.com/api/x", code, "err", {}, io.BytesIO(body)) @@ -847,7 +874,7 @@ def _cloud_error(code: int, server_code: str, message: str): @pytest.fixture def model_file(tmp_path): path = tmp_path / "style.safetensors" - path.write_bytes(b"0123456789abcdef") + path.write_bytes(_MODEL_BYTES) return path @@ -867,6 +894,7 @@ def _upload( puts: list, cloud: _Cloud | None = None, storage: _Storage | None = None, + app: Any = None, ) -> tuple[dict, int, _Cloud, _Storage, str]: cloud, storage = cloud or _Cloud(routes), storage or _Storage(puts) monkeypatch.setattr("urllib.request.urlopen", cloud) @@ -876,7 +904,7 @@ def _upload( ) r.mode = OutputMode.JSON set_renderer(r) - result = CliRunner().invoke(assets_library.app, ["upload", *args, "--where", "cloud"], standalone_mode=False) + result = CliRunner().invoke(app or assets_library.app, ["upload", *args, "--where", "cloud"], standalone_mode=False) captured = capsys.readouterr() everything = captured.out + captured.err + result.output rc = result.exit_code or (result.return_value if isinstance(result.return_value, int) else 0) @@ -887,6 +915,7 @@ def _upload( _UPLOAD_ROUTES = { ("GET", "/api/features"): _FOLDERS, + ("GET", "/api/assets"): _NONE_NAMED, ("POST", "/api/assets/uploads"): [_started(1), _started(2)], ("POST", "/api/assets/uploads/00000000-0000-0000-0000-000000000001/complete"): { "task_id": "task-1", @@ -897,6 +926,8 @@ def _upload( "status": "created", }, ("GET", "/api/tasks/task-1"): [{"id": "task-1", "status": "running"}, _DONE], + # Read after the hash check, for Comfy Cloud's loader_path. + ("GET", "/api/assets/asset-9"): {"id": "asset-9", "hash": _MODEL_HASH, "loader_path": "style.safetensors"}, } @@ -915,9 +946,15 @@ def test_uploads_completes_and_reports_the_stored_asset( "id": "asset-9", "name": "style.safetensors", "folder": "loras", - "hash": "blake3:" + "cd" * 32, + "loader_path": "style.safetensors", + "hash": _MODEL_HASH, + "sha256": _MODEL_SHA256, "size": 16, + "workspace_id": None, "task_id": "task-1", + "uploaded": True, + "already_present": False, + "dry_run": False, } schema = json.loads( (Path(assets_library.__file__).resolve().parents[1] / "schemas" / "assets_library.json").read_text() @@ -925,13 +962,15 @@ def test_uploads_completes_and_reports_the_stored_asset( jsonschema.Draft202012Validator(schema).validate(env["data"]) assert cloud.paths() == [ "/api/features", + "/api/assets", "/api/assets/uploads", "/api/assets/uploads/00000000-0000-0000-0000-000000000001/complete", "/api/tasks/task-1", "/api/tasks/task-1", + "/api/assets/asset-9", ] - assert cloud.calls[1][2] == {"filename": "style.safetensors", "size_bytes": 16, "tags": ["models", "loras"]} - assert cloud.calls[2][2] == {"filename": "style.safetensors", "tags": ["models", "loras"]} + assert cloud.calls[2][2] == {"filename": "style.safetensors", "size_bytes": 16, "tags": ["models", "loras"]} + assert cloud.calls[3][2] == {"filename": "style.safetensors", "tags": ["models", "loras"]} (put,) = storage.puts assert put["url"] == f"{_SIGNED}1" and put["headers"] == _PUT_HEADERS and put["allow_redirects"] is False assert put["body"] == model_file.read_bytes() @@ -985,6 +1024,7 @@ def test_unknown_type_is_refused_listing_the_valid_ones_before_the_upload( def test_without_upload_folders_the_start_call_decides(self, cloud_target, monkeypatch, capsys, model_file): routes = { ("GET", "/api/features"): {"max_upload_size": 1}, + ("GET", "/api/assets"): _NONE_NAMED, ("POST", "/api/assets/uploads"): _cloud_error( 403, "FEATURE_DISABLED", "Uploading a model file is not enabled for this workspace" ), @@ -1124,7 +1164,15 @@ def test_an_unusable_answer_is_an_error_not_a_success( assert rc == 1 and env["ok"] is False and env["error"]["code"] == "cloud_http_error" def test_new_codes_are_registered(self): - for code in ("asset_upload_invalid", "asset_upload_refused", "asset_upload_failed", "asset_upload_timeout"): + for code in ( + "asset_upload_invalid", + "asset_upload_refused", + "asset_upload_failed", + "asset_upload_timeout", + "asset_upload_unverified", + "asset_upload_wrong_workspace", + "model_upload_needs_confirm", + ): assert error_codes.is_registered(code), code @@ -1193,7 +1241,7 @@ class TestUploadOutlastsTheToken: def test_a_401_on_complete_refreshes_the_sign_in_and_completes( self, signed_in, monkeypatch, capsys, model_file, no_wait ): - cloud = _ExpiringCloud(dict(_UPLOAD_ROUTES), expires_at=2) # features, start, then expired + cloud = _ExpiringCloud(dict(_UPLOAD_ROUTES), expires_at=3) # features, lookup, start, then expired env, rc, _, storage, _ = _upload( monkeypatch, capsys, [str(model_file), "--type", "loras"], {}, [200], cloud=cloud ) @@ -1202,12 +1250,12 @@ def test_a_401_on_complete_refreshes_the_sign_in_and_completes( assert signed_in.forced == 1 assert len(storage.puts) == 1 assert cloud.paths().count(_COMPLETE_1) == 2 - assert cloud.auth[2:] == ["Bearer old-token"] + ["Bearer new-token"] * 3 + assert cloud.auth[3:] == ["Bearer old-token"] + ["Bearer new-token"] * 4 def test_a_401_on_a_task_poll_refreshes_the_sign_in_and_keeps_waiting( self, signed_in, monkeypatch, capsys, model_file, no_wait ): - cloud = _ExpiringCloud(dict(_UPLOAD_ROUTES), expires_at=3) # expires after complete + cloud = _ExpiringCloud(dict(_UPLOAD_ROUTES), expires_at=4) # expires after complete env, rc, _, storage, _ = _upload( monkeypatch, capsys, [str(model_file), "--type", "loras"], {}, [200], cloud=cloud ) @@ -1221,7 +1269,7 @@ def test_an_api_key_is_not_refreshed_and_its_401_surfaces( self, cloud_target, signed_in, monkeypatch, capsys, model_file, no_wait ): monkeypatch.setattr("comfy_cli.target.resolve_target", lambda **kw: cloud_target) - cloud = _ExpiringCloud(dict(_UPLOAD_ROUTES), expires_at=2) + cloud = _ExpiringCloud(dict(_UPLOAD_ROUTES), expires_at=3) env, rc, _, storage, _ = _upload( monkeypatch, capsys, [str(model_file), "--type", "loras"], {}, [200], cloud=cloud ) @@ -1234,7 +1282,7 @@ def test_a_token_handed_in_by_env_is_not_refreshed_even_beside_a_matching_sign_i self, signed_in, monkeypatch, capsys, model_file, no_wait ): monkeypatch.setenv("COMFY_CLOUD_AUTH_TOKEN", "old-token") - cloud = _ExpiringCloud(dict(_UPLOAD_ROUTES), expires_at=2) + cloud = _ExpiringCloud(dict(_UPLOAD_ROUTES), expires_at=3) env, rc, _, storage, _ = _upload( monkeypatch, capsys, [str(model_file), "--type", "loras"], {}, [200], cloud=cloud ) @@ -1335,7 +1383,7 @@ class TestUploadKeepsItsAccount: """ @pytest.mark.parametrize( - "lands_during", [pytest.param(1, id="before-complete"), pytest.param(3, id="during-polling")] + "lands_during", [pytest.param(2, id="before-complete"), pytest.param(4, id="during-polling")] ) def test_another_login_does_not_change_the_upload_owner( self, stored_login, monkeypatch, capsys, model_file, no_wait, lands_during @@ -1346,13 +1394,13 @@ def test_another_login_does_not_change_the_upload_owner( ) assert rc == 0 and env["data"]["id"] == "asset-9" - assert cloud.auth == [f"Bearer {_jwt('a')}"] * 5 + assert cloud.auth == [f"Bearer {_jwt('a')}"] * 7 assert stored_login.login == ("b", 1) and stored_login.forced == 0 assert len(storage.puts) == 1 @pytest.mark.parametrize( ("lands_during", "a_expires_from", "operation"), - [pytest.param(1, 2, "complete", id="at-complete"), pytest.param(3, 4, "task", id="during-polling")], + [pytest.param(2, 3, "complete", id="at-complete"), pytest.param(4, 5, "task", id="during-polling")], ) def test_an_expired_token_with_another_login_stored_ends_with_a_clear_error( self, stored_login, monkeypatch, capsys, model_file, no_wait, lands_during, a_expires_from, operation @@ -1375,19 +1423,19 @@ def test_an_expired_token_with_another_login_stored_ends_with_a_clear_error( def test_the_same_login_refreshed_in_another_shell_is_picked_up( self, stored_login, monkeypatch, capsys, model_file, no_wait ): - cloud = _AccountCloud(dict(_UPLOAD_ROUTES), stored_login, lands=("a", 2), lands_during=1, a_expires_from=2) + cloud = _AccountCloud(dict(_UPLOAD_ROUTES), stored_login, lands=("a", 2), lands_during=2, a_expires_from=3) env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], {}, [200], cloud=cloud) assert rc == 0 and env["data"]["id"] == "asset-9" - assert cloud.auth[2:] == [f"Bearer {_jwt('a', 2)}"] * 3 + assert cloud.auth[3:] == [f"Bearer {_jwt('a', 2)}"] * 4 assert stored_login.forced == 0 def test_the_same_login_is_refreshed_after_a_401(self, stored_login, monkeypatch, capsys, model_file, no_wait): - cloud = _AccountCloud(dict(_UPLOAD_ROUTES), stored_login, a_expires_from=2) + cloud = _AccountCloud(dict(_UPLOAD_ROUTES), stored_login, a_expires_from=3) env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], {}, [200], cloud=cloud) assert rc == 0 and env["data"]["id"] == "asset-9" - assert cloud.auth[2:] == [f"Bearer {_jwt('a')}"] + [f"Bearer {_jwt('a', 2)}"] * 3 + assert cloud.auth[3:] == [f"Bearer {_jwt('a')}"] + [f"Bearer {_jwt('a', 2)}"] * 4 assert stored_login.forced == 1 @pytest.mark.parametrize("explicit", ["env-token", "api-key"]) @@ -1401,11 +1449,11 @@ def test_an_explicit_credential_is_sent_unchanged_when_another_login_lands( else: monkeypatch.setattr("comfy_cli.target.resolve_target", lambda **kw: cloud_target) sent = "test-key" - cloud = _AccountCloud(dict(_UPLOAD_ROUTES), stored_login, lands_during=1) + cloud = _AccountCloud(dict(_UPLOAD_ROUTES), stored_login, lands_during=2) env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], {}, [200], cloud=cloud) assert rc == 0 and env["data"]["id"] == "asset-9" - assert cloud.auth == [sent] * 5 + assert cloud.auth == [sent] * 7 assert stored_login.forced == 0 @@ -1665,33 +1713,6 @@ def __call__(self, url, data=None, **kw): assert cloud.paths("POST").count("/api/assets/uploads") == 1 assert "SECRETSIG" not in everything - def test_a_file_error_while_sizing_the_body_fails_without_a_retry( - self, cloud_target, monkeypatch, capsys, model_file, no_wait - ): - # The body is sized from the open handle before Requests sees it; on a - # network filesystem that can raise a stale handle before any read. - import errno - - class _Os: - def __getattr__(self, name): - return getattr(os, name) - - def fstat(self, fd): - raise OSError(errno.ESTALE, "Stale NFS file handle") - - monkeypatch.setattr(assets_library, "os", _Os()) - env, rc, cloud, storage, everything = _upload( - monkeypatch, capsys, [str(model_file), "--type", "loras"], dict(_UPLOAD_ROUTES), [200, 200] - ) - - assert rc == 1 and env["error"]["code"] == "asset_upload_failed" - assert env["error"]["details"]["reason"] == "file_unreadable" - assert env["error"]["details"]["attempts"] == 1 - assert "Stale NFS file handle" in env["error"]["message"] - assert storage.puts == [] - assert cloud.paths("POST").count("/api/assets/uploads") == 1 - assert "SECRETSIG" not in everything - def test_requests_sends_the_body_with_its_content_length_not_chunked( self, cloud_target, monkeypatch, capsys, model_file, no_wait ): @@ -1850,3 +1871,480 @@ def test_the_status_is_classified_when_the_error_body_is_cut_off( assert rc == 1 and env["error"]["code"] == code assert env["error"]["details"]["status"] == status + + +# --- the upload checks the bytes, the name and the workspace ------------------ + + +def _row(name: str = "style.safetensors", hash_: str = _MODEL_HASH, asset_id: str = "asset-3") -> dict: + return {"id": asset_id, "name": name, "hash": hash_, "tags": ["models", "loras"], "loader_path": name} + + +class _UrlCloud(_Cloud): + """``_Cloud`` that also keeps each request's full URL.""" + + def __init__(self, routes: dict): + super().__init__(routes) + self.urls: list[str] = [] + + def __call__(self, req, timeout=None): + self.urls.append(req.full_url) + return super().__call__(req, timeout) + + +class TestUploadVerifies: + def test_a_dry_run_reports_the_hashes_and_sends_nothing(self, cloud_target, monkeypatch, capsys, model_file): + env, rc, cloud, storage, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras", "--dry-run"], dict(_UPLOAD_ROUTES), [] + ) + + assert rc == 0 and env["ok"] is True and env["changed"] is False + assert env["data"] == { + "id": None, + "name": "style.safetensors", + "folder": "loras", + # No record exists yet, so Comfy Cloud has no loader_path to give. + "loader_path": None, + "hash": _MODEL_HASH, + "sha256": _MODEL_SHA256, + "size": 16, + "workspace_id": None, + "task_id": None, + "uploaded": False, + "already_present": False, + "dry_run": True, + } + assert cloud.paths() == ["/api/features", "/api/assets"] and storage.puts == [] + + def test_the_lookup_asks_for_the_models_of_that_name_in_that_folder( + self, cloud_target, monkeypatch, capsys, model_file + ): + import urllib.parse + + cloud = _UrlCloud(dict(_UPLOAD_ROUTES)) + _upload(monkeypatch, capsys, [str(model_file), "--type", "loras", "--dry-run"], {}, [], cloud=cloud) + + query = urllib.parse.parse_qs(urllib.parse.urlsplit(cloud.urls[1]).query) + assert query == { + "include_tags": ["models,loras"], + "name_contains": ["style.safetensors"], + # The workspace's library, every member's models, is what a run + # loads from; the public catalog is not. + "workspace_library": ["true"], + "include_public": ["false"], + "limit": ["500"], + } + + @pytest.mark.parametrize("dry_run", [False, True]) + def test_the_same_bytes_under_the_name_are_not_uploaded_again( + self, cloud_target, monkeypatch, capsys, model_file, dry_run + ): + routes = dict(_UPLOAD_ROUTES) + stored_form = _MODEL_HASH.removeprefix("blake3:").upper() + ".safetensors" + routes[("GET", "/api/assets")] = { + # A longer name that only contains the file's name is not the file's. + "assets": [_row(hash_=stored_form), _row("old-style.safetensors", "blake3:" + "ab" * 32, "asset-4")], + "has_more": False, + } + args = [str(model_file), "--type", "loras"] + (["--dry-run"] if dry_run else []) + env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, args, routes, []) + + assert rc == 0 and env["changed"] is False + assert env["data"]["id"] == "asset-3" and env["data"]["loader_path"] == "style.safetensors" + assert env["data"]["already_present"] is True and env["data"]["uploaded"] is False + assert env["data"]["dry_run"] is dry_run + assert cloud.paths() == ["/api/features", "/api/assets"] and storage.puts == [] + + def test_a_name_already_holding_other_bytes_is_refused(self, cloud_target, monkeypatch, capsys, model_file): + routes = dict(_UPLOAD_ROUTES) + other = "blake3:" + "ab" * 32 + routes[("GET", "/api/assets")] = {"assets": [_row(hash_=other), _row()], "has_more": False} + args = [str(model_file), "--type", "loras", "--yes"] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, routes, []) + + assert rc == 1 and env["error"]["code"] == "asset_upload_invalid" + assert env["error"]["details"]["reason"] == "name_in_use" + assert env["error"]["details"]["hash"] == _MODEL_HASH + assert env["error"]["details"]["existing"] == [{"id": "asset-3", "hash": other}] + assert cloud.paths() == ["/api/features", "/api/assets"] and storage.puts == [] + + def test_a_row_with_no_hash_counts_as_other_bytes(self, cloud_target, monkeypatch, capsys, model_file): + routes = dict(_UPLOAD_ROUTES) + routes[("GET", "/api/assets")] = {"assets": [_row(hash_=None)], "has_more": False} + args = [str(model_file), "--type", "loras", "--yes"] + env, rc, _, storage, _ = _model_upload(monkeypatch, capsys, args, routes, []) + + assert rc == 1 and env["error"]["details"]["reason"] == "name_in_use" and storage.puts == [] + + def test_assets_library_upload_still_uploads_a_new_version_of_a_name( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + # A script that replaces a model by uploading new bytes under its name keeps working. + routes = dict(_UPLOAD_ROUTES) + routes[("GET", "/api/assets")] = {"assets": [_row(hash_="blake3:" + "ab" * 32)], "has_more": False} + env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + assert rc == 0 and env["ok"] is True and env["changed"] is True + assert env["data"]["id"] == "asset-9" and env["data"]["uploaded"] is True + assert env["data"]["already_present"] is False + (put,) = storage.puts + assert put["body"] == model_file.read_bytes() + assert "/api/assets/uploads/00000000-0000-0000-0000-000000000001/complete" in cloud.paths() + + def test_assets_library_upload_sends_the_same_bytes_again_over_a_newer_version( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + # The name also holds other bytes, so only a new upload makes this + # file the one the name loads; it is not "already present". + routes = dict(_UPLOAD_ROUTES) + other = "blake3:" + "ab" * 32 + routes[("GET", "/api/assets")] = {"assets": [_row(hash_=other, asset_id="asset-4"), _row()], "has_more": False} + env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + assert rc == 0 and env["data"]["uploaded"] is True and env["data"]["already_present"] is False + (put,) = storage.puts + assert put["body"] == model_file.read_bytes() + + def test_a_lookup_too_long_to_settle_the_name_is_refused_for_model_upload( + self, cloud_target, monkeypatch, capsys, model_file + ): + routes = dict(_UPLOAD_ROUTES) + routes[("GET", "/api/assets")] = {"assets": [], "has_more": True} + args = [str(model_file), "--type", "loras", "--yes"] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, routes, []) + + assert rc == 1 and env["error"]["code"] == "asset_upload_invalid" + assert env["error"]["details"]["reason"] == "name_unverified" + assert "/api/assets/uploads" not in cloud.paths() and storage.puts == [] + + def test_assets_library_upload_uploads_when_the_lookup_is_too_long_to_settle_the_name( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + routes = dict(_UPLOAD_ROUTES) + routes[("GET", "/api/assets")] = {"assets": [_row()], "has_more": True} + env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + assert rc == 0 and env["data"]["uploaded"] is True and env["data"]["already_present"] is False + (put,) = storage.puts + assert put["body"] == model_file.read_bytes() + + def test_an_unusable_lookup_is_an_error_not_a_go_ahead(self, cloud_target, monkeypatch, capsys, model_file): + routes = dict(_UPLOAD_ROUTES) + routes[("GET", "/api/assets")] = {"assets": "none"} + env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, []) + + assert rc == 1 and env["error"]["code"] == "cloud_http_error" and storage.puts == [] + + def test_a_file_changed_after_it_was_hashed_sends_none_of_the_change( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + routes = dict(_UPLOAD_ROUTES) + # Same size, other bytes: written after the hash and before the PUT reads it. + rewrite = _then(lambda: model_file.write_bytes(b"fedcba9876543210"), _started(1)) + routes[("POST", "/api/assets/uploads")] = [rewrite, _started(2)] + storage = _SentSoFar([200, 200]) + env, rc, cloud, storage, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=storage + ) + + assert rc == 1 and env["error"]["code"] == "asset_upload_failed" + details = env["error"]["details"] + assert details["reason"] == "file_changed" and details["hash"] == _MODEL_HASH + assert details["attempts"] == 1 + # The one block is checked before any of it is sent. + assert [p["body"] for p in storage.puts] == [b""] + assert cloud.paths("POST").count("/api/assets/uploads") == 1 + assert not any(p.endswith("/complete") for p in cloud.paths()) + + def test_only_the_blocks_before_a_changed_one_are_sent( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + from comfy_cli import hashing + + monkeypatch.setattr(hashing, "_CHUNK_SIZE", 4) # blocks 0123 4567 89ab cdef + routes = dict(_UPLOAD_ROUTES) + rewrite = _then(lambda: model_file.write_bytes(b"01234567XXXXcdef"), _started(1)) + routes[("POST", "/api/assets/uploads")] = [rewrite, _started(2)] + storage = _SentSoFar([200, 200]) + env, rc, cloud, storage, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=storage + ) + + assert rc == 1 and env["error"]["details"]["reason"] == "file_changed" + assert [p["body"] for p in storage.puts] == [b"01234567"] + assert not any(p.endswith("/complete") for p in cloud.paths()) + + def test_a_file_cut_short_after_it_was_hashed_is_not_completed( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + from comfy_cli import hashing + + monkeypatch.setattr(hashing, "_CHUNK_SIZE", 4) + routes = dict(_UPLOAD_ROUTES) + routes[("POST", "/api/assets/uploads")] = [_then(lambda: model_file.write_bytes(b"0123456789"), _started(1))] + storage = _SentSoFar([200]) + env, rc, cloud, storage, _ = _upload( + monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [], storage=storage + ) + + assert rc == 1 and env["error"]["details"]["reason"] == "file_changed" + assert [p["body"] for p in storage.puts] == [b"01234567"] + assert not any(p.endswith("/complete") for p in cloud.paths()) + + def test_bytes_added_after_the_hash_are_not_sent(self, cloud_target, monkeypatch, capsys, model_file, no_wait): + routes = dict(_UPLOAD_ROUTES) + grow = _then(lambda: model_file.write_bytes(_MODEL_BYTES + b"tail"), _started(1)) + routes[("POST", "/api/assets/uploads")] = [grow, _started(2)] + env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + # The hashed bytes are whole, so they are the upload; the added tail stays here. + assert rc == 0 and env["data"]["hash"] == _MODEL_HASH and env["data"]["size"] == len(_MODEL_BYTES) + assert [p["body"] for p in storage.puts] == [_MODEL_BYTES] + + def test_a_stored_hash_that_is_not_the_files_fails_the_upload( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + routes = dict(_UPLOAD_ROUTES) + wrong = "blake3:" + "cd" * 32 + routes[("GET", "/api/tasks/task-1")] = {**_DONE, "result": {**_DONE["result"], "hash": wrong}} + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + assert rc == 1 and env["error"]["code"] == "asset_upload_unverified" + assert env["error"]["details"] == { + "reason": "hash_mismatch", + "asset_id": "asset-9", + "task_id": "task-1", + "hash": _MODEL_HASH, + "stored_hash": wrong, + } + + def test_a_task_with_no_hash_is_checked_against_the_stored_asset( + self, cloud_target, monkeypatch, capsys, model_file, no_wait + ): + routes = dict(_UPLOAD_ROUTES) + result = {k: v for k, v in _DONE["result"].items() if k != "hash"} + routes[("GET", "/api/tasks/task-1")] = {**_DONE, "result": result} + routes[("GET", "/api/assets/asset-9")] = { + "id": "asset-9", + "hash": _MODEL_HASH.removeprefix("blake3:"), + "loader_path": "style.safetensors", + } + env, rc, cloud, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + assert rc == 0 and env["data"]["uploaded"] is True and env["data"]["loader_path"] == "style.safetensors" + assert cloud.paths()[-1] == "/api/assets/asset-9" + assert cloud.paths().count("/api/assets/asset-9") == 1 + + @pytest.mark.parametrize("answer", [_http_error(500), b"not json"], ids=["asset-unreadable", "asset-unparseable"]) + def test_a_record_that_cannot_be_read_after_a_verified_upload_leaves_loader_path_null( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, answer + ): + routes = dict(_UPLOAD_ROUTES) + routes[("GET", "/api/assets/asset-9")] = answer + env, rc, cloud, storage, _ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + # The task's hash already matched, so the bytes are verified; only the loader_path is missing. + assert rc == 0 and env["ok"] is True and env["data"]["uploaded"] is True + assert env["data"]["id"] == "asset-9" and env["data"]["loader_path"] is None + assert cloud.paths()[-1] == "/api/assets/asset-9" and len(storage.puts) == 1 + + @pytest.mark.parametrize("present", [False, True], ids=["uploaded", "already-present"]) + def test_a_record_without_loader_path_reports_null_not_the_file_name( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, present + ): + import jsonschema + + # Comfy Cloud gives no loader_path when the name does not form a valid model path. + routes = dict(_UPLOAD_ROUTES) + routes[("GET", "/api/assets/asset-9")] = {"id": "asset-9", "hash": _MODEL_HASH, "loader_path": None} + if present: + row = {k: v for k, v in _row().items() if k != "loader_path"} + routes[("GET", "/api/assets")] = {"assets": [row], "has_more": False} + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + assert rc == 0 and env["data"]["loader_path"] is None + assert env["data"]["already_present"] is present and env["data"]["uploaded"] is not present + jsonschema.Draft202012Validator(_model_schema()).validate(env["data"]) + + @pytest.mark.parametrize( + "answer", [_http_error(500), {"id": "asset-9"}], ids=["asset-unreadable", "asset-without-hash"] + ) + def test_a_stored_hash_that_cannot_be_read_fails_the_upload( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, answer + ): + routes = dict(_UPLOAD_ROUTES) + result = {k: v for k, v in _DONE["result"].items() if k != "hash"} + routes[("GET", "/api/tasks/task-1")] = {**_DONE, "result": result} + routes[("GET", "/api/assets/asset-9")] = answer + env, rc, *_ = _upload(monkeypatch, capsys, [str(model_file), "--type", "loras"], routes, [200]) + + assert rc == 1 and env["error"]["code"] == "asset_upload_unverified" + assert env["error"]["details"]["reason"] == "hash_unavailable" + assert env["error"]["details"]["stored_hash"] is None + + @pytest.mark.parametrize( + ("value", "expected"), + [ + (_MODEL_HASH, _MODEL_HASH), + (_MODEL_HASH.removeprefix("blake3:").upper(), _MODEL_HASH), + (_MODEL_HASH.removeprefix("blake3:") + ".safetensors", _MODEL_HASH), + ("blake3:abc", None), + (None, None), + (42, None), + ], + ) + def test_content_hash_reads_every_form_comfy_cloud_reports(self, value, expected): + assert assets_library._content_hash(value) == expected + + +# --- comfy model upload ------------------------------------------------------- + +_WORKSPACE = {"id": "ws-1", "name": "Studio", "type": "team", "auth_method": "api_key"} + + +def _model_upload(monkeypatch, capsys, args: list[str], routes: dict, puts: list, **kw): + from comfy_cli.command.models import models + + return _upload(monkeypatch, capsys, args, routes, puts, app=models.app, **kw) + + +@pytest.fixture +def agent_caller(monkeypatch): + """The caller is an agent: nothing can answer a prompt.""" + monkeypatch.setattr( + "comfy_cli.interaction.detect_caller", lambda: Caller(kind="agent", agentic=True, source_env="CLAUDECODE") + ) + monkeypatch.setattr("comfy_cli.interaction._skip_prompt_flag", lambda: True) + + +def _model_schema() -> dict: + path = Path(assets_library.__file__).resolve().parents[1] / "schemas" / "model_upload.json" + return json.loads(path.read_text()) + + +class TestModelUpload: + def test_the_dry_run_names_the_workspace_and_the_sha256_to_approve( + self, cloud_target, monkeypatch, capsys, model_file, agent_caller + ): + import jsonschema + + routes = {**_UPLOAD_ROUTES, ("GET", "/api/workspaces/current"): _WORKSPACE} + args = [str(model_file), "--type", "loras", "--workspace", "ws-1", "--dry-run"] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, routes, []) + + assert rc == 0 and env["command"] == "model upload" + data = env["data"] + assert data["sha256"] == _MODEL_SHA256 and data["hash"] == _MODEL_HASH + assert data["workspace_id"] == "ws-1" and data["loader_path"] is None + assert data["dry_run"] is True and data["uploaded"] is False and data["already_present"] is False + jsonschema.Draft202012Validator(_model_schema()).validate(data) + assert cloud.paths() == ["/api/workspaces/current", "/api/features", "/api/assets"] + assert storage.puts == [] + + def test_the_approved_upload_runs_and_reports_the_loader_path( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, agent_caller + ): + import jsonschema + + routes = {**_UPLOAD_ROUTES, ("GET", "/api/workspaces/current"): _WORKSPACE} + args = [str(model_file), "--type", "loras", "--workspace", "ws-1", "--yes", "--expected-sha256", _MODEL_SHA256] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, routes, [200]) + + assert rc == 0 and env["changed"] is True + data = env["data"] + assert data["id"] == "asset-9" and data["uploaded"] is True and data["task_id"] == "task-1" + assert data["loader_path"] == "style.safetensors" and data["workspace_id"] == "ws-1" + jsonschema.Draft202012Validator(_model_schema()).validate(data) + (put,) = storage.puts + assert put["body"] == model_file.read_bytes() + + def test_the_sha256_may_carry_a_prefix_and_any_case( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, agent_caller + ): + args = [str(model_file), "--type", "loras", "--yes", "--expected-sha256", "sha256:" + _MODEL_SHA256.upper()] + env, rc, *_ = _model_upload(monkeypatch, capsys, args, dict(_UPLOAD_ROUTES), [200]) + + assert rc == 0 and env["data"]["uploaded"] is True + + def test_without_yes_an_agent_is_refused_before_anything_is_sent( + self, cloud_target, monkeypatch, capsys, model_file, agent_caller + ): + args = [str(model_file), "--type", "loras", "--expected-sha256", _MODEL_SHA256] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, dict(_UPLOAD_ROUTES), []) + + assert rc == 1 and env["error"]["code"] == "model_upload_needs_confirm" + details = env["error"]["details"] + assert details["missing"] == ["--yes"] and details["sha256"] == _MODEL_SHA256 + assert details["type"] == "loras" and details["file"] == str(model_file) + assert "/api/assets/uploads" not in cloud.paths() and storage.puts == [] + + def test_bytes_other_than_the_approved_ones_are_not_uploaded( + self, cloud_target, monkeypatch, capsys, model_file, agent_caller + ): + args = [str(model_file), "--type", "loras", "--yes", "--expected-sha256", "0" * 64] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, dict(_UPLOAD_ROUTES), []) + + assert rc == 1 and env["error"]["code"] == "asset_upload_invalid" + assert env["error"]["details"]["reason"] == "file_changed" + assert env["error"]["details"]["sha256"] == _MODEL_SHA256 + assert env["error"]["details"]["expected_sha256"] == "0" * 64 + assert cloud.paths() == ["/api/features"] and storage.puts == [] + + def test_a_file_replaced_after_its_approval_was_checked_sends_none_of_the_new_bytes( + self, cloud_target, monkeypatch, capsys, model_file, no_wait, agent_caller + ): + # The sha256 matches when it is checked; another process then replaces + # the file with same-sized bytes nobody approved, before the PUT. + routes = dict(_UPLOAD_ROUTES) + swap = _then(lambda: model_file.write_bytes(b"fedcba9876543210"), _started(1)) + routes[("POST", "/api/assets/uploads")] = [swap, _started(2)] + storage = _SentSoFar([200, 200]) + args = [str(model_file), "--type", "loras", "--yes", "--expected-sha256", _MODEL_SHA256] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, routes, [], storage=storage) + + assert rc == 1 and env["error"]["code"] == "asset_upload_failed" + assert env["error"]["details"]["reason"] == "file_changed" + assert [p["body"] for p in storage.puts] == [b""] + assert not any(p.endswith("/complete") for p in cloud.paths()) + + def test_a_malformed_sha256_is_refused_before_any_request( + self, cloud_target, monkeypatch, capsys, model_file, agent_caller + ): + args = [str(model_file), "--type", "loras", "--yes", "--expected-sha256", "abc"] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, dict(_UPLOAD_ROUTES), []) + + assert rc == 1 and env["error"]["details"]["reason"] == "bad_expected_sha256" + assert cloud.calls == [] and storage.puts == [] + + def test_another_workspace_gets_nothing(self, cloud_target, monkeypatch, capsys, model_file, agent_caller): + routes = {**_UPLOAD_ROUTES, ("GET", "/api/workspaces/current"): {**_WORKSPACE, "id": "ws-2"}} + args = [str(model_file), "--type", "loras", "--workspace", "ws-1", "--yes", "--expected-sha256", _MODEL_SHA256] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, routes, []) + + assert rc == 1 and env["error"]["code"] == "asset_upload_wrong_workspace" + assert env["error"]["details"] == { + "reason": "workspace_mismatch", + "workspace": "ws-1", + "current_workspace": "ws-2", + } + assert cloud.paths() == ["/api/workspaces/current"] and storage.puts == [] + + @pytest.mark.parametrize("answer", [_http_error(404), {"name": "Studio"}], ids=["older-server", "no-id"]) + def test_a_workspace_comfy_cloud_does_not_name_gets_nothing( + self, cloud_target, monkeypatch, capsys, model_file, agent_caller, answer + ): + routes = {**_UPLOAD_ROUTES, ("GET", "/api/workspaces/current"): answer} + args = [str(model_file), "--type", "loras", "--workspace", "ws-1", "--dry-run"] + env, rc, cloud, storage, _ = _model_upload(monkeypatch, capsys, args, routes, []) + + assert rc == 1 and env["error"]["code"] == "asset_upload_wrong_workspace" + assert env["error"]["details"]["reason"] == "workspace_unknown" + assert cloud.paths() == ["/api/workspaces/current"] and storage.puts == [] + + def test_it_is_registered_under_the_model_noun_with_a_schema(self): + from comfy_cli.command.models import models + from comfy_cli.discovery import COMMAND_SCHEMAS, STREAM_EVENT_SCHEMAS + + assert "upload" in {c.name for c in models.app.registered_commands} + assert COMMAND_SCHEMAS["comfy model upload"] == "model_upload" + assert STREAM_EVENT_SCHEMAS["comfy model upload"] == "build_push_event" diff --git a/tests/comfy_cli/test_hashing.py b/tests/comfy_cli/test_hashing.py index 22a033b58..3669ec9c3 100644 --- a/tests/comfy_cli/test_hashing.py +++ b/tests/comfy_cli/test_hashing.py @@ -30,3 +30,36 @@ def test_blake3_file_matches_one_shot_hashing_across_multiple_chunks(tmp_path: P # Then expected = f"blake3:{blake3(path.read_bytes()).hexdigest()}" assert digest == expected + + +def test_file_digests_reads_once_for_both_hashes_and_each_block(tmp_path: Path) -> None: + # Given + import hashlib + + path = tmp_path / "multi-chunk.bin" + data = (b"comfy" * ((_CHUNK_SIZE * 2) // 5 + 1)) + b"tail" + path.write_bytes(data) + + # When + digests = hashing.file_digests(path) + + # Then + assert digests.blake3 == hashing.blake3_file(path) == f"blake3:{blake3(data).hexdigest()}" + assert digests.sha256 == hashlib.sha256(data).hexdigest() + assert digests.size == len(data) and digests.block_size == _CHUNK_SIZE + blocks = [data[i : i + _CHUNK_SIZE] for i in range(0, len(data), _CHUNK_SIZE)] + assert len(blocks) == 3 + assert digests.blocks == tuple(blake3(b).digest() for b in blocks) + + +def test_file_digests_takes_a_block_size(tmp_path: Path) -> None: + # Given + path = tmp_path / "small.bin" + path.write_bytes(b"0123456789") + + # When + digests = hashing.file_digests(path, block_size=4) + + # Then + assert digests.block_size == 4 and digests.size == 10 + assert digests.blocks == tuple(blake3(b).digest() for b in (b"0123", b"4567", b"89")) diff --git a/tests/comfy_cli/test_tracking.py b/tests/comfy_cli/test_tracking.py index e489ee746..5f49d4b70 100644 --- a/tests/comfy_cli/test_tracking.py +++ b/tests/comfy_cli/test_tracking.py @@ -546,6 +546,21 @@ def context(repository, commit, file): assert "private" not in str(properties) assert "a-secret-sha" not in str(properties) + def test_model_upload_workspace_and_hash_are_redacted(self, tracking_module): + tracking_module.config_manager.set(constants.CONFIG_KEY_ENABLE_TRACKING, "True") + + @tracking_module.track_command("model") + def upload(workspace_id, expected_sha256, folder): + return None + + upload(workspace_id="w-private", expected_sha256="a-private-hash", folder="loras") + + _, _, properties = _first_track_call(tracking_module.provider) + assert properties["workspace_id"] == "" + assert properties["expected_sha256"] == "" + assert properties["folder"] == "loras" + assert "private" not in str(properties) + def test_add_node_text_is_redacted(self, tracking_module): # `comfy workflow add-node Note --text "..."` carries a note body # the user wrote: free-form prose that can hold anything (paths, prompts,