Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,20 @@ history.

### Added

- `comfy deploy run --allow-upload <hash>` (repeatable) uploads a local file
the deployment lacks only if its hash is listed; any other file fails with
`deploy_asset_missing` and is not sent. A file Cloud already holds is reused
by hash with no copy. A file it must upload is copied once to a private
temporary file, checked against the approved hash and sent from that copy,
so a file rewritten in between cannot send bytes nobody approved. A temp
folder without room for the whole file is refused before any byte is
copied, with `deploy_asset_upload_failed`. That file is not uploaded;
approved files before it may already be, and a retry reuses them. The
`--allow-upload` refusal, and the one from
`--no-upload`, now lists every such file in `details.missing[]` (absolute
`local_path`, `bytes`, `hash`) rather than stopping at the first, so a person
can approve the exact files the scanner picked. When two asset folders hold
files of the same name, this shows which one it picked.
- `comfy workflow check <file> --where cloud` says whether Comfy Cloud runs a
workflow with no custom build. Signed in, it reads Cloud's node list and your
uploaded models, and answers `covered`, `covered_with_remaps` (a model sits in
Expand Down
9 changes: 9 additions & 0 deletions comfy_cli/command/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,14 @@ def run_cmd(
list[Path] | None,
typer.Option("--asset-root", help="Extra directory workflow inputs may name files inside. Repeatable."),
] = None,
allow_upload: Annotated[
list[str] | None,
typer.Option(
"--allow-upload",
help="Upload a missing local file only if its hash is this one (a `--no-upload` refusal's "
"`details.missing[].hash`). Repeatable. Any other missing file fails with deploy_asset_missing.",
),
] = None,
) -> None:
_deploy_run.run_deploy(
ctx,
Expand All @@ -140,6 +148,7 @@ def run_cmd(
timeout,
no_upload,
tuple(asset_root or ()),
frozenset(allow_upload) if allow_upload else None,
),
)

Expand Down
8 changes: 7 additions & 1 deletion comfy_cli/command/deploy_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ class DeployRunRequest:
timeout: float | None
no_upload: bool
asset_roots: tuple[Path, ...] = ()
allowed_uploads: frozenset[str] | None = None


@dataclass(frozen=True, slots=True)
Expand Down Expand Up @@ -248,7 +249,12 @@ def run_deploy(ctx: typer.Context, request: DeployRunRequest) -> None:
data_target = Target(kind="cloud", base_url=endpoint_origin, path_prefix="/api/v2", auth_token=data_credential)
assets = resolve_assets(
plan,
AssetResolveContext(DeployAssetClient(endpoint_origin, token), renderer, not request.no_upload),
AssetResolveContext(
DeployAssetClient(endpoint_origin, token),
renderer,
not request.no_upload,
request.allowed_uploads,
),
)
submitted = DeployJobClient(endpoint_origin, token.current()).submit_job(
JobSubmitRequest(assets.workflow, str(uuid.uuid4()), deployment_id, partner_credential),
Expand Down
92 changes: 74 additions & 18 deletions comfy_cli/command/deploy_run_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,14 @@

from comfy_cli.command.build_spec import JsonObject, JsonValue
from comfy_cli.command.deploy_types import server_shape_error
from comfy_cli.command.deploy_workflow import WorkflowAssetPlan, materialize_workflow_assets
from comfy_cli.deploy_assets import AssetResolveRequest, AssetResolveResult, DeployAssetClient
from comfy_cli.command.deploy_workflow import WorkflowAsset, WorkflowAssetPlan, materialize_workflow_assets
from comfy_cli.deploy_api_errors import DeployAPIError
from comfy_cli.deploy_assets import (
AssetResolveRequest,
AssetResolveResult,
DeployAssetClient,
DeployAssetMissingError,
)
from comfy_cli.output.renderer import Renderer


Expand Down Expand Up @@ -60,31 +66,81 @@ class AssetResolveContext:
client: DeployAssetClient
renderer: Renderer
upload_enabled: bool
allowed_hashes: frozenset[str] | None = None


def _missing_error(missing: list[JsonValue]) -> DeployAPIError:
"""One refusal naming every file the run may not upload.

It lists them all, not just the first, so ``details.missing`` is the whole
upload plan: each absolute ``local_path`` the scanner picked, its ``bytes``
and its ``hash``. ``file_path`` and ``hash`` still name the first, as before.
"""
first = missing[0]
assert isinstance(first, dict)
count = len(missing)
return DeployAPIError(
code="deploy_asset_missing",
message=f"{count} local file{'' if count == 1 else 's'} the deployment does not have may not be uploaded",
details={"file_path": first["file_path"], "hash": first["hash"], "missing": missing},
)


def resolve_assets(plan: WorkflowAssetPlan, context: AssetResolveContext) -> ResolvedAssets:
asset_ids: dict[str, str] = {}
results: list[AssetResolveResult] = []
disclosed: list[DisclosedAsset] = []
for asset in plan.assets:
results: dict[int, tuple[AssetResolveResult, DisclosedAsset]] = {}
missing: list[JsonValue] = []
# With --allow-upload, every file's approval is settled before the first
# upload: the first pass only probes, so a refused file fails the run
# before an approved one is sent and left unused on the deployment.
approved = context.allowed_hashes if context.upload_enabled else None
to_upload: list[tuple[int, WorkflowAsset, int]] = []

def record(index: int, asset: WorkflowAsset, size: int, result: AssetResolveResult) -> None:
asset_id = result.asset.get("id")
if not isinstance(asset_id, str) or not asset_id:
raise server_shape_error("the data plane returned an asset without an id")
asset_ids[asset.marker] = asset_id
# An upload reports the bytes it sent: an approved file can be replaced
# by other approved bytes between the probe and the upload.
sent = result.bytes if result.uploaded else size
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
results[index] = (result, DisclosedAsset(asset.local_path, asset.file_path, sent, bool(result.uploaded)))

for index, asset in enumerate(plan.assets):
size = asset.local_path.stat().st_size
if context.renderer.is_pretty():
# Announced BEFORE the call, not after: a disclosure the user only
# reads once the bytes are already gone is not a disclosure.
context.renderer.info(f"Sending workflow asset {asset.local_path} ({size} bytes)")
result = context.client.resolve_asset(
AssetResolveRequest(asset.local_path, asset.file_path, context.upload_enabled)
)
asset_id = result.asset.get("id")
if not isinstance(asset_id, str) or not asset_id:
raise server_shape_error("the data plane returned an asset without an id")
asset_ids[asset.marker] = asset_id
results.append(result)
disclosed.append(DisclosedAsset(asset.local_path, asset.file_path, size, bool(result.uploaded)))
upload_now = context.upload_enabled and approved is None
try:
result = context.client.resolve_asset(
AssetResolveRequest(asset.local_path, asset.file_path, upload_now, context.allowed_hashes)
)
except DeployAssetMissingError as error:
if approved is not None and error.details["hash"] in approved:
to_upload.append((index, asset, size))
continue
# Keep going, so the refusal names every file, not only the first.
missing.append(error.details)
Comment thread
vqt123 marked this conversation as resolved.
Resolved
continue
record(index, asset, size, result)
if missing:
raise _missing_error(missing)
for index, asset, size in to_upload:
try:
result = context.client.resolve_asset(
AssetResolveRequest(asset.local_path, asset.file_path, True, context.allowed_hashes)
)
except DeployAssetMissingError as error:
# The file changed after the first pass and no longer has an approved hash.
raise _missing_error([error.details]) from error
record(index, asset, size, result)
ordered = [results[index] for index in sorted(results)]
return ResolvedAssets(
workflow=materialize_workflow_assets(plan, asset_ids),
uploaded=sum(result.uploaded for result in results),
deduped=sum(result.deduped for result in results),
bytes=sum(result.bytes for result in results),
files=tuple(disclosed),
uploaded=sum(result.uploaded for result, _ in ordered),
deduped=sum(result.deduped for result, _ in ordered),
bytes=sum(result.bytes for result, _ in ordered),
files=tuple(disclosed for _, disclosed in ordered),
)
132 changes: 125 additions & 7 deletions comfy_cli/deploy_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,18 @@

from __future__ import annotations

import errno
import json
import mimetypes
import os
import shutil
import tempfile
import urllib.error
import urllib.parse
import urllib.request
import uuid
from collections.abc import Iterator
from contextlib import contextmanager, suppress
from dataclasses import dataclass, replace
from pathlib import Path
from typing import BinaryIO, Final
Expand Down Expand Up @@ -51,9 +55,43 @@

@dataclass(frozen=True, slots=True)
class AssetResolveRequest:
"""One local file to turn into a data-plane asset.

``allowed_hashes`` is ``--allow-upload``: when set, a file the deployment
does not already hold is uploaded only if its hash is listed. A person
approves the exact files a ``--no-upload`` refusal listed, by hash, so a
same-named file from another root, or a file edited since, is refused
rather than sent. An approved upload reads one private snapshot, checked
against the approved hash, so a file rewritten during the probe or the
upload cannot send bytes nobody approved.
"""

local_path: Path
file_path: str
upload_enabled: bool = True
allowed_hashes: frozenset[str] | None = None


class DeployAssetMissingError(DeployAPIError):
"""A file the deployment lacks, which this run may not upload.

``details`` names it in full (the absolute ``local_path``, its ``bytes``
and ``hash``) so a refusal can serve as the exact upload plan to approve.
"""

def __init__(self, request: AssetResolveRequest, expected_hash: str, size: int) -> None:
# ``local_path`` is always the file the scanner picked, never a snapshot.
reason = "uploads are disabled" if not request.upload_enabled else "its hash is not one --allow-upload names"
super().__init__(
code="deploy_asset_missing",
message=f"the asset is not already present and {reason}",
details={
"file_path": request.file_path,
"hash": expected_hash,
"local_path": str(request.local_path),
"bytes": size,
},
)


@dataclass(frozen=True, slots=True)
Expand Down Expand Up @@ -240,15 +278,25 @@ def resolve_asset(self, request: AssetResolveRequest) -> AssetResolveResult:
raise _invalid_response("mint", status)

def _resolve_miss(self, request: AssetResolveRequest, expected_hash: str) -> AssetResolveResult:
if not request.upload_enabled:
raise DeployAPIError(
code="deploy_asset_missing",
message="the asset is not already present and uploads are disabled",
details={"file_path": request.file_path, "hash": expected_hash},
)
approved = request.allowed_hashes is None or expected_hash in request.allowed_hashes
if not request.upload_enabled or not approved:
raise DeployAssetMissingError(request, expected_hash, request.local_path.stat().st_size)
if request.allowed_hashes is None:
return self._upload(request, request.local_path, expected_hash)
# An approval binds bytes, not a path: the path can be rewritten after
# the hash above. Copy it once, check the copy, and send only the copy.
# The copy is made only here, on a miss, so a file Cloud already holds
# never needs temporary disk space.
with _snapshot(request) as snapshot:
snapshot_hash = blake3_file(snapshot)
if snapshot_hash != expected_hash:
raise DeployAssetMissingError(request, snapshot_hash, snapshot.stat().st_size)
return self._upload(request, snapshot, snapshot_hash)

def _upload(self, request: AssetResolveRequest, source: Path, expected_hash: str) -> AssetResolveResult:
# The handle stays open across the urlopen because urllib consumes the
# body iterator lazily, after the request is built.
with request.local_path.open("rb") as file:
with source.open("rb") as file:
upload, file_size = _multipart_request(self.target, request, expected_hash, file)
try:
with no_redirect_urlopen(upload, timeout=_UPLOAD_TIMEOUT) as response:
Expand All @@ -269,3 +317,73 @@ def _resolve_miss(self, request: AssetResolveRequest, expected_hash: str) -> Ass
if status in {200, 201} and isinstance(parsed, dict):
return AssetResolveResult(asset=parsed, uploaded=1, deduped=0, bytes=file_size)
raise _invalid_response("upload", status)


_SNAPSHOT_PREFIX = "comfy-cli-asset-"


def _snapshot_error(request: AssetResolveRequest, error: OSError) -> DeployAPIError:
"""The copy could not be made (a full disk, say): a local failure, not the server's."""
reason = error.strerror or str(error)
# Not ``tempfile.gettempdir()``: when no temp folder is usable, asking again
# raises the same error and this report never reaches the person.
copy = str(error.filename) if error.filename else ""
temp_dir = os.path.dirname(copy) if os.path.basename(copy).startswith(_SNAPSHOT_PREFIX) else tempfile.tempdir
return DeployAPIError(
code="deploy_asset_upload_failed",
message=f"could not copy {request.local_path} to a private temporary file to upload it: {reason}",
details={
"file_path": request.file_path,
"local_path": str(request.local_path),
"temp_dir": temp_dir,
"reason": reason,
},
hint=(
"this file was not uploaded, but approved files before it may already be on Comfy Cloud, and a "
"retry reuses them; free space in `details.temp_dir` or set TMPDIR to a folder with room, then retry"
),
)


def _require_free_space(snapshot: str, size: int) -> None:
"""Refuse before copying when the temp folder cannot hold the whole file.

The copy is as large as the file. Checking first fails a large file at once,
rather than after it has filled the disk. A folder that fills up during the
copy still fails the write, which reports the same way. A folder whose free
space cannot be read is not checked.
"""
try:
free = shutil.disk_usage(os.path.dirname(snapshot)).free
except OSError:
return
if free < size:
raise OSError(errno.ENOSPC, f"the copy needs {size:,} bytes and {free:,} are free", snapshot)


@contextmanager
def _snapshot(request: AssetResolveRequest) -> Iterator[Path]:
"""A private copy of the request's file in the system temp folder, removed on every exit.

``mkstemp`` creates it readable by this user only. The ``comfy-cli-`` prefix
names the tool that left it, should a crash skip the ``finally``. A folder
without room for the whole file is refused before any byte is copied. A
failure to make the copy is reported as such, naming the file; an error
raised while the copy is in use passes through unchanged.
"""
try:
descriptor, name = tempfile.mkstemp(prefix=_SNAPSHOT_PREFIX)
except OSError as error:
raise _snapshot_error(request, error) from error
snapshot = Path(name)
try:
try:
with os.fdopen(descriptor, "wb") as copy, request.local_path.open("rb") as original:
_require_free_space(name, os.fstat(original.fileno()).st_size)
shutil.copyfileobj(original, copy, UPLOAD_CHUNK_BYTES)
except OSError as error:
raise _snapshot_error(request, error) from error
yield snapshot
finally:
with suppress(OSError):
snapshot.unlink(missing_ok=True)
16 changes: 11 additions & 5 deletions comfy_cli/error_codes.py
Original file line number Diff line number Diff line change
Expand Up @@ -2022,15 +2022,21 @@ class ErrorCode:
ErrorCode(
"deploy_asset_missing",
"An input the run needs is not an asset this account can reach. Either a v2 asset hash probe found "
"no blob the caller may mint from while uploads were disabled (`details.file_path` and `details.hash` "
"identify it), or the submitted workflow referenced an asset id the account cannot mint, which the "
"server reports as `missing_asset` in `details.server_code`.",
"remove `--no-upload` to permit a streamed upload, or upload the input before retrying",
"no blob the caller may mint from while uploads were disabled by `--no-upload` or not allowed by "
"`--allow-upload` (`details.missing[]` names every such file with its absolute `local_path`, `bytes` "
"and `hash`; `details.file_path` and `details.hash` repeat the first), or the submitted workflow "
"referenced an asset id the account cannot mint, which the server reports as `missing_asset` in "
"`details.server_code`.",
"approve the files in `details.missing[]` and rerun with `--allow-upload <hash>` for each, remove "
"`--no-upload` to permit any upload, or upload the input before retrying",
),
ErrorCode(
"deploy_asset_upload_failed",
"A v2 multipart asset upload failed or the server rejected its `expected_hash`. A hash mismatch "
"mints no asset and reports `hash_mismatch` in `details.server_code`.",
"mints no asset and reports `hash_mismatch` in `details.server_code`. With `--allow-upload`, it also "
"covers a private temporary copy of the file that could not be made (a full disk, say): "
"`details.local_path` names the file and `details.temp_dir` the folder. That file was not uploaded; "
"approved files before it may already be, and a retry reuses them.",
"verify the local file is stable and readable, then retry the upload",
),
# --- knowledge -----------------------------------------------------------
Expand Down
Loading
Loading