diff --git a/app/asset_export.py b/app/asset_export.py new file mode 100644 index 00000000000..30a650f3a88 --- /dev/null +++ b/app/asset_export.py @@ -0,0 +1,834 @@ +"""Local implementation of the shared asset export contract. + +POST /api/assets/export zips assets selected by job (a job's live output +records, plus its temp records when previews are requested) and by asset id on +a background thread, reporting progress through the ``asset_export`` websocket +event as the cloud runtime does. The finished archive lands in the temp +directory, so like every other temp file it lives until the next start, and +GET /api/assets/exports/{exportName} points the owner at it through /view. +Everything resolves through the assets system, so every route answers 503 when +it is disabled. +""" + +import hashlib +import json +import logging +import os +import posixpath +import re +import threading +import time +import uuid +import zipfile +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any, Callable, Literal, Sequence, get_args +from urllib.parse import urlencode + +from aiohttp import web + +import folder_paths +from app.assets.api.routes import _build_error_response, _require_assets_feature_enabled +from app.assets.services.path_utils import get_asset_category_and_relative_path +from app.assets.services.schemas import ExportableAssetFile +from comfy_execution.jobs import validate_job_id + +NamingStrategy = Literal["group_by_job_id", "preserve", "asset_id", "group_by_job_time"] + +TASK_NAME = "task:export" +EVENT_NAME = "asset_export" +NAMING_STRATEGIES = frozenset(get_args(NamingStrategy)) +DEFAULT_NAMING_STRATEGY: NamingStrategy = "group_by_job_time" +EXPORT_NAME_RE = re.compile(r"^[a-zA-Z0-9_-]+\.zip$") +EXPORT_SUBFOLDER = "exports" +COPY_CHUNK_BYTES = 4 * 1024 * 1024 +PROGRESS_INTERVAL_SECONDS = 0.25 +MAX_CONCURRENT_EXPORTS = 2 +MAX_TRACKED_EXPORTS = 10000 +EXPORTABLE_CATEGORIES = ("input", "output", "temp") +JOB_TIME_FOLDER_FORMAT = "%Y-%m-%dT%H-%M-%S" + +JobFilesLookup = Callable[[Sequence[str], bool], list[ExportableAssetFile]] +AssetFileLookup = Callable[[str], ExportableAssetFile] +JobCreateTimesLookup = Callable[[Sequence[str]], dict[str, int]] +EventSink = Callable[[str, dict[str, Any]], None] +UserResolver = Callable[[web.Request], str] + + +class ExportStatus: + CREATED = "created" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +class FailureReason: + NOT_FOUND = "not_found" + MODEL_NOT_EXPORTABLE = "model_not_exportable" + UNSUPPORTED_ASSET_TYPE = "unsupported_asset_type" + FILENAME_COLLISION = "filename_collision" + FETCH_FAILED = "fetch_failed" + + REJECTING = frozenset({MODEL_NOT_EXPORTABLE, UNSUPPORTED_ASSET_TYPE}) + + +class ExportRequestError(ValueError): + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + +@dataclass(frozen=True) +class ExportRequest: + job_ids: list[str] + asset_ids: list[str] + naming_strategy: NamingStrategy + job_asset_name_filters: dict[str, list[str]] + include_previews: bool + + def to_payload(self) -> dict[str, Any]: + return { + "job_ids": self.job_ids, + "asset_ids": self.asset_ids, + "naming_strategy": self.naming_strategy, + "job_asset_name_filters": self.job_asset_name_filters, + "include_previews": self.include_previews, + } + + def idempotency_key(self, owner_id: str) -> str: + """Key under which a repeated request joins the export already running for it.""" + parts = [*self.job_ids, *self.asset_ids] + for job_id, names in self.job_asset_name_filters.items(): + if job_id in self.job_ids: + parts.append(f"filter:{job_id}:{'|'.join(sorted(names))}") + digest = hashlib.sha256(",".join(sorted(parts)).encode("utf-8")).hexdigest() + previews = "true" if self.include_previews else "false" + return f"export:{owner_id}:{self.naming_strategy}:{previews}:{digest}" + + +@dataclass(frozen=True) +class ExportEntry: + asset_id: str + abs_path: str + name: str + job_id: str | None + created_at: datetime + is_preview: bool = False + + +@dataclass(frozen=True) +class FailedAsset: + asset_id: str + error: str + job_id: str | None = None + asset_name: str | None = None + + def to_dict(self) -> dict[str, str]: + failed = {"asset_id": self.asset_id, "error": self.error} + if self.job_id: + failed["job_id"] = self.job_id + if self.asset_name: + failed["asset_name"] = self.asset_name + return failed + + +def _unique(values: list[str]) -> list[str]: + return list(dict.fromkeys(values)) + + +def _string_list(body: dict, key: str) -> list[str]: + value = body.get(key) + if value is None: + return [] + if not isinstance(value, list) or not all(isinstance(v, str) for v in value): + raise ExportRequestError("INVALID_REQUEST", f"{key} must be an array of strings") + return value + + +def parse_export_request(body: Any) -> ExportRequest: + if not isinstance(body, dict): + raise ExportRequestError("INVALID_REQUEST", "Request body must be a JSON object") + + try: + job_ids = _unique([validate_job_id(v) for v in _string_list(body, "job_ids")]) + except ValueError as e: + raise ExportRequestError("INVALID_REQUEST", f"Invalid job id: {e}") from e + + try: + asset_ids = _unique([str(uuid.UUID(v)) for v in _string_list(body, "asset_ids")]) + except ValueError as e: + raise ExportRequestError("INVALID_REQUEST", "asset_ids must contain UUIDs") from e + + if not job_ids and not asset_ids: + raise ExportRequestError("INVALID_REQUEST", "job_ids and/or asset_ids is required") + + naming_strategy = body.get("naming_strategy", DEFAULT_NAMING_STRATEGY) + if naming_strategy not in NAMING_STRATEGIES: + raise ExportRequestError( + "INVALID_NAMING_STRATEGY", f"Invalid naming strategy: {naming_strategy}" + ) + + raw_filters = body.get("job_asset_name_filters") + if raw_filters is None: + raw_filters = {} + if not isinstance(raw_filters, dict): + raise ExportRequestError("INVALID_REQUEST", "job_asset_name_filters must be an object") + filters: dict[str, list[str]] = {} + for job_id, names in raw_filters.items(): + if ( + not isinstance(names, list) + or not names + or not all(isinstance(n, str) for n in names) + ): + raise ExportRequestError( + "INVALID_REQUEST", + "job_asset_name_filters values must be non-empty arrays of strings", + ) + filters[job_id] = _unique(names) + + include_previews = body.get("include_previews", False) + if not isinstance(include_previews, bool): + raise ExportRequestError("INVALID_REQUEST", "include_previews must be a boolean") + + return ExportRequest( + job_ids=job_ids, + asset_ids=asset_ids, + naming_strategy=naming_strategy, + job_asset_name_filters=filters, + include_previews=include_previews, + ) + + +def export_location(file: ExportableAssetFile) -> tuple[str | None, str | None]: + """Root category of an asset file and its name inside the archive. + + The archive name is the asset's name in the file's directory under the + input, output or temp root, so uploads stored under their hash and renamed + assets keep the name the user sees. It is None for files under any other + root (models included), and the category is None outside every root. + """ + try: + category, relative = get_asset_category_and_relative_path(file.path) + except ValueError: + return None, None + if category not in EXPORTABLE_CATEGORIES: + return category, None + directory = posixpath.dirname(relative) + return category, posixpath.join(directory, file.name) if file.name else relative + + +def matches_name_filter( + file: ExportableAssetFile, archive_name: str | None, names: set[str] | None +) -> bool: + if names is None: + return True + return ( + file.name in names + or archive_name in names + or (file.hash is not None and file.hash in names) + ) + + +def safe_archive_path(name: str) -> str: + parts = [p for p in re.split(r"[\\/]+", name) if p not in ("", ".", "..")] + return "/".join(parts) + + +def _utc(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def build_job_time_folders( + entries: list[ExportEntry], job_create_times: dict[str, datetime] | None = None +) -> dict[str, str]: + """Folder name per job, from its creation time in UTC. + + A job whose creation time is unknown is named after its earliest asset. + Two jobs landing on the same second are bumped a second apart so they never + share a folder. + """ + job_times: dict[str, datetime] = {} + for entry in entries: + if entry.job_id is None: + continue + created = _utc(entry.created_at) + if entry.job_id not in job_times or created < job_times[entry.job_id]: + job_times[entry.job_id] = created + for job_id, create_time in (job_create_times or {}).items(): + if job_id in job_times: + job_times[job_id] = _utc(create_time) + folders: dict[str, str] = {} + used: set[str] = set() + for job_id, job_time in sorted(job_times.items(), key=lambda item: (item[1], item[0])): + moment = job_time.replace(microsecond=0) + folder = moment.strftime(JOB_TIME_FOLDER_FORMAT) + while folder in used: + moment += timedelta(seconds=1) + folder = moment.strftime(JOB_TIME_FOLDER_FORMAT) + used.add(folder) + folders[job_id] = folder + return folders + + +def _archive_name(entry: ExportEntry, strategy: NamingStrategy, job_folders: dict[str, str]) -> str: + name = safe_archive_path(entry.name) or os.path.basename(entry.abs_path) + if strategy == "asset_id": + return entry.asset_id + os.path.splitext(name)[1] + if strategy == "group_by_job_id": + folder = entry.job_id or "unknown" + elif strategy == "group_by_job_time": + folder = ( + job_folders[entry.job_id] + if entry.job_id is not None + else _utc(entry.created_at).strftime(JOB_TIME_FOLDER_FORMAT) + ) + else: + return name + return f"{folder}/previews/{name}" if entry.is_preview else f"{folder}/{name}" + + +def assign_archive_names( + entries: list[ExportEntry], + strategy: NamingStrategy, + job_create_times: dict[str, datetime] | None = None, +) -> tuple[list[tuple[ExportEntry, str]], list[ExportEntry]]: + """Pair entries with archive names, and list the entries dropped on a name collision. + + Entries are taken in asset id order, so the entry kept on a collision (the + first one) does not depend on the order the entries came in. + """ + job_folders = ( + build_job_time_folders(entries, job_create_times) + if strategy == "group_by_job_time" + else {} + ) + assigned: list[tuple[ExportEntry, str]] = [] + collided: list[ExportEntry] = [] + used: set[str] = set() + for entry in sorted(entries, key=lambda e: e.asset_id): + arcname = _archive_name(entry, strategy, job_folders) + if arcname in used: + collided.append(entry) + continue + used.add(arcname) + assigned.append((entry, arcname)) + return assigned, collided + + +def write_archive( + assigned: list[tuple[ExportEntry, str]], + dest_path: str, + on_bytes: Callable[[int], None], + on_file_done: Callable[[ExportEntry, bool], None], +) -> int: + """Write a stored (uncompressed) zip and return the number of files written. + + A file that cannot be opened is skipped and reported as failed. A read + error after copying started propagates, since the entry could otherwise be + left truncated inside an otherwise valid archive. + """ + written = 0 + with zipfile.ZipFile( + dest_path, "w", compression=zipfile.ZIP_STORED, allowZip64=True + ) as archive: + for entry, arcname in assigned: + try: + info = zipfile.ZipInfo.from_file(entry.abs_path, arcname, strict_timestamps=False) + if info.is_dir(): + raise IsADirectoryError(entry.abs_path) + source = open(entry.abs_path, "rb") + except OSError: + logging.warning("Asset export skipped unreadable file %s", entry.abs_path) + on_file_done(entry, False) + continue + read_size = max(1, min(info.file_size, COPY_CHUNK_BYTES)) + with source, archive.open(info, "w") as target: + while chunk := source.read(read_size): + target.write(chunk) + on_bytes(len(chunk)) + written += 1 + on_file_done(entry, True) + return written + + +def _iso(timestamp: float | None) -> str | None: + if timestamp is None: + return None + return datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat().replace("+00:00", "Z") + + +def _new_export_name() -> str: + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + return f"comfy-export-{today}-{uuid.uuid4().hex[:8]}.zip" + + +@dataclass +class ExportTask: + id: str + owner_id: str + request: ExportRequest + idempotency_key: str + status: str = ExportStatus.CREATED + created_at: float = field(default_factory=time.time) + updated_at: float = field(default_factory=time.time) + started_at: float | None = None + completed_at: float | None = None + export_name: str | None = None + assets_total: int = 0 + assets_attempted: int = 0 + assets_failed: int = 0 + bytes_total: int = 0 + bytes_processed: int = 0 + used_job_time_fallback: bool = False + result: dict[str, Any] | None = None + error_message: str | None = None + + @property + def finished(self) -> bool: + return self.status in (ExportStatus.COMPLETED, ExportStatus.FAILED) + + @property + def succeeded(self) -> bool: + return self.result is not None and self.result["success"] + + def progress(self) -> float: + if self.succeeded: + return 1.0 + if self.bytes_total <= 0: + return 0.0 + return min(self.bytes_processed / self.bytes_total, 1.0) + + def event(self) -> dict[str, Any]: + """The asset_export websocket payload, shaped like cloud's AssetExportMessage.""" + event: dict[str, Any] = { + "task_id": self.id, + "assets_total": self.assets_total, + "assets_attempted": self.assets_attempted, + "assets_failed": self.assets_failed, + "bytes_total": self.bytes_total, + "bytes_processed": self.bytes_processed, + "progress": self.progress(), + "status": self.status, + } + if self.succeeded: + event["export_name"] = self.export_name + elif self.finished: + event["status"] = ExportStatus.FAILED + event["error"] = self.error_message or (self.result or {}).get("error") + if self.used_job_time_fallback: + event["used_job_time_fallback"] = True + return event + + def task_response(self) -> dict[str, Any]: + response: dict[str, Any] = { + "id": self.id, + "idempotency_key": self.idempotency_key, + "task_name": TASK_NAME, + "payload": self.request.to_payload(), + "status": self.status, + "create_time": _iso(self.created_at), + "update_time": _iso(self.updated_at), + } + if self.started_at is not None: + response["started_at"] = _iso(self.started_at) + if self.completed_at is not None: + response["completed_at"] = _iso(self.completed_at) + if self.result is not None: + response["result"] = self.result + if self.error_message: + response["error_message"] = self.error_message + return response + + +def _export_result( + success: bool, + *, + export_name: str | None = None, + export_size: int = 0, + uncompressed_size: int = 0, + assets_total: int = 0, + assets_succeeded: int = 0, + used_job_time_fallback: bool = False, + failed_assets: Sequence[FailedAsset] = (), + error: str | None = None, +) -> dict[str, Any]: + """Task result shaped like cloud's ExportResult, omitting empty fields the same way.""" + result: dict[str, Any] = {"success": success} + if export_name: + result["export_name"] = export_name + if export_size > 0: + result["export_size"] = export_size + if uncompressed_size > 0: + result["uncompressed_size"] = uncompressed_size + if assets_total > 0: + result["assets_total"] = assets_total + result["assets_succeeded"] = assets_succeeded + if used_job_time_fallback: + result["used_job_time_fallback"] = True + if failed_assets: + result["failed_assets"] = [failed.to_dict() for failed in failed_assets] + if error: + result["error"] = error + return result + + +class AssetExportManager: + def __init__( + self, + list_job_files: JobFilesLookup, + get_asset_file: AssetFileLookup, + get_job_create_times: JobCreateTimesLookup, + event_sink: EventSink, + get_user_id: UserResolver, + ): + self._list_job_files = list_job_files + self._get_asset_file = get_asset_file + self._get_job_create_times = get_job_create_times + self._event_sink = event_sink + self._get_user_id = get_user_id + self._tasks: dict[str, ExportTask] = {} + self._lock = threading.Lock() + self._slots = threading.BoundedSemaphore(MAX_CONCURRENT_EXPORTS) + + def register_routes(self, app: web.Application) -> None: + gated = _require_assets_feature_enabled + app.add_routes([ + web.post("/api/assets/export", gated(self._create_export_route)), + web.get("/api/assets/exports/{exportName}", gated(self._export_url_route)), + web.get("/api/tasks/{task_id}", gated(self._get_task_route)), + ]) + + @staticmethod + def export_directory() -> str: + return os.path.join(folder_paths.get_temp_directory(), EXPORT_SUBFOLDER) + + def export_path(self, export_name: str) -> str: + return os.path.join(self.export_directory(), export_name) + + def create_export(self, request: ExportRequest, owner_id: str) -> ExportTask: + """Start an export, or return the one still running for the same request.""" + key = request.idempotency_key(owner_id) + with self._lock: + running = next( + (t for t in self._tasks.values() if t.idempotency_key == key and not t.finished), + None, + ) + if running is not None: + return running + task = ExportTask( + id=str(uuid.uuid4()), owner_id=owner_id, request=request, idempotency_key=key + ) + self._tasks[task.id] = task + forgotten = self._forget_oldest_finished() + if forgotten is not None and forgotten.export_name: + try: + os.remove(self.export_path(forgotten.export_name)) + except OSError: + pass + threading.Thread( + target=self._run, args=(task,), name=f"asset-export-{task.id[:8]}", daemon=True + ).start() + return task + + def _forget_oldest_finished(self) -> ExportTask | None: + """Cap the registry the way the prompt history is capped. + + The forgotten export can no longer be fetched, so its zip is removed too. + """ + if len(self._tasks) <= MAX_TRACKED_EXPORTS: + return None + oldest = next((t for t in self._tasks.values() if t.finished), None) + if oldest is not None: + del self._tasks[oldest.id] + return oldest + + def get_task(self, task_id: str) -> ExportTask | None: + with self._lock: + return self._tasks.get(task_id) + + def _find_by_export_name(self, export_name: str) -> ExportTask | None: + with self._lock: + return next((t for t in self._tasks.values() if t.export_name == export_name), None) + + def _update(self, task: ExportTask, **changes: Any) -> dict[str, Any]: + with self._lock: + for key, value in changes.items(): + setattr(task, key, value) + task.updated_at = time.time() + return task.event() + + def _emit(self, event: dict[str, Any]) -> None: + try: + self._event_sink(EVENT_NAME, event) + except Exception: + logging.warning("Failed to send asset export event", exc_info=True) + + def _resolve_entries(self, request: ExportRequest) -> tuple[list[ExportEntry], list[FailedAsset]]: + """Entries for the job files and assets of a request, and the assets that failed to resolve.""" + failed: list[FailedAsset] = [] + entries: list[ExportEntry] = [] + seen_ids: set[str] = set() + + def add(file: ExportableAssetFile, category: str | None, archive_name: str | None) -> None: + if file.id in seen_ids: + return + seen_ids.add(file.id) + if archive_name is None: + reason = ( + FailureReason.MODEL_NOT_EXPORTABLE + if category == "models" + else FailureReason.UNSUPPORTED_ASSET_TYPE + ) + failed.append(FailedAsset(file.id, reason, file.job_id, file.name)) + return + entries.append( + ExportEntry( + asset_id=file.id, + abs_path=file.path, + name=archive_name, + job_id=file.job_id, + created_at=file.created_at, + is_preview=category == "temp", + ) + ) + + for file in self._list_job_files(request.job_ids, request.include_previews): + names = request.job_asset_name_filters.get(file.job_id or "") + category, archive_name = export_location(file) + if matches_name_filter(file, archive_name, set(names) if names is not None else None): + add(file, category, archive_name) + for asset_id in request.asset_ids: + if asset_id in seen_ids: + continue + try: + file = self._get_asset_file(asset_id) + except (ValueError, OSError): + failed.append(FailedAsset(asset_id, FailureReason.NOT_FOUND)) + continue + add(file, *export_location(file)) + return entries, failed + + def _lookup_job_create_times(self, entries: list[ExportEntry]) -> dict[str, datetime] | None: + """Creation times of the jobs behind the entries, as far as the server still knows them. + + None when the lookup failed, so every job falls back to its asset times. + """ + job_ids = sorted({entry.job_id for entry in entries if entry.job_id is not None}) + if not job_ids: + return {} + try: + create_times = self._get_job_create_times(job_ids) + except Exception: + logging.warning( + "Asset export could not look up job creation times, using asset times", + exc_info=True, + ) + return None + return { + job_id: datetime.fromtimestamp(create_time / 1000, tz=timezone.utc) + for job_id, create_time in create_times.items() + } + + def _finish(self, task: ExportTask, result: dict[str, Any], **changes: Any) -> None: + self._emit( + self._update( + task, + status=ExportStatus.COMPLETED, + result=result, + completed_at=time.time(), + **changes, + ) + ) + + def _run(self, task: ExportTask) -> None: + with self._slots: + self._emit(self._update(task, status=ExportStatus.RUNNING, started_at=time.time())) + partial_path = os.path.join(self.export_directory(), f"{task.id}.zip.part") + try: + self._export(task, partial_path) + except Exception as e: + logging.exception("Asset export %s failed", task.id) + try: + os.remove(partial_path) + except OSError: + pass + self._emit( + self._update( + task, + status=ExportStatus.FAILED, + error_message=str(e) or "Export failed", + completed_at=time.time(), + ) + ) + + def _export(self, task: ExportTask, partial_path: str) -> None: + entries, failed = self._resolve_entries(task.request) + if any(f.error in FailureReason.REJECTING for f in failed): + self._finish( + task, + _export_result( + False, failed_assets=failed, error="One or more assets are not exportable" + ), + ) + return + + strategy = task.request.naming_strategy + job_create_times = None + used_fallback = False + if strategy == "group_by_job_time": + job_create_times = self._lookup_job_create_times(entries) + job_ids = {entry.job_id for entry in entries if entry.job_id is not None} + used_fallback = bool(job_ids) and ( + job_create_times is None or bool(job_ids - job_create_times.keys()) + ) + assigned, collided = assign_archive_names(entries, strategy, job_create_times) + failed += [ + FailedAsset(entry.asset_id, FailureReason.FILENAME_COLLISION, entry.job_id, entry.name) + for entry in collided + ] + if not assigned: + self._finish( + task, + _export_result( + False, + used_job_time_fallback=used_fallback, + failed_assets=failed, + error="No valid assets to export", + ), + used_job_time_fallback=used_fallback, + ) + return + + bytes_total = 0 + for entry, _ in assigned: + try: + bytes_total += os.path.getsize(entry.abs_path) + except OSError: + pass + self._update( + task, + assets_total=len(assigned), + bytes_total=bytes_total, + used_job_time_fallback=used_fallback, + ) + os.makedirs(self.export_directory(), exist_ok=True) + last_emit = [0.0] + + def record(bytes_count: int = 0, file_succeeded: bool | None = None) -> None: + with self._lock: + task.bytes_processed += bytes_count + if file_succeeded is not None: + task.assets_attempted += 1 + if not file_succeeded: + task.assets_failed += 1 + task.updated_at = time.time() + now = time.monotonic() + if now - last_emit[0] < PROGRESS_INTERVAL_SECONDS: + return + last_emit[0] = now + event = task.event() + self._emit(event) + + def on_bytes(count: int) -> None: + record(bytes_count=count) + + def on_file_done(entry: ExportEntry, succeeded: bool) -> None: + if not succeeded: + failed.append( + FailedAsset(entry.asset_id, FailureReason.FETCH_FAILED, entry.job_id, entry.name) + ) + record(file_succeeded=succeeded) + + written = write_archive(assigned, partial_path, on_bytes, on_file_done) + common = { + "assets_total": len(assigned), + "assets_succeeded": written, + "used_job_time_fallback": used_fallback, + "failed_assets": failed, + } + if written == 0: + os.remove(partial_path) + self._finish( + task, + _export_result(False, error="All assets failed to export", **common), + bytes_processed=bytes_total, + ) + return + + export_name = _new_export_name() + export_path = self.export_path(export_name) + os.replace(partial_path, export_path) + self._finish( + task, + _export_result( + True, + export_name=export_name, + export_size=os.path.getsize(export_path), + uncompressed_size=bytes_total, + **common, + ), + export_name=export_name, + bytes_processed=bytes_total, + ) + + def _owner_or_none(self, request: web.Request) -> str | None: + try: + return self._get_user_id(request) + except KeyError: + return None + + async def _create_export_route(self, request: web.Request) -> web.Response: + owner_id = self._owner_or_none(request) + if owner_id is None: + return _build_error_response(401, "UNAUTHORIZED", "Unknown user") + try: + body = await request.json() + except (json.JSONDecodeError, UnicodeDecodeError): + return _build_error_response(400, "INVALID_REQUEST", "Request body must be valid JSON") + try: + export_request = parse_export_request(body) + except ExportRequestError as e: + return _build_error_response(400, e.code, str(e)) + task = self.create_export(export_request, owner_id) + return web.json_response( + { + "task_id": task.id, + "status": ExportStatus.CREATED, + "message": "Export task created. Use task_id to track progress.", + }, + status=202, + ) + + async def _export_url_route(self, request: web.Request) -> web.Response: + export_name = request.match_info["exportName"] + if not EXPORT_NAME_RE.match(export_name): + return _build_error_response(400, "INVALID_EXPORT_NAME", "Invalid export name") + owner_id = self._owner_or_none(request) + if owner_id is None: + return _build_error_response(401, "UNAUTHORIZED", "Unknown user") + task = self._find_by_export_name(export_name) + if ( + task is None + or task.owner_id != owner_id + or not os.path.isfile(self.export_path(export_name)) + ): + return _build_error_response(404, "EXPORT_NOT_FOUND", "Export not found") + query = urlencode({"filename": export_name, "type": "temp", "subfolder": EXPORT_SUBFOLDER}) + return web.json_response({"url": f"/api/view?{query}"}) + + async def _get_task_route(self, request: web.Request) -> web.Response: + try: + task_id = str(uuid.UUID(request.match_info["task_id"])) + except ValueError: + return _build_error_response(404, "TASK_NOT_FOUND", "Task not found") + owner_id = self._owner_or_none(request) + if owner_id is None: + return _build_error_response(401, "UNAUTHORIZED", "Unknown user") + task = self.get_task(task_id) + if task is None or task.owner_id != owner_id: + return _build_error_response(404, "TASK_NOT_FOUND", "Task not found") + with self._lock: + response = task.task_response() + return web.json_response(response) diff --git a/app/assets/services/asset_management.py b/app/assets/services/asset_management.py index 25be239060f..f06495c2d83 100644 --- a/app/assets/services/asset_management.py +++ b/app/assets/services/asset_management.py @@ -1,7 +1,8 @@ """Serves the per-asset operations behind the API: reading an asset's detail, -updating its name, tags, metadata and preview, deleting a record, and resolving -a hash to a servable path. Name and tag updates move ``updated_at`` only when -the requested values differ. Other supplied metadata fields record a write. +updating its name, tags, metadata and preview, deleting a record, resolving +a hash to a servable path, and looking up the files an export zips. Name and +tag updates move ``updated_at`` only when the requested values differ. Other +supplied metadata fields record a write. """ import mimetypes @@ -30,6 +31,7 @@ AssetData, AssetDetailResult, DownloadResolutionResult, + ExportableAssetFile, ReferenceData, UserMetadata, ) @@ -269,3 +271,62 @@ def resolve_asset_for_download( content_type=ctype, download_name=download_name, ) + + +def list_job_export_files( + job_ids: Sequence[str], include_previews: bool +) -> list[ExportableAssetFile]: + """Live output records of the given jobs, plus their temp records when previews are requested.""" + if not job_ids: + return [] + tags = ["output", "temp"] if include_previews else ["output"] + with create_session() as session: + rows = session.execute( + select( + Asset.id, + Asset.name, + Asset.job_id, + Asset.created_at, + AssetContent.path, + AssetContent.hash, + ) + .join(AssetContent, Asset.content_id == AssetContent.id) + .where( + Asset.job_id.in_(list(job_ids)), + AssetContent.is_missing.is_(False), + Asset.id.in_( + select(AssetTag.asset_id).where(AssetTag.tag_name.in_(tags)) + ), + ) + ).all() + return [ + ExportableAssetFile( + id=row.id, + name=row.name, + path=row.path, + hash=row.hash, + job_id=row.job_id, + created_at=row.created_at, + ) + for row in rows + ] + + +def get_export_file(reference_id: str) -> ExportableAssetFile: + with create_session() as session: + record = get_record_by_id(session, reference_id) + if record is None: + raise ValueError(f"AssetReference {reference_id} not found") + content = session.get(AssetContent, record.content_id) + if content is None or content.is_missing: + raise FileNotFoundError( + f"No live content for AssetReference {reference_id}" + ) + return ExportableAssetFile( + id=record.id, + name=record.name, + path=content.path, + hash=content.hash, + job_id=record.job_id, + created_at=record.created_at, + ) diff --git a/app/assets/services/schemas.py b/app/assets/services/schemas.py index 54ff299e275..8f3efa6cb9f 100644 --- a/app/assets/services/schemas.py +++ b/app/assets/services/schemas.py @@ -86,6 +86,16 @@ class DownloadResolutionResult: download_name: str +@dataclass(frozen=True) +class ExportableAssetFile: + id: str + name: str + path: str + hash: str | None + job_id: str | None + created_at: datetime + + @dataclass(frozen=True) class RegisteredAsset: id: str diff --git a/comfy_execution/jobs.py b/comfy_execution/jobs.py index 60f9b8f9048..0476d31ad43 100644 --- a/comfy_execution/jobs.py +++ b/comfy_execution/jobs.py @@ -418,6 +418,38 @@ def get_job(prompt_id: str, running: list, queued: list, history: dict) -> Optio return None +def get_job_create_times( + prompt_ids: list[str], + running: list, + queued: list, + history: dict, +) -> dict[str, int]: + """ + Get the creation time (ms since epoch) of each given job that is still known. + + A job is known while it is queued, running or in history. + + Args: + prompt_ids: The prompt IDs to look up + running: List of currently running queue items + queued: List of pending queue items + history: Dict of history items keyed by prompt_id + + Returns: + Dict of creation times keyed by prompt_id; jobs the server no longer knows are left out + """ + extra_data_by_prompt = {item[1]: item[3] for item in [*running, *queued]} + for prompt_id, history_item in history.items(): + extra_data_by_prompt[prompt_id] = history_item['prompt'][3] + + create_times = {} + for prompt_id in prompt_ids: + create_time = (extra_data_by_prompt.get(prompt_id) or {}).get('create_time') + if isinstance(create_time, int): + create_times[prompt_id] = create_time + return create_times + + def get_all_jobs( running: list, queued: list, diff --git a/server.py b/server.py index 0bf74ed8ffa..547bb5242f5 100644 --- a/server.py +++ b/server.py @@ -12,6 +12,7 @@ JobStatus, get_job, get_all_jobs, + get_job_create_times, validate_job_id, cancel_job, CANCEL_PENDING, @@ -48,7 +49,12 @@ from app.database.db import dependencies_available if dependencies_available(): - from app.assets.services.asset_management import resolve_hash_to_path + from app.assets.services.asset_management import ( + get_export_file, + list_job_export_files, + resolve_hash_to_path, + ) + from app.asset_export import AssetExportManager from app.user_manager import UserManager from app.model_manager import ModelFileManager @@ -260,6 +266,26 @@ def __init__(self, loop, asset_manager: "AssetManager"): logging.info(f"[Prompt Server] web root: {self.web_root}") self.asset_manager.register_routes(self.app, self.user_manager) self.asset_manager.set_event_sink(self.send_sync) + + def _job_create_times(prompt_ids): + """Creation times of the given jobs, from the queue and history.""" + running, queued = self.prompt_queue.get_current_queue_volatile() + history = {} + for prompt_id in prompt_ids: + history.update( + self.prompt_queue.get_history(prompt_id=prompt_id, map_function=lambda item: item) + ) + return get_job_create_times(prompt_ids, running, queued, history) + + if dependencies_available(): + self.asset_export_manager = AssetExportManager( + list_job_files=list_job_export_files, + get_asset_file=get_export_file, + get_job_create_times=_job_create_times, + event_sink=self.send_sync, + get_user_id=self.user_manager.get_request_user_id, + ) + self.asset_export_manager.register_routes(self.app) if self.asset_manager.enabled: emit("assets.enabled", hashing_enabled=args.enable_asset_hashing) routes = web.RouteTableDef() diff --git a/tests-unit/asset_export_test/__init__.py b/tests-unit/asset_export_test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests-unit/asset_export_test/test_asset_export.py b/tests-unit/asset_export_test/test_asset_export.py new file mode 100644 index 00000000000..7c8c7e50e6b --- /dev/null +++ b/tests-unit/asset_export_test/test_asset_export.py @@ -0,0 +1,707 @@ +import asyncio +import os +import re +import threading +import time +import uuid +import zipfile +from datetime import datetime, timedelta, timezone +from urllib.parse import parse_qs, urlsplit + +import pytest +from aiohttp import web + +import folder_paths +from app import asset_export +from app.asset_export import ( + AssetExportManager, + ExportEntry, + ExportRequestError, + assign_archive_names, + export_location, + matches_name_filter, + parse_export_request, + write_archive, +) +from app.assets.api import routes +from app.assets.services import path_utils +from app.assets.services.schemas import ExportableAssetFile + +JOB_A = "11111111-1111-4111-8111-111111111111" +JOB_B = "22222222-2222-4222-8222-222222222222" +JOB_C = "44444444-4444-4444-8444-444444444444" +ASSET_ID = "33333333-3333-4333-8333-333333333333" +CREATED = datetime(2026, 9, 21, 14, 13, 20, tzinfo=timezone.utc) +QUEUED = CREATED - timedelta(minutes=3) +EXPORT_NAME_PATTERN = re.compile(r"^comfy-export-\d{4}-\d{2}-\d{2}-[0-9a-f]{8}\.zip$") + + +@pytest.fixture(autouse=True) +def assets_enabled(monkeypatch): + monkeypatch.setattr(routes, "_ASSETS_ENABLED", True) + + +@pytest.fixture +def dirs(tmp_path, monkeypatch): + roots = {category: tmp_path / category for category in ("input", "output", "temp")} + for category, root in roots.items(): + root.mkdir() + monkeypatch.setattr(folder_paths, f"get_{category}_directory", lambda root=root: str(root)) + roots["models"] = tmp_path / "models" / "checkpoints" + roots["models"].mkdir(parents=True) + monkeypatch.setattr( + path_utils, + "get_comfy_models_folders", + lambda: [("checkpoints", [str(roots["models"])], set())], + ) + return roots + + +def write_file(directory, relative, data): + path = directory / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return str(path) + + +def asset_file(path, job_id=JOB_A, asset_id=None, file_hash=None, created_at=CREATED, name=None): + return ExportableAssetFile( + id=asset_id or str(uuid.uuid4()), + name=name or os.path.basename(path), + path=str(path), + hash=file_hash, + job_id=job_id, + created_at=created_at, + ) + + +@pytest.mark.parametrize( + "body, code", + [ + ([], "INVALID_REQUEST"), + ({}, "INVALID_REQUEST"), + ({"job_ids": []}, "INVALID_REQUEST"), + ({"job_ids": ["not-a-uuid"]}, "INVALID_REQUEST"), + ({"job_ids": ["AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA"]}, "INVALID_REQUEST"), + ({"asset_ids": ["nope"]}, "INVALID_REQUEST"), + ({"job_ids": "abc"}, "INVALID_REQUEST"), + ({"job_ids": [JOB_A], "naming_strategy": "flat"}, "INVALID_NAMING_STRATEGY"), + ({"job_ids": [JOB_A], "job_asset_name_filters": []}, "INVALID_REQUEST"), + ({"job_ids": [JOB_A], "job_asset_name_filters": {JOB_A: []}}, "INVALID_REQUEST"), + ({"job_ids": [JOB_A], "job_asset_name_filters": {JOB_A: [1]}}, "INVALID_REQUEST"), + ({"job_ids": [JOB_A], "include_previews": "yes"}, "INVALID_REQUEST"), + ], +) +def test_parse_export_request_rejects_invalid_bodies(body, code): + with pytest.raises(ExportRequestError) as error: + parse_export_request(body) + assert error.value.code == code + + +def test_parse_export_request_applies_contract_defaults_and_dedupes(): + request = parse_export_request({"job_ids": [JOB_A, JOB_A], "asset_ids": [ASSET_ID.upper()]}) + assert request.job_ids == [JOB_A] + assert request.asset_ids == [ASSET_ID] + assert request.naming_strategy == "group_by_job_time" + assert request.job_asset_name_filters == {} + assert request.include_previews is False + + +@pytest.mark.parametrize( + "other, owner, same", + [ + ({"job_ids": [JOB_B, JOB_A], "job_asset_name_filters": {JOB_A: ["b", "a"]}}, "alice", True), + ({"job_ids": [JOB_A, JOB_B], "job_asset_name_filters": {JOB_A: ["a", "b"], JOB_C: ["x"]}}, "alice", True), + ({"job_ids": [JOB_A, JOB_B], "job_asset_name_filters": {JOB_A: ["a", "b"]}}, "bob", False), + ({"job_ids": [JOB_A, JOB_B]}, "alice", False), + ({"job_ids": [JOB_A, JOB_B], "job_asset_name_filters": {JOB_A: ["a", "b"]}, "include_previews": True}, "alice", False), + ], + ids=["reordered", "filter of a job not requested", "other owner", "no filter", "previews"], +) +def test_idempotency_key_identifies_the_same_request_of_the_same_owner(other, owner, same): + base = parse_export_request({"job_ids": [JOB_A, JOB_B], "job_asset_name_filters": {JOB_A: ["a", "b"]}}) + + matches = parse_export_request(other).idempotency_key(owner) == base.idempotency_key("alice") + + assert matches is same + + +@pytest.mark.parametrize( + "category, relative, name, expected", + [ + ("output", "EXR/shot/a.exr", None, ("output", "EXR/shot/a.exr")), + ("temp", "p.png", None, ("temp", "p.png")), + ("input", "ref.png", None, ("input", "ref.png")), + ("input", "0123abcd.png", "cat.png", ("input", "cat.png")), + ("output", "renders/old.png", "new.png", ("output", "renders/new.png")), + ("models", "x.safetensors", None, ("models", None)), + (None, "elsewhere/x.png", None, (None, None)), + ], + ids=["output", "temp", "input", "upload stored under its hash", "renamed", "model", "outside every root"], +) +def test_export_location_names_the_file_after_its_asset(dirs, tmp_path, category, relative, name, expected): + root = dirs[category] if category else tmp_path + assert export_location(asset_file(root / relative, name=name)) == expected + + +@pytest.mark.parametrize( + "names, expected", + [ + (None, True), + ({"a.png"}, True), + ({"sub/a.png"}, True), + ({"blake3:abc"}, True), + ({"b.png", "sub/b.png"}, False), + ], +) +def test_name_filters_match_asset_name_archive_path_or_hash(names, expected): + file = asset_file("/out/sub/a.png", file_hash="blake3:abc") + assert matches_name_filter(file, "sub/a.png", names) is expected + + +def entry(name, asset_id, job_id=JOB_A, is_preview=False, created_at=CREATED): + return ExportEntry(asset_id, f"/abs/{name}", name, job_id, created_at, is_preview) + + +@pytest.mark.parametrize( + "strategy, entries, expected", + [ + ("preserve", [entry("a.png", "2"), entry("a.png", "1", job_id=JOB_B)], [("1", "a.png")]), + ( + "group_by_job_id", + [entry("a.png", "1"), entry("a.png", "2", job_id=JOB_B)], + [("1", f"{JOB_A}/a.png"), ("2", f"{JOB_B}/a.png")], + ), + ("group_by_job_id", [entry("p.png", "1", is_preview=True)], [("1", f"{JOB_A}/previews/p.png")]), + ("group_by_job_id", [entry("a.png", "1", job_id=None)], [("1", "unknown/a.png")]), + ( + "group_by_job_time", + [entry("b.png", "2", created_at=CREATED + timedelta(minutes=5)), entry("a.png", "1")], + [("1", "2026-09-21T14-13-20/a.png"), ("2", "2026-09-21T14-13-20/b.png")], + ), + ( + "group_by_job_time", + [ + entry("a.png", "1", job_id=JOB_B), + entry("a.png", "2", job_id=JOB_A), + entry("a.png", "3", job_id=JOB_C, created_at=CREATED + timedelta(seconds=1)), + ], + [ + ("1", "2026-09-21T14-13-21/a.png"), + ("2", "2026-09-21T14-13-20/a.png"), + ("3", "2026-09-21T14-13-22/a.png"), + ], + ), + ( + "group_by_job_time", + [entry("a.png", "1", job_id=None, created_at=CREATED.replace(tzinfo=None))], + [("1", "2026-09-21T14-13-20/a.png")], + ), + ("asset_id", [entry("x/a.png", ASSET_ID)], [(ASSET_ID, f"{ASSET_ID}.png")]), + ("preserve", [entry("../../evil/./a.png", "1")], [("1", "evil/a.png")]), + ], +) +def test_assign_archive_names(strategy, entries, expected): + assigned, collided = assign_archive_names(entries, strategy) + assert [(e.asset_id, name) for e, name in assigned] == expected + assert len(collided) == len(entries) - len(expected) + + +@pytest.mark.parametrize( + "entries, job_create_times, expected", + [ + ([entry("a.png", "1")], {JOB_A: QUEUED}, ["2026-09-21T14-10-20/a.png"]), + ( + [entry("a.png", "1"), entry("b.png", "2", job_id=JOB_B)], + {JOB_A: QUEUED}, + ["2026-09-21T14-10-20/a.png", "2026-09-21T14-13-20/b.png"], + ), + ( + [entry("a.png", "1", created_at=CREATED + timedelta(minutes=1)), entry("a.png", "2", job_id=JOB_B)], + {JOB_A: CREATED}, + ["2026-09-21T14-13-20/a.png", "2026-09-21T14-13-21/a.png"], + ), + ([entry("a.png", "1")], {JOB_B: QUEUED}, ["2026-09-21T14-13-20/a.png"]), + ], + ids=[ + "creation time wins over asset time", + "unknown job falls back to its earliest asset", + "creation and fallback times on one second get separate folders", + "creation times of other jobs are ignored", + ], +) +def test_job_time_folders_prefer_the_job_creation_time(entries, job_create_times, expected): + assigned, _ = assign_archive_names(entries, "group_by_job_time", job_create_times) + assert [name for _, name in assigned] == expected + + +def test_write_archive_stores_files_and_skips_unreadable(tmp_path): + contents = { + "large.bin": b"x" * (asset_export.COPY_CHUNK_BYTES + 10), + "small.bin": b"small", + "empty.bin": b"", + } + assigned = [ + (ExportEntry(str(i), write_file(tmp_path, name, data), name, None, CREATED), f"out/{name}") + for i, (name, data) in enumerate(contents.items()) + ] + assigned += [ + (ExportEntry("8", str(tmp_path / "missing.bin"), "missing.bin", None, CREATED), "missing.bin"), + (ExportEntry("9", str(tmp_path), "dir", None, CREATED), "dir"), + ] + byte_counts, outcomes = [], [] + dest = tmp_path / "out.zip" + + written = write_archive( + assigned, str(dest), byte_counts.append, lambda e, ok: outcomes.append((e.asset_id, ok)) + ) + + assert written == 3 + assert outcomes == [("0", True), ("1", True), ("2", True), ("8", False), ("9", False)] + assert sum(byte_counts) == sum(len(data) for data in contents.values()) + with zipfile.ZipFile(dest) as archive: + assert archive.testzip() is None + assert archive.namelist() == [f"out/{name}" for name in contents] + for name, data in contents.items(): + assert archive.getinfo(f"out/{name}").compress_type == zipfile.ZIP_STORED + assert archive.read(f"out/{name}") == data + + +class Harness: + def __init__(self, files, job_create_times, **overrides): + self.files = {f.id: f for f in files} + self.job_create_times = job_create_times + self.job_time_lookups = [] + self.events = [] + dependencies = { + "list_job_files": self.list_job_files, + "get_asset_file": self.get_asset_file, + "get_job_create_times": self.get_job_create_times, + "event_sink": lambda name, data: self.events.append((name, dict(data))), + "get_user_id": lambda request: request.headers.get("comfy-user", "default"), + } + self.manager = AssetExportManager(**{**dependencies, **overrides}) + self.app = web.Application() + self.manager.register_routes(self.app) + + def list_job_files(self, job_ids, include_previews): + return [ + f for f in self.files.values() + if f.job_id in job_ids and (include_previews or "temp" not in f.path) + ] + + def get_job_create_times(self, job_ids): + self.job_time_lookups.append(list(job_ids)) + return {job_id: ms for job_id, ms in self.job_create_times.items() if job_id in job_ids} + + def get_asset_file(self, asset_id): + if asset_id not in self.files: + raise ValueError(f"AssetReference {asset_id} not found") + return self.files[asset_id] + + async def wait_finished(self, task_id, timeout=10): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + task = self.manager.get_task(task_id) + if task is not None and task.finished: + return task + await asyncio.sleep(0.02) + raise AssertionError("export did not finish") + + def archive_names(self, task): + with zipfile.ZipFile(self.manager.export_path(task.export_name)) as archive: + return sorted(archive.namelist()) + + def final_event(self): + return self.events[-1][1] + + +@pytest.fixture +def make_harness(): + def make(files=(), job_create_times=None, **overrides): + return Harness(files, job_create_times or {}, **overrides) + + return make + + +async def start_export(client, body, user="default"): + response = await client.post("/api/assets/export", json=body, headers={"comfy-user": user}) + assert response.status == 202 + data = await response.json() + assert data["status"] == "created" + return data["task_id"] + + +@pytest.mark.asyncio +async def test_export_job_outputs_end_to_end(aiohttp_client, make_harness, dirs): + frames = {f"frame_{i:04d}.exr": os.urandom(1024 + i) for i in range(3)} + files = [asset_file(write_file(dirs["output"], f"EXR/{name}", data)) for name, data in frames.items()] + files.append(asset_file(write_file(dirs["temp"], "preview.png", b"preview"))) + harness = make_harness(files) + client = await aiohttp_client(harness.app) + + task_id = await start_export(client, {"job_ids": [JOB_A], "naming_strategy": "preserve"}) + finished = await harness.wait_finished(task_id) + assert harness.job_time_lookups == [] + + task_response = await client.get(f"/api/tasks/{task_id}") + assert task_response.status == 200 + task = await task_response.json() + assert task["status"] == "completed" + assert task["task_name"] == "task:export" + assert task["idempotency_key"].startswith("export:default:preserve:false:") + assert task["payload"]["job_ids"] == [JOB_A] + total = sum(len(d) for d in frames.values()) + export_name = task["result"]["export_name"] + assert EXPORT_NAME_PATTERN.match(export_name) + assert task["result"] == { + "success": True, + "export_name": export_name, + "export_size": os.path.getsize(harness.manager.export_path(export_name)), + "uncompressed_size": total, + "assets_total": 3, + "assets_succeeded": 3, + } + + statuses = [data["status"] for name, data in harness.events if name == "asset_export"] + assert statuses[0] == "running" and statuses[-1] == "completed" + assert all("export_name" not in data for _, data in harness.events[:-1]) + assert harness.final_event() == { + "task_id": task_id, + "export_name": export_name, + "assets_total": 3, + "assets_attempted": 3, + "assets_failed": 0, + "bytes_total": total, + "bytes_processed": total, + "progress": 1.0, + "status": "completed", + } + + url_response = await client.get(f"/api/assets/exports/{export_name}") + assert url_response.status == 200 + url = urlsplit((await url_response.json())["url"]) + assert url.path == "/api/view" + assert parse_qs(url.query) == {"filename": [export_name], "type": ["temp"], "subfolder": ["exports"]} + assert harness.manager.export_path(export_name) == os.path.join( + folder_paths.get_temp_directory(), "exports", export_name + ) + assert harness.archive_names(finished) == sorted(f"EXR/{n}" for n in frames) + + +@pytest.mark.asyncio +async def test_export_honours_name_filters_and_previews(aiohttp_client, make_harness, dirs): + harness = make_harness([ + asset_file(write_file(dirs["output"], "a.png", b"a")), + asset_file(write_file(dirs["output"], "sub/b.png", b"b")), + asset_file(write_file(dirs["output"], "c.png", b"c"), file_hash="blake3:c"), + asset_file(write_file(dirs["output"], "d.png", b"d")), + asset_file(write_file(dirs["temp"], "p.png", b"p")), + asset_file(write_file(dirs["output"], "e.png", b"e"), job_id=JOB_B), + ]) + client = await aiohttp_client(harness.app) + + task_id = await start_export(client, { + "job_ids": [JOB_A, JOB_B], + "job_asset_name_filters": {JOB_A: ["a.png", "sub/b.png", "blake3:c", "p.png"]}, + "include_previews": True, + "naming_strategy": "group_by_job_id", + }) + task = await harness.wait_finished(task_id) + + assert harness.archive_names(task) == [ + f"{JOB_A}/a.png", + f"{JOB_A}/c.png", + f"{JOB_A}/previews/p.png", + f"{JOB_A}/sub/b.png", + f"{JOB_B}/e.png", + ] + + +@pytest.mark.asyncio +async def test_records_sharing_a_file_are_all_exported(aiohttp_client, make_harness, dirs): + shared = write_file(dirs["output"], "a.png", b"a") + harness = make_harness([asset_file(shared, job_id=JOB_A), asset_file(shared, job_id=JOB_B)]) + client = await aiohttp_client(harness.app) + + task = await harness.wait_finished( + await start_export(client, {"job_ids": [JOB_A, JOB_B], "naming_strategy": "group_by_job_id"}) + ) + + assert harness.archive_names(task) == [f"{JOB_A}/a.png", f"{JOB_B}/a.png"] + + +@pytest.mark.asyncio +async def test_uploaded_assets_are_named_after_the_asset(aiohttp_client, make_harness, dirs): + upload = asset_file(write_file(dirs["input"], "0123abcd.png", b"cat"), job_id=None, name="cat.png") + harness = make_harness([upload]) + client = await aiohttp_client(harness.app) + + task = await harness.wait_finished( + await start_export(client, {"asset_ids": [upload.id], "naming_strategy": "preserve"}) + ) + + assert harness.archive_names(task) == ["cat.png"] + + +@pytest.mark.asyncio +async def test_export_reports_each_asset_it_could_not_export(aiohttp_client, make_harness, dirs): + exported = asset_file(write_file(dirs["output"], "a.png", b"a"), asset_id="00000000-0000-4000-8000-000000000001") + renamed = asset_file( + write_file(dirs["output"], "b.png", b"b"), asset_id="00000000-0000-4000-8000-000000000002", name="a.png" + ) + deleted = asset_file(dirs["output"] / "gone.png", asset_id="00000000-0000-4000-8000-000000000003") + unknown = str(uuid.uuid4()) + harness = make_harness([exported, renamed, deleted]) + client = await aiohttp_client(harness.app) + + task = await harness.wait_finished(await start_export(client, { + "job_ids": [JOB_A], + "asset_ids": [exported.id, unknown], + "naming_strategy": "preserve", + })) + + assert task.status == "completed" + assert task.result["success"] is True + assert task.result["assets_total"] == 2 + assert task.result["assets_succeeded"] == 1 + assert sorted(task.result["failed_assets"], key=lambda f: f["error"]) == [ + {"asset_id": deleted.id, "error": "fetch_failed", "job_id": JOB_A, "asset_name": "gone.png"}, + {"asset_id": renamed.id, "error": "filename_collision", "job_id": JOB_A, "asset_name": "a.png"}, + {"asset_id": unknown, "error": "not_found"}, + ] + assert harness.final_event()["assets_failed"] == 1 + assert harness.archive_names(task) == ["a.png"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stored, expected_error", + [ + ("nothing", "No valid assets to export"), + ("deleted", "All assets failed to export"), + ("model", "One or more assets are not exportable"), + ("outside", "One or more assets are not exportable"), + ], +) +async def test_export_reports_failure_when_nothing_can_be_exported( + aiohttp_client, make_harness, dirs, tmp_path, stored, expected_error +): + files = { + "nothing": [], + "deleted": [asset_file(dirs["output"] / "gone.png")], + "model": [asset_file(write_file(dirs["models"], "x.safetensors", b"m"))], + "outside": [asset_file(write_file(tmp_path, "elsewhere/x.png", b"x"))], + }[stored] + harness = make_harness(files) + client = await aiohttp_client(harness.app) + + task_id = await start_export(client, {"job_ids": [JOB_A]}) + task = await harness.wait_finished(task_id) + + assert task.status == "completed" + assert task.result["success"] is False + assert task.result["error"] == expected_error + assert task.export_name is None + assert harness.final_event()["status"] == "failed" + assert harness.final_event()["error"] == expected_error + export_directory = harness.manager.export_directory() + assert not os.path.isdir(export_directory) or os.listdir(export_directory) == [] + body = await (await client.get(f"/api/tasks/{task_id}")).json() + assert body["status"] == "completed" + assert body["result"]["success"] is False + assert "error_message" not in body + + +@pytest.mark.asyncio +async def test_unexpected_errors_fail_the_task(aiohttp_client, make_harness, dirs): + def broken_lookup(job_ids, include_previews): + raise RuntimeError("database is locked") + + harness = make_harness(list_job_files=broken_lookup) + client = await aiohttp_client(harness.app) + + task_id = await start_export(client, {"job_ids": [JOB_A]}) + task = await harness.wait_finished(task_id) + + assert task.status == "failed" + assert harness.final_event()["status"] == "failed" + assert harness.final_event()["error"] == "database is locked" + body = await (await client.get(f"/api/tasks/{task_id}")).json() + assert body["error_message"] == "database is locked" + assert "result" not in body + + +@pytest.mark.asyncio +async def test_export_completes_when_events_cannot_be_sent(aiohttp_client, make_harness, dirs): + def broken_sink(name, data): + raise ConnectionResetError() + + harness = make_harness([asset_file(write_file(dirs["output"], "a.png", b"a"))], event_sink=broken_sink) + client = await aiohttp_client(harness.app) + + task = await harness.wait_finished(await start_export(client, {"job_ids": [JOB_A]})) + + assert task.succeeded + + +@pytest.mark.asyncio +async def test_asset_ids_are_exported_by_id(aiohttp_client, make_harness, dirs): + stored = asset_file(write_file(dirs["output"], "renders/shot.exr", b"asset bytes"), asset_id=ASSET_ID) + harness = make_harness([stored]) + client = await aiohttp_client(harness.app) + + task = await harness.wait_finished( + await start_export(client, {"asset_ids": [ASSET_ID], "naming_strategy": "asset_id"}) + ) + + with zipfile.ZipFile(harness.manager.export_path(task.export_name)) as archive: + assert archive.namelist() == [f"{ASSET_ID}.exr"] + assert archive.read(f"{ASSET_ID}.exr") == b"asset bytes" + + +@pytest.mark.asyncio +async def test_repeating_a_running_request_joins_its_export(aiohttp_client, make_harness, dirs): + release = threading.Event() + files = [asset_file(write_file(dirs["output"], "a.png", b"a"))] + + def slow_lookup(job_ids, include_previews): + release.wait(5) + return files + + harness = make_harness(files, list_job_files=slow_lookup) + client = await aiohttp_client(harness.app) + body = {"job_ids": [JOB_A]} + + first = await start_export(client, body) + second = await start_export(client, body) + other_user = await start_export(client, body, user="bob") + release.set() + await harness.wait_finished(first) + after_finish = await start_export(client, body) + + assert second == first + assert other_user != first + assert after_finish != first + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "job_create_times, lookup_fails, expected_folder, used_fallback", + [ + ({JOB_A: int(QUEUED.timestamp() * 1000)}, False, "2026-09-21T14-10-20", False), + ({}, False, "2026-09-21T14-13-20", True), + ({JOB_A: int(QUEUED.timestamp() * 1000)}, True, "2026-09-21T14-13-20", True), + ], + ids=["known job", "job the server forgot", "failed lookup"], +) +async def test_job_time_folders_use_the_job_creation_time( + aiohttp_client, make_harness, dirs, job_create_times, lookup_fails, expected_folder, used_fallback +): + def broken_lookup(job_ids): + raise RuntimeError("queue unavailable") + + overrides = {"get_job_create_times": broken_lookup} if lookup_fails else {} + harness = make_harness( + [asset_file(write_file(dirs["output"], "a.png", b"a"))], + job_create_times=job_create_times, + **overrides, + ) + client = await aiohttp_client(harness.app) + + task = await harness.wait_finished(await start_export(client, {"job_ids": [JOB_A]})) + + assert harness.archive_names(task) == [f"{expected_folder}/a.png"] + assert task.result.get("used_job_time_fallback", False) is used_fallback + assert harness.final_event().get("used_job_time_fallback", False) is used_fallback + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method, path", + [ + ("post", "/api/assets/export"), + ("get", "/api/assets/exports/unknown.zip"), + ("get", f"/api/tasks/{ASSET_ID}"), + ], +) +async def test_every_route_needs_the_assets_system(aiohttp_client, make_harness, dirs, monkeypatch, method, path): + monkeypatch.setattr(routes, "_ASSETS_ENABLED", False) + client = await aiohttp_client(make_harness().app) + + response = await getattr(client, method)(path, json={"job_ids": [JOB_A]}) + + assert response.status == 503 + assert (await response.json())["error"]["code"] == "SERVICE_DISABLED" + + +@pytest.mark.asyncio +async def test_exports_are_private_to_their_owner(aiohttp_client, make_harness, dirs): + harness = make_harness([asset_file(write_file(dirs["output"], "a.png", b"a"))]) + client = await aiohttp_client(harness.app) + task_id = await start_export(client, {"job_ids": [JOB_A]}, user="alice") + name = (await harness.wait_finished(task_id)).export_name + + assert (await client.get(f"/api/tasks/{task_id}", headers={"comfy-user": "bob"})).status == 404 + assert (await client.get(f"/api/assets/exports/{name}", headers={"comfy-user": "bob"})).status == 404 + assert (await client.get(f"/api/assets/exports/{name}", headers={"comfy-user": "alice"})).status == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method, path, kwargs, status, code", + [ + ("post", "/api/assets/export", {"data": "not json"}, 400, "INVALID_REQUEST"), + ("post", "/api/assets/export", {"json": {"job_ids": [JOB_A], "naming_strategy": "x"}}, 400, "INVALID_NAMING_STRATEGY"), + ("get", "/api/assets/exports/bad name.zip", {}, 400, "INVALID_EXPORT_NAME"), + ("get", "/api/assets/exports/nothing.tar", {}, 400, "INVALID_EXPORT_NAME"), + ("get", "/api/assets/exports/unknown.zip", {}, 404, "EXPORT_NOT_FOUND"), + ("get", "/api/tasks/not-a-uuid", {}, 404, "TASK_NOT_FOUND"), + ("get", f"/api/tasks/{uuid.uuid4()}", {}, 404, "TASK_NOT_FOUND"), + ], +) +async def test_invalid_requests_return_contract_errors(aiohttp_client, make_harness, dirs, method, path, kwargs, status, code): + client = await aiohttp_client(make_harness().app) + + response = await getattr(client, method)(path, **kwargs) + + assert response.status == status + assert (await response.json())["error"]["code"] == code + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method, path", + [ + ("post", "/api/assets/export"), + ("get", "/api/assets/exports/unknown.zip"), + ("get", f"/api/tasks/{ASSET_ID}"), + ], +) +async def test_unknown_user_is_rejected(aiohttp_client, make_harness, dirs, method, path): + def reject(request): + raise KeyError("Unknown user") + + client = await aiohttp_client(make_harness(get_user_id=reject).app) + + response = await getattr(client, method)(path, json={"job_ids": [JOB_A]}) + + assert response.status == 401 + + +@pytest.mark.asyncio +async def test_the_oldest_finished_export_is_forgotten_past_the_cap(aiohttp_client, make_harness, dirs, monkeypatch): + monkeypatch.setattr(asset_export, "MAX_TRACKED_EXPORTS", 1) + harness = make_harness([ + asset_file(write_file(dirs["output"], "a.png", b"a")), + asset_file(write_file(dirs["output"], "b.png", b"b"), job_id=JOB_B), + ]) + client = await aiohttp_client(harness.app) + first = await harness.wait_finished(await start_export(client, {"job_ids": [JOB_A]})) + + second = await harness.wait_finished(await start_export(client, {"job_ids": [JOB_B]})) + + assert harness.manager.get_task(first.id) is None + assert harness.manager.get_task(second.id) is second + assert not os.path.exists(harness.manager.export_path(first.export_name)) + assert os.path.isfile(harness.manager.export_path(second.export_name)) diff --git a/tests-unit/assets_test/services/test_export_files.py b/tests-unit/assets_test/services/test_export_files.py new file mode 100644 index 00000000000..12195e5890b --- /dev/null +++ b/tests-unit/assets_test/services/test_export_files.py @@ -0,0 +1,85 @@ +import pytest + +from app.assets.database.queries.records import ( + create_content, + create_record, + mark_content_missing, +) +from app.assets.services.asset_management import get_export_file, list_job_export_files + +JOB_A = "11111111-1111-4111-8111-111111111111" +JOB_B = "22222222-2222-4222-8222-222222222222" + + +def seed_job_records(session): + records = {} + for key, path, job_id, tags in [ + ("saved", "/output/shots/a.exr", JOB_A, ["output"]), + ("preview", "/temp/p.png", JOB_A, ["temp"]), + ("both", "/output/both.png", JOB_A, ["temp", "output"]), + ("other_job", "/output/b.png", JOB_B, ["output"]), + ("input", "/input/ref.png", JOB_A, ["input"]), + ("missing", "/output/gone.png", JOB_A, ["output"]), + ]: + content = create_content(session, path, hash=f"blake3:{key}") + records[key] = create_record(session, content.id, path.rsplit("/", 1)[-1], job_id=job_id, tags=tags) + mark_content_missing(session, records["missing"].content_id) + session.commit() + return records + + +@pytest.mark.parametrize( + "include_previews, expected", + [ + (False, ["both", "saved"]), + (True, ["both", "preview", "saved"]), + ], +) +def test_job_export_files_are_live_outputs_plus_requested_previews( + session, mock_create_session, include_previews, expected +) -> None: + records = seed_job_records(session) + key_by_id = {record.id: key for key, record in records.items()} + + files = list_job_export_files([JOB_A], include_previews) + + assert sorted(key_by_id[f.id] for f in files) == expected + + +def test_job_export_files_carry_the_content_path_and_hash(session, mock_create_session) -> None: + records = seed_job_records(session) + + [saved] = [f for f in list_job_export_files([JOB_A], False) if f.id == records["saved"].id] + + assert saved.name == "a.exr" + assert saved.job_id == JOB_A + assert saved.path.replace("\\", "/").endswith("/output/shots/a.exr") + assert saved.hash == "blake3:saved" + + +def test_job_export_files_for_no_jobs_is_empty(session, mock_create_session) -> None: + seed_job_records(session) + + assert list_job_export_files([], True) == [] + + +@pytest.mark.parametrize("key", ["saved", "preview", "input"]) +def test_export_file_resolves_a_record(session, mock_create_session, key) -> None: + records = seed_job_records(session) + + file = get_export_file(records[key].id) + + assert file.id == records[key].id + assert file.name == records[key].name + + +@pytest.mark.parametrize( + "reference, error", + [("unknown", ValueError), ("missing", FileNotFoundError)], +) +def test_export_file_rejects_unknown_or_missing_records(session, mock_create_session, reference, error) -> None: + records = seed_job_records(session) + reference_id = records[reference].id if reference in records else "33333333-3333-4333-8333-333333333333" + + with pytest.raises(error): + get_export_file(reference_id) diff --git a/tests/execution/test_jobs.py b/tests/execution/test_jobs.py index ffa06943514..5086f18b7fd 100644 --- a/tests/execution/test_jobs.py +++ b/tests/execution/test_jobs.py @@ -9,6 +9,7 @@ normalize_history_item, normalize_output_item, normalize_outputs, + get_job_create_times, get_outputs_summary, count_previewable_outputs, apply_sorting, @@ -495,6 +496,33 @@ def test_sort_with_none_values(self): assert result[0]['id'] == 'b' # None treated as 0, comes first +class TestGetJobCreateTimes: + """get_job_create_times reads each job's create_time wherever the job still lives.""" + + @staticmethod + def queue_item(prompt_id, extra_data): + return (1, prompt_id, {'nodes': {}}, extra_data, ['node1']) + + @pytest.mark.parametrize( + "running, queued, history, expected", + [ + ([], [], {'job-h': {'prompt': (1, 'job-h', {}, {'create_time': 1000}, [])}}, {'job-h': 1000}), + ([('job-r', {'create_time': 2000})], [], {}, {'job-r': 2000}), + ([], [('job-q', {'create_time': 3000})], {}, {'job-q': 3000}), + ([('job-r', {})], [], {}, {}), + ([], [], {'job-h': {'prompt': (1, 'job-h', {}, {'create_time': '1000'}, [])}}, {}), + ([], [('job-other', {'create_time': 4000})], {}, {}), + ], + ids=["history", "running", "queued", "missing create_time", "non-integer create_time", "unrequested job"], + ) + def test_reads_create_times_of_known_jobs(self, running, queued, history, expected): + requested = ['job-h', 'job-r', 'job-q', 'job-gone'] + running_items = [self.queue_item(prompt_id, extra) for prompt_id, extra in running] + queued_items = [self.queue_item(prompt_id, extra) for prompt_id, extra in queued] + + assert get_job_create_times(requested, running_items, queued_items, history) == expected + + class TestNormalizeQueueItem: """Unit tests for normalize_queue_item()"""