From 5e14d797169e6417e1bd45082ff8d78cdc32b3c5 Mon Sep 17 00:00:00 2001 From: dante01yoon Date: Thu, 10 Sep 2026 21:43:53 +0000 Subject: [PATCH 1/2] Support secure refreshed API node credentials --- comfy_api/credential_registry.py | 148 +++++ comfy_api/feature_flags.py | 12 +- comfy_api_nodes/util/_helpers.py | 30 +- comfy_api_nodes/util/client.py | 36 +- execution.py | 54 +- main.py | 1 + server.py | 181 +++++- .../credential_registry_test.py | 574 ++++++++++++++++++ tests-unit/feature_flags_test.py | 10 + 9 files changed, 1013 insertions(+), 33 deletions(-) create mode 100644 comfy_api/credential_registry.py create mode 100644 tests-unit/comfy_api_nodes_test/credential_registry_test.py diff --git a/comfy_api/credential_registry.py b/comfy_api/credential_registry.py new file mode 100644 index 00000000000..dbd6d8eeda7 --- /dev/null +++ b/comfy_api/credential_registry.py @@ -0,0 +1,148 @@ +import threading +import time +from dataclasses import dataclass, field + + +DEFAULT_CREDENTIAL_TTL = 2 * 60 * 60 +DEFAULT_PROMPT_TTL = 24 * 60 * 60 + + +@dataclass(frozen=True) +class Credential: + token: str | None = field(repr=False) + generation: int + + +@dataclass +class _ClientCredential: + token: str | None = field(repr=False) + generation: int + updated_at: float + connected: bool = True + + +@dataclass +class _PromptClient: + client_id: str + updated_at: float + + +class CredentialRegistry: + def __init__( + self, + *, + credential_ttl: float = DEFAULT_CREDENTIAL_TTL, + prompt_ttl: float = DEFAULT_PROMPT_TTL, + clock=time.monotonic, + ): + self.credential_ttl = credential_ttl + self.prompt_ttl = prompt_ttl + self._clock = clock + self._lock = threading.RLock() + self._clients: dict[str, _ClientCredential] = {} + self._prompts: dict[str, _PromptClient] = {} + + def update(self, client_id: str, token: str | None) -> int: + now = self._clock() + with self._lock: + self._cleanup(now) + current = self._clients.get(client_id) + if current is None: + generation = 1 + elif current.token != token: + generation = current.generation + 1 + else: + current.updated_at = now + current.connected = True + return current.generation + self._clients[client_id] = _ClientCredential(token, generation, now) + return generation + + def bind_prompt(self, prompt_id: str, client_id: str) -> bool: + now = self._clock() + with self._lock: + self._cleanup(now) + if prompt_id in self._prompts: + return False + self._prompts[prompt_id] = _PromptClient(client_id, now) + current = self._clients.get(client_id) + if current is not None: + current.connected = True + return True + + def connect(self, client_id: str) -> None: + with self._lock: + current = self._clients.get(client_id) + if current is not None: + current.connected = True + + def get_for_prompt(self, prompt_id: str) -> Credential | None: + now = self._clock() + with self._lock: + self._cleanup(now) + prompt = self._prompts.get(prompt_id) + if prompt is None: + return None + prompt.updated_at = now + current = self._clients.get(prompt.client_id) + if current is None: + return None + return Credential(current.token, current.generation) + + def release_prompt(self, prompt_id: str) -> str | None: + with self._lock: + prompt = self._prompts.pop(prompt_id, None) + if prompt is None: + return None + current = self._clients.get(prompt.client_id) + if current is not None and not current.connected and not self._has_prompts(prompt.client_id): + self._clients.pop(prompt.client_id, None) + return prompt.client_id + if current is None and not self._has_prompts(prompt.client_id): + return prompt.client_id + return None + + def disconnect(self, client_id: str) -> bool: + with self._lock: + current = self._clients.get(client_id) + if current is None: + return self._has_prompts(client_id) + current.connected = False + if not self._has_prompts(client_id): + self._clients.pop(client_id, None) + return False + return True + + def is_protected(self, client_id: str) -> bool: + with self._lock: + self._cleanup(self._clock()) + return client_id in self._clients or self._has_prompts(client_id) + + def clear(self) -> None: + with self._lock: + self._clients.clear() + self._prompts.clear() + + def _has_prompts(self, client_id: str) -> bool: + return any(prompt.client_id == client_id for prompt in self._prompts.values()) + + def _cleanup(self, now: float) -> None: + expired_prompts = [ + prompt_id for prompt_id, prompt in self._prompts.items() if now - prompt.updated_at >= self.prompt_ttl + ] + for prompt_id in expired_prompts: + self._prompts.pop(prompt_id, None) + + active_clients = {prompt.client_id for prompt in self._prompts.values()} + expired_clients = [ + client_id + for client_id, credential in self._clients.items() + if not credential.connected + and client_id not in active_clients + and now - credential.updated_at >= self.credential_ttl + ] + for client_id in expired_clients: + self._clients.pop(client_id, None) + + +api_node_credentials = CredentialRegistry() diff --git a/comfy_api/feature_flags.py b/comfy_api/feature_flags.py index 9fbef82609a..cd6628a684b 100644 --- a/comfy_api/feature_flags.py +++ b/comfy_api/feature_flags.py @@ -106,6 +106,11 @@ def _parse_cli_feature_flags() -> dict[str, Any]: _CORE_FEATURE_FLAGS: dict[str, Any] = { "supports_preview_metadata": True, "supports_model_type_tags": True, + "comfy_api_credentials": { + "version": 1, + "endpoint": "/api/credentials", + "websocket_auth_message": "credential_auth", + }, "max_upload_size": args.max_upload_size * 1024 * 1024, # Convert MB to bytes "extension": {"manager": {"supports_v4": True}}, "node_replacements": True, @@ -161,11 +166,14 @@ def supports_feature( return get_connection_feature(sockets_metadata, sid, feature_name, False) is True -def get_server_features() -> dict[str, Any]: +def get_server_features(*, include_credentials: bool = True) -> dict[str, Any]: """ Get the server's feature flags. Returns: Dictionary of server feature flags """ - return SERVER_FEATURE_FLAGS.copy() + features = SERVER_FEATURE_FLAGS.copy() + if not include_credentials: + features.pop("comfy_api_credentials", None) + return features diff --git a/comfy_api_nodes/util/_helpers.py b/comfy_api_nodes/util/_helpers.py index ddfb3b65c9d..b82a1c8f262 100644 --- a/comfy_api_nodes/util/_helpers.py +++ b/comfy_api_nodes/util/_helpers.py @@ -14,6 +14,7 @@ from comfy.comfy_api_env import normalize_comfy_api_base from comfy.deploy_environment import get_deploy_environment from comfy.model_management import processing_interrupted +from comfy_api.credential_registry import api_node_credentials from comfy_api.latest import IO from comfy_execution.utils import get_executing_context from comfyui_version import __version__ as comfyui_version @@ -33,12 +34,24 @@ def get_node_id(node_cls: type[IO.ComfyNode]) -> str: return node_cls.hidden.unique_id -def get_auth_header(node_cls: type[IO.ComfyNode]) -> dict[str, str]: +def get_auth_header_with_generation(node_cls: type[IO.ComfyNode]) -> tuple[dict[str, str], int | None]: + ctx = get_executing_context() + if ctx is not None and not node_cls.hidden.api_key_comfy_org: + credential = api_node_credentials.get_for_prompt(ctx.prompt_id) + if credential is not None: + if credential.token: + return {"Authorization": f"Bearer {credential.token}"}, credential.generation + return {}, credential.generation + if node_cls.hidden.auth_token_comfy_org: - return {"Authorization": f"Bearer {node_cls.hidden.auth_token_comfy_org}"} + return {"Authorization": f"Bearer {node_cls.hidden.auth_token_comfy_org}"}, None if node_cls.hidden.api_key_comfy_org: - return {"X-API-KEY": node_cls.hidden.api_key_comfy_org} - return {} + return {"X-API-KEY": node_cls.hidden.api_key_comfy_org}, None + return {}, None + + +def get_auth_header(node_cls: type[IO.ComfyNode]) -> dict[str, str]: + return get_auth_header_with_generation(node_cls)[0] def get_usage_source(node_cls: type[IO.ComfyNode]) -> str: @@ -58,8 +71,13 @@ def get_comfy_api_headers(node_cls: type[IO.ComfyNode]) -> dict[str, str]: relative/cloud URLs resolved against ``default_base_url()``; because the result includes auth, callers must not attach it to arbitrary absolute/presigned URLs. """ + return get_comfy_api_headers_with_generation(node_cls)[0] + + +def get_comfy_api_headers_with_generation(node_cls: type[IO.ComfyNode]) -> tuple[dict[str, str], int | None]: + auth_headers, generation = get_auth_header_with_generation(node_cls) headers = { - **get_auth_header(node_cls), + **auth_headers, "Comfy-Env": get_deploy_environment(), "Comfy-Usage-Source": get_usage_source(node_cls), "Comfy-Core-Version": comfyui_version, @@ -67,7 +85,7 @@ def get_comfy_api_headers(node_cls: type[IO.ComfyNode]) -> dict[str, str]: ctx = get_executing_context() if ctx is not None: headers["Comfy-Job-Id"] = ctx.prompt_id - return headers + return headers, generation def default_base_url() -> str: diff --git a/comfy_api_nodes/util/client.py b/comfy_api_nodes/util/client.py index 4f62c297b9b..7834b88a0fd 100644 --- a/comfy_api_nodes/util/client.py +++ b/comfy_api_nodes/util/client.py @@ -25,7 +25,7 @@ from ._helpers import ( _retry_after_wait, default_base_url, - get_comfy_api_headers, + get_comfy_api_headers_with_generation, get_node_id, is_processing_interrupted, sleep_with_interrupt, @@ -71,6 +71,7 @@ class _RequestConfig: price_extractor: Callable[[dict[str, Any]], float | None] | None = None is_rate_limited: Callable[[int, Any], bool] | None = None response_header_validator: Callable[[dict[str, str]], None] | None = None + retry_401_on_auth_change: bool = False @dataclass @@ -236,6 +237,7 @@ async def sync_op_raw( max_retries_on_rate_limit: int = 16, is_rate_limited: Callable[[int, Any], bool] | None = None, response_header_validator: Callable[[dict[str, str]], None] | None = None, + retry_401_on_auth_change: bool = False, ) -> dict[str, Any] | bytes: """ Make a single network request. @@ -268,6 +270,7 @@ async def sync_op_raw( max_retries_on_rate_limit=max_retries_on_rate_limit, is_rate_limited=is_rate_limited, response_header_validator=response_header_validator, + retry_401_on_auth_change=retry_401_on_auth_change, ) return await _request_base(cfg, expect_binary=as_binary) @@ -355,6 +358,7 @@ async def _ticker(): as_binary=False, final_label_on_success=None, monitor_progress=False, + retry_401_on_auth_change=True, ) if not isinstance(resp_json, dict): raise Exception("Polling endpoint returned non-JSON response.") @@ -709,6 +713,8 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): operation_succeeded: bool = False final_elapsed_seconds: int | None = None extracted_price: float | None = None + retried_after_auth_change = False + auth_change_attempts = 0 while True: attempt += 1 stop_event = asyncio.Event() @@ -719,10 +725,15 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): logging.debug("[DEBUG] HTTP %s %s (attempt %d)", method, url, attempt) payload_headers = {"Accept": "*/*"} if expect_binary else {"Accept": "application/json"} + auth_generation: int | None = None if is_comfy_api_request: - payload_headers.update(get_comfy_api_headers(cfg.node_cls)) + comfy_headers, auth_generation = get_comfy_api_headers_with_generation(cfg.node_cls) + payload_headers.update(comfy_headers) if cfg.endpoint.headers: payload_headers.update(cfg.endpoint.headers) + endpoint_header_names = {name.lower() for name in cfg.endpoint.headers} + if "authorization" in endpoint_header_names or "x-api-key" in endpoint_header_names: + auth_generation = None payload_kw: dict[str, Any] = {"headers": payload_headers} if method == "GET": @@ -811,6 +822,19 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): is_rl = resp.status == 429 or ( cfg.is_rate_limited is not None and cfg.is_rate_limited(resp.status, body) ) + if ( + resp.status == 401 + and cfg.retry_401_on_auth_change + and method.upper() in {"GET", "HEAD"} + and auth_generation is not None + and not retried_after_auth_change + ): + latest_headers, latest_generation = get_comfy_api_headers_with_generation(cfg.node_cls) + if latest_generation != auth_generation and "Authorization" in latest_headers: + retried_after_auth_change = True + auth_change_attempts += 1 + retry_label = "credential refresh retry" + should_retry = True if is_rl and rate_limit_attempts < cfg.max_retries_on_rate_limit: rate_limit_attempts += 1 wait_time = min(rate_limit_delay, 30.0) @@ -820,11 +844,11 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): elif ( resp.status in _RETRY_STATUS and not _is_terminal_service_refusal(body) - and (attempt - rate_limit_attempts) <= cfg.max_retries + and (attempt - rate_limit_attempts - auth_change_attempts) <= cfg.max_retries ): wait_time = delay delay *= cfg.retry_backoff - retry_label = f"retry {attempt - rate_limit_attempts} of {cfg.max_retries}" + retry_label = f"retry {attempt - rate_limit_attempts - auth_change_attempts} of {cfg.max_retries}" should_retry = True if should_retry: @@ -932,13 +956,13 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): logging.debug("Polling was interrupted by user") raise except (ClientError, OSError) as e: - if (attempt - rate_limit_attempts) <= cfg.max_retries: + if (attempt - rate_limit_attempts - auth_change_attempts) <= cfg.max_retries: logging.warning( "Connection error calling %s %s. Retrying in %.2fs (%d/%d): %s", method, url, delay, - attempt - rate_limit_attempts, + attempt - rate_limit_attempts - auth_change_attempts, cfg.max_retries, str(e), ) diff --git a/execution.py b/execution.py index 48c6f641b2e..cbd5e18dc55 100644 --- a/execution.py +++ b/execution.py @@ -421,6 +421,39 @@ def format_value(x): else: return str(x) + +def format_input_data(input_data_all): + formatted = {} + for name, inputs in input_data_all.items(): + if name in SENSITIVE_EXTRA_DATA_KEYS: + formatted[name] = ["***"] + else: + formatted[name] = [format_value(x) for x in inputs] + return formatted + + +def redact_sensitive_text(text, extra_data): + sensitive_values = [ + value + for key in SENSITIVE_EXTRA_DATA_KEYS + if isinstance((value := extra_data.get(key)), str) and value + ] + for value in sorted(sensitive_values, key=len, reverse=True): + text = text.replace(value, "***") + return text + + +def log_execution_exception(ex, tb, extra_data): + exception_message = redact_sensitive_text(str(ex), extra_data) + traceback_lines = [redact_sensitive_text(line, extra_data) for line in traceback.format_tb(tb)] + formatted_exception = "".join( + redact_sensitive_text(line, extra_data) for line in traceback.format_exception(type(ex), ex, tb) + ) + logging.error("!!! Exception during processing !!! %s", exception_message) + logging.error(formatted_exception) + return exception_message, traceback_lines + + def _is_intermediate_output(dynprompt, node_id): class_type = dynprompt.get_node(node_id)["class_type"] class_def = nodes.NODE_CLASS_MAPPINGS[class_type] @@ -630,12 +663,9 @@ async def await_completion(): exception_type = full_type_name(typ) input_data_formatted = {} if input_data_all is not None: - input_data_formatted = {} - for name, inputs in input_data_all.items(): - input_data_formatted[name] = [format_value(x) for x in inputs] + input_data_formatted = format_input_data(input_data_all) - logging.error(f"!!! Exception during processing !!! {ex}") - logging.error(traceback.format_exc()) + exception_message, traceback_lines = log_execution_exception(ex, tb, extra_data) tips = "" if comfy.model_management.is_oom(ex): @@ -648,9 +678,9 @@ async def await_completion(): error_details = { "node_id": real_node_id, - "exception_message": "{}\n{}".format(ex, tips), + "exception_message": "{}\n{}".format(exception_message, tips), "exception_type": exception_type, - "traceback": traceback.format_tb(tb), + "traceback": traceback_lines, "current_inputs": input_data_formatted } @@ -1261,9 +1291,17 @@ def __init__(self, server): def put(self, item): with self.mutex: + prompt_id = item[1] + if ( + prompt_id in self.history + or any(queued[1] == prompt_id for queued in self.queue) + or any(running[1] == prompt_id for running in self.currently_running.values()) + ): + return False heapq.heappush(self.queue, item) self.server.queue_updated() self.not_empty.notify() + return True def get(self, timeout=None): with self.not_empty: @@ -1345,8 +1383,10 @@ def get_tasks_remaining(self): def wipe_queue(self): with self.mutex: + removed = self.queue self.queue = [] self.server.queue_updated() + return removed def delete_queue_item(self, function): with self.mutex: diff --git a/main.py b/main.py index 372c9aead96..2d4fa2d9fbe 100644 --- a/main.py +++ b/main.py @@ -402,6 +402,7 @@ def prompt_worker(q, server_instance): status_str='success' if e.success else 'error', completed=e.success, messages=e.status_messages), process_item=remove_sensitive) + server_instance.release_api_node_prompt(prompt_id) if server_instance.client_id is not None: server_instance.send_sync("executing", {"node": None, "prompt_id": prompt_id}, server_instance.client_id) diff --git a/server.py b/server.py index c9ffcaa0da9..44e17d832f5 100644 --- a/server.py +++ b/server.py @@ -25,6 +25,8 @@ import ssl import socket import ipaddress +import hmac +import secrets from PIL import Image, ImageOps from PIL.PngImagePlugin import PngInfo from io import BytesIO @@ -39,6 +41,7 @@ import comfy.utils import comfy.model_management from comfy_api import feature_flags +from comfy_api.credential_registry import api_node_credentials from comfy.comfy_api_env import get_environment_overrides import node_helpers from comfyui_version import __version__ @@ -123,7 +126,7 @@ async def cors_middleware(request: web.Request, handler): response.headers['Access-Control-Allow-Origin'] = allowed_origin response.headers['Access-Control-Allow-Methods'] = 'POST, GET, DELETE, PUT, OPTIONS, PATCH' - response.headers['Access-Control-Allow-Headers'] = 'Content-Type, Authorization' + response.headers['Access-Control-Allow-Headers'] = 'Content-Type, Authorization, X-Comfy-Client-Id, X-Comfy-Credential-Key' response.headers['Access-Control-Allow-Credentials'] = 'true' return response @@ -156,6 +159,39 @@ def is_loopback(host): return loopback +_CLIENT_ID_HEADER = "X-Comfy-Client-Id" +_CREDENTIAL_KEY_HEADER = "X-Comfy-Credential-Key" + + +def _credential_transport_enabled(request: web.Request) -> bool: + """Allow credentials only over direct TLS or from a direct loopback peer.""" + transport = request.transport + if transport is None: + return False + if transport.get_extra_info("sslcontext") is not None or transport.get_extra_info("ssl_object") is not None: + return True + + peername = transport.get_extra_info("peername") + if not isinstance(peername, (tuple, list)) or not peername: + return False + try: + return ipaddress.ip_address(peername[0].split("%", 1)[0]).is_loopback + except (AttributeError, ValueError): + return False + + +def _authenticated_client_id(request: web.Request, sockets_metadata: dict) -> str | None: + client_id = request.headers.get(_CLIENT_ID_HEADER) + credential_key = request.headers.get(_CREDENTIAL_KEY_HEADER) + if not client_id or not credential_key: + return None + metadata = sockets_metadata.get(client_id) + expected_key = metadata.get("credential_key") if metadata else None + if not expected_key or not hmac.compare_digest(credential_key, expected_key): + return None + return client_id + + def create_origin_only_middleware(): @web.middleware async def origin_only_middleware(request: web.Request, handler): @@ -268,9 +304,46 @@ def __init__(self, loop): @routes.get('/ws') async def websocket_handler(request): + credential_transport_enabled = _credential_transport_enabled(request) + sid = request.rel_url.query.get('clientId', '') + existing_metadata = self.sockets_metadata.get(sid) if sid else None + if existing_metadata is not None and sid not in self.sockets and not api_node_credentials.is_protected(sid): + self.sockets_metadata.pop(sid, None) + existing_metadata = None + existing_credential_key = existing_metadata.get("credential_key") if existing_metadata else None + if not credential_transport_enabled and existing_credential_key is not None: + # Do not let a legacy plaintext connection replace protected + # credential state for the requested client id. + sid = "" + existing_metadata = None + existing_credential_key = None + requires_credential_auth = credential_transport_enabled and existing_credential_key is not None + ws = web.WebSocketResponse() await ws.prepare(request) - sid = request.rel_url.query.get('clientId', '') + if requires_credential_auth: + try: + auth_message = await ws.receive(timeout=5) + auth_data = json.loads(auth_message.data) if auth_message.type == aiohttp.WSMsgType.TEXT else {} + except (asyncio.TimeoutError, json.JSONDecodeError): + auth_data = {} + if not isinstance(auth_data, dict): + auth_data = {} + auth_payload = auth_data.get("data") + presented_key = ( + auth_payload.get("credential_key") + if auth_data.get("type") == "credential_auth" and isinstance(auth_payload, dict) + else None + ) + expected_key = existing_metadata.get("credential_key") + if ( + not isinstance(presented_key, str) + or not isinstance(expected_key, str) + or not hmac.compare_digest(presented_key, expected_key) + ): + await ws.close(code=aiohttp.WSCloseCode.POLICY_VIOLATION, message=b"Invalid client credentials") + return ws + if sid: # Reusing existing session, remove old self.sockets.pop(sid, None) @@ -280,17 +353,38 @@ async def websocket_handler(request): # Store WebSocket for backward compatibility self.sockets[sid] = ws # Store metadata separately - self.sockets_metadata[sid] = {"feature_flags": {}} + credential_key = existing_credential_key or ( + secrets.token_urlsafe(32) if credential_transport_enabled else None + ) + connection_id = secrets.token_hex(16) + self.sockets_metadata[sid] = { + "feature_flags": {}, + "connection_id": connection_id, + } + if credential_key is not None: + self.sockets_metadata[sid]["credential_key"] = credential_key + api_node_credentials.connect(sid) try: # Send initial state to the new client - await self.send("status", {"status": self.get_queue_info(), "sid": sid}, sid) + status_data = { + "status": self.get_queue_info(), + "sid": sid, + } + if credential_key is not None: + status_data["credential_key"] = credential_key + await self.send( + "status", + status_data, + sid, + ) # On reconnect if we are the currently executing client send the current node if self.client_id == sid and self.last_node_id is not None: await self.send("executing", { "node": self.last_node_id }, sid) # Flag to track if we've received the first message first_message = True + allow_credential_auth_prelude = not requires_credential_auth async for msg in ws: if msg.type == aiohttp.WSMsgType.ERROR: @@ -298,6 +392,16 @@ async def websocket_handler(request): elif msg.type == aiohttp.WSMsgType.TEXT: try: data = json.loads(msg.data) + if not isinstance(data, dict): + first_message = False + continue + if ( + first_message + and allow_credential_auth_prelude + and data.get("type") == "credential_auth" + ): + allow_credential_auth_prelude = False + continue # Check if first message is feature flags if first_message and data.get("type") == "feature_flags": # Store client feature flags @@ -307,7 +411,9 @@ async def websocket_handler(request): # Send server feature flags in response await self.send( "feature_flags", - feature_flags.get_server_features(), + feature_flags.get_server_features( + include_credentials=credential_transport_enabled + ), sid, ) @@ -322,8 +428,11 @@ async def websocket_handler(request): except Exception as e: logging.error(f"Error processing WebSocket message: {e}") finally: - self.sockets.pop(sid, None) - self.sockets_metadata.pop(sid, None) + metadata = self.sockets_metadata.get(sid) + if metadata is not None and metadata.get("connection_id") == connection_id: + self.sockets.pop(sid, None) + if not credential_transport_enabled or not api_node_credentials.disconnect(sid): + self.sockets_metadata.pop(sid, None) return ws @routes.get("/") @@ -738,12 +847,35 @@ async def system_stats(request): @routes.get("/features") async def get_features(request): - features = feature_flags.get_server_features() + features = feature_flags.get_server_features( + include_credentials=_credential_transport_enabled(request) + ) overrides = get_environment_overrides() if overrides: features.update(overrides) return web.json_response(features) + @routes.post("/credentials") + async def update_api_credentials(request): + if not _credential_transport_enabled(request): + return web.json_response({"error": "Credential transport requires HTTPS or loopback"}, status=403) + client_id = _authenticated_client_id(request, self.sockets_metadata) + if client_id is None: + return web.json_response({"error": "Invalid client credentials"}, status=403) + + try: + json_data = await request.json() + except (json.JSONDecodeError, TypeError): + return web.json_response({"error": "Request body must be valid JSON"}, status=400) + if not isinstance(json_data, dict) or "auth_token_comfy_org" not in json_data: + return web.json_response({"error": "auth_token_comfy_org is required"}, status=400) + token = json_data["auth_token_comfy_org"] + if token is not None and (not isinstance(token, str) or not token or len(token) > 16384): + return web.json_response({"error": "auth_token_comfy_org must be a non-empty string or null"}, status=400) + + generation = api_node_credentials.update(client_id, token) + return web.json_response({"generation": generation}) + @routes.get("/prompt") async def get_prompt(request): return web.json_response(self.get_queue_info()) @@ -963,7 +1095,10 @@ def interrupt(prompt_id): def dequeue(prompt_id): logging.info(f"Cancelling pending prompt {prompt_id}") - return self.prompt_queue.delete_queue_item(lambda a: a[1] == prompt_id) + deleted = self.prompt_queue.delete_queue_item(lambda a: a[1] == prompt_id) + if deleted: + self.release_api_node_prompt(prompt_id) + return deleted classification = cancel_job(job_id, running, queued, history, interrupt, dequeue) return classification in (CANCEL_RUNNING, CANCEL_PENDING) @@ -1072,8 +1207,16 @@ async def get_queue(request): @routes.post("/prompt") async def post_prompt(request): logging.info("got prompt") + has_credential_headers = _CLIENT_ID_HEADER in request.headers or _CREDENTIAL_KEY_HEADER in request.headers + if has_credential_headers and not _credential_transport_enabled(request): + return web.json_response({"error": "Credential transport requires HTTPS or loopback"}, status=403) json_data = await request.json() json_data = self.trigger_on_prompt(json_data) + authenticated_client = _authenticated_client_id(request, self.sockets_metadata) + if has_credential_headers and authenticated_client is None: + return web.json_response({"error": "Invalid client credentials"}, status=403) + if authenticated_client is not None and json_data.get("client_id") != authenticated_client: + return web.json_response({"error": "client_id does not match authenticated client"}, status=403) if "number" in json_data: number = float(json_data['number']) @@ -1128,7 +1271,13 @@ async def post_prompt(request): if sensitive_val in extra_data: sensitive[sensitive_val] = extra_data.pop(sensitive_val) extra_data["create_time"] = int(time.time() * 1000) # timestamp in milliseconds - self.prompt_queue.put((number, prompt_id, prompt, extra_data, outputs_to_execute, sensitive)) + if authenticated_client is not None and not api_node_credentials.bind_prompt(prompt_id, authenticated_client): + return web.json_response({"error": "prompt_id already exists"}, status=409) + queued = self.prompt_queue.put((number, prompt_id, prompt, extra_data, outputs_to_execute, sensitive)) + if not queued: + if authenticated_client is not None: + self.release_api_node_prompt(prompt_id) + return web.json_response({"error": "prompt_id already exists"}, status=409) response = {"prompt_id": prompt_id, "number": number, "node_errors": valid[3]} return web.json_response(response) else: @@ -1148,12 +1297,15 @@ async def post_queue(request): json_data = await request.json() if "clear" in json_data: if json_data["clear"]: - self.prompt_queue.wipe_queue() + removed = self.prompt_queue.wipe_queue() + for item in removed: + self.release_api_node_prompt(item[1]) if "delete" in json_data: to_delete = json_data['delete'] for id_to_delete in to_delete: delete_func = lambda a: a[1] == id_to_delete - self.prompt_queue.delete_queue_item(delete_func) + if self.prompt_queue.delete_queue_item(delete_func): + self.release_api_node_prompt(id_to_delete) return web.Response(status=200) @@ -1287,6 +1439,11 @@ def get_queue_info(self): prompt_info['exec_info'] = exec_info return prompt_info + def release_api_node_prompt(self, prompt_id): + client_id = api_node_credentials.release_prompt(prompt_id) + if client_id is not None and client_id not in self.sockets: + self.sockets_metadata.pop(client_id, None) + async def send(self, event, data, sid=None): if event == BinaryEventTypes.UNENCODED_PREVIEW_IMAGE: await self.send_image(data, sid=sid) diff --git a/tests-unit/comfy_api_nodes_test/credential_registry_test.py b/tests-unit/comfy_api_nodes_test/credential_registry_test.py new file mode 100644 index 00000000000..4d445131b44 --- /dev/null +++ b/tests-unit/comfy_api_nodes_test/credential_registry_test.py @@ -0,0 +1,574 @@ +import asyncio +import json +import sys +from types import SimpleNamespace + +import pytest +import torch +from aiohttp import WSCloseCode, WSMsgType, web + +from comfy.cli_args import args + +if not torch.cuda.is_available(): + args.cpu = True + +import execution +from comfy_api.credential_registry import CredentialRegistry, api_node_credentials +from comfy_api_nodes.util import _helpers +from comfy_api_nodes.util import client as api_client +from comfy_api_nodes.util import request_logger +from comfy_execution.utils import CurrentNodeContext +from server import PromptServer, _authenticated_client_id, _credential_transport_enabled + + +class _Node: + hidden = SimpleNamespace( + auth_token_comfy_org="snapshot-token", + api_key_comfy_org=None, + comfy_usage_source=None, + unique_id="node", + ) + + +class _Transport: + def __init__(self, *, peername, sslcontext=None, ssl_object=None): + self._extra = { + "peername": peername, + "sslcontext": sslcontext, + "ssl_object": ssl_object, + } + + def get_extra_info(self, name): + return self._extra.get(name) + + +@pytest.fixture(autouse=True) +def clear_credentials(monkeypatch): + api_node_credentials.clear() + monkeypatch.setattr(api_client, "is_processing_interrupted", lambda: False) + monkeypatch.setattr(api_client, "_display_time_progress", lambda *args, **kwargs: None) + yield + api_node_credentials.clear() + + +def test_registry_isolates_clients_and_late_binds_updates(): + registry = CredentialRegistry() + registry.update("client-a", "a1") + registry.update("client-b", "b1") + assert registry.bind_prompt("prompt-a", "client-a") is True + assert registry.bind_prompt("prompt-b", "client-b") is True + assert registry.bind_prompt("prompt-a", "client-b") is False + + assert registry.get_for_prompt("prompt-a").token == "a1" + assert registry.get_for_prompt("prompt-b").token == "b1" + + first_generation = registry.get_for_prompt("prompt-a").generation + registry.update("client-a", "a2") + assert registry.get_for_prompt("prompt-a").token == "a2" + assert registry.get_for_prompt("prompt-a").generation == first_generation + 1 + assert registry.get_for_prompt("prompt-b").token == "b1" + + +def test_registry_cleanup_on_completion_disconnect_and_ttl(): + now = [0.0] + registry = CredentialRegistry(credential_ttl=10, prompt_ttl=20, clock=lambda: now[0]) + registry.update("client", "secret") + registry.bind_prompt("prompt", "client") + registry.disconnect("client") + assert registry.get_for_prompt("prompt").token == "secret" + + registry.release_prompt("prompt") + assert registry.get_for_prompt("prompt") is None + + registry.update("stale-client", "stale") + registry.bind_prompt("stale-prompt", "stale-client") + now[0] = 21 + assert registry.get_for_prompt("stale-prompt") is None + + +def test_connected_registry_credentials_do_not_expire(): + now = [0.0] + registry = CredentialRegistry(credential_ttl=10, clock=lambda: now[0]) + registry.connect("client") + registry.update("client", "secret") + registry.bind_prompt("prompt", "client") + registry.release_prompt("prompt") + + now[0] = 11 + registry.bind_prompt("next-prompt", "client") + assert registry.get_for_prompt("next-prompt").token == "secret" + + +def test_auth_header_uses_registry_then_falls_back_and_preserves_api_key(): + with CurrentNodeContext("legacy-prompt", "node"): + assert _helpers.get_auth_header(_Node) == {"Authorization": "Bearer snapshot-token"} + + api_node_credentials.update("client", "fresh-token") + api_node_credentials.bind_prompt("prompt", "client") + with CurrentNodeContext("prompt", "node"): + assert _helpers.get_auth_header(_Node) == {"Authorization": "Bearer fresh-token"} + + api_node_credentials.update("client", None) + with CurrentNodeContext("prompt", "node"): + assert _helpers.get_auth_header(_Node) == {} + + api_key_node = type( + "ApiKeyNode", + (), + { + "hidden": SimpleNamespace( + auth_token_comfy_org=None, + api_key_comfy_org="api-key", + comfy_usage_source=None, + unique_id="node", + ) + }, + ) + with CurrentNodeContext("prompt", "node"): + assert _helpers.get_auth_header(api_key_node) == {"X-API-KEY": "api-key"} + + api_key_node.hidden.auth_token_comfy_org = "legacy-token" + with CurrentNodeContext("prompt", "node"): + assert _helpers.get_auth_header(api_key_node) == {"Authorization": "Bearer legacy-token"} + + +def test_client_request_authentication_requires_matching_session_key(): + request = SimpleNamespace(headers={"X-Comfy-Client-Id": "client", "X-Comfy-Credential-Key": "key"}) + metadata = {"client": {"credential_key": "key"}} + assert _authenticated_client_id(request, metadata) == "client" + + request.headers["X-Comfy-Credential-Key"] = "wrong" + assert _authenticated_client_id(request, metadata) is None + + +@pytest.mark.parametrize("peername", [("127.0.0.1", 8188), ("::1", 8188, 0, 0)]) +def test_credential_transport_allows_direct_loopback(peername): + request = SimpleNamespace(transport=_Transport(peername=peername), headers={}) + assert _credential_transport_enabled(request) is True + + +def test_credential_transport_allows_direct_tls(): + request = SimpleNamespace( + transport=_Transport(peername=("203.0.113.10", 8188), sslcontext=object()), + headers={}, + ) + assert _credential_transport_enabled(request) is True + + +def test_credential_transport_rejects_plaintext_remote_and_ignores_forwarded_proto(): + request = SimpleNamespace( + transport=_Transport(peername=("203.0.113.10", 8188)), + headers={"X-Forwarded-Proto": "https", "Forwarded": "proto=https"}, + ) + assert _credential_transport_enabled(request) is False + + +@pytest.mark.asyncio +async def test_credential_endpoint_updates_and_clears_without_returning_secret(aiohttp_client): + prompt_server = PromptServer(None) + prompt_server.sockets_metadata["client"] = {"credential_key": "key", "feature_flags": {}} + route = next(route for route in prompt_server.routes if route.path == "/credentials") + app = web.Application() + app.router.add_post("/api/credentials", route.handler) + http_client = await aiohttp_client(app) + headers = {"X-Comfy-Client-Id": "client", "X-Comfy-Credential-Key": "key"} + + response = await http_client.post( + "/api/credentials", + headers=headers, + json={"auth_token_comfy_org": "top-secret"}, + ) + assert response.status == 200 + response_text = await response.text() + assert json.loads(response_text) == {"generation": 1} + assert "top-secret" not in response_text + + response = await http_client.post( + "/api/credentials", + headers=headers, + json={"auth_token_comfy_org": None}, + ) + assert response.status == 200 + assert await response.json() == {"generation": 2} + + response = await http_client.post( + "/api/credentials", + headers={"X-Comfy-Client-Id": "client", "X-Comfy-Credential-Key": "wrong"}, + json={"auth_token_comfy_org": "stolen"}, + ) + assert response.status == 403 + + +@pytest.mark.asyncio +async def test_remote_plaintext_rejects_credentials_but_allows_legacy_prompt( + aiohttp_client, monkeypatch +): + monkeypatch.setattr("server._credential_transport_enabled", lambda request: False) + + async def validate_prompt(*args, **kwargs): + return True, None, [], {} + + monkeypatch.setattr(execution, "validate_prompt", validate_prompt) + prompt_server = PromptServer(None) + monkeypatch.setattr(prompt_server.prompt_queue, "put", lambda item: True) + routes = {route.path: route for route in prompt_server.routes} + app = web.Application() + app.router.add_get("/api/features", routes["/features"].handler) + app.router.add_post("/api/credentials", routes["/credentials"].handler) + app.router.add_post("/api/prompt", routes["/prompt"].handler) + http_client = await aiohttp_client(app) + credential_headers = { + "X-Comfy-Client-Id": "client", + "X-Comfy-Credential-Key": "key", + } + + response = await http_client.post( + "/api/credentials", + headers=credential_headers, + json={"auth_token_comfy_org": "top-secret"}, + ) + assert response.status == 403 + + response = await http_client.post("/api/prompt", headers=credential_headers, json={}) + assert response.status == 403 + + response = await http_client.post( + "/api/prompt", + json={"client_id": "legacy-client", "prompt": {}}, + ) + assert response.status == 200 + assert api_node_credentials.is_protected("legacy-client") is False + + response = await http_client.get("/api/features") + assert response.status == 200 + assert "comfy_api_credentials" not in await response.json() + + +@pytest.mark.asyncio +async def test_remote_plaintext_websocket_omits_credentials_and_reconnects( + aiohttp_client, monkeypatch +): + monkeypatch.setattr("server._credential_transport_enabled", lambda request: False) + prompt_server = PromptServer(None) + route = next(route for route in prompt_server.routes if route.path == "/ws") + app = web.Application() + app.router.add_get("/ws", route.handler) + http_client = await aiohttp_client(app) + + websocket = await http_client.ws_connect("/ws?clientId=legacy-client") + status = await websocket.receive_json() + assert status["data"]["sid"] == "legacy-client" + assert "credential_key" not in status["data"] + await websocket.send_json({"type": "feature_flags", "data": {}}) + server_features = await websocket.receive_json() + assert "comfy_api_credentials" not in server_features["data"] + assert "credential_key" not in prompt_server.sockets_metadata["legacy-client"] + assert api_node_credentials.is_protected("legacy-client") is False + await websocket.close() + await asyncio.sleep(0) + + websocket = await http_client.ws_connect("/ws?clientId=legacy-client") + status = await websocket.receive_json() + assert status["data"]["sid"] == "legacy-client" + assert "credential_key" not in status["data"] + await websocket.close() + + +@pytest.mark.asyncio +async def test_protected_client_id_requires_websocket_credential_on_reconnect(aiohttp_client): + prompt_server = PromptServer(None) + route = next(route for route in prompt_server.routes if route.path == "/ws") + app = web.Application() + app.router.add_get("/ws", route.handler) + http_client = await aiohttp_client(app) + + websocket = await http_client.ws_connect("/ws?clientId=client") + status = await websocket.receive_json() + credential_key = status["data"]["credential_key"] + + attacker = await http_client.ws_connect("/ws?clientId=client") + await attacker.send_json({"type": "credential_auth", "data": {"credential_key": "wrong"}}) + assert (await attacker.receive()).type in {WSMsgType.CLOSE, WSMsgType.CLOSED} + + api_node_credentials.update("client", "top-secret") + api_node_credentials.bind_prompt("prompt", "client") + await websocket.close() + await asyncio.sleep(0) + + websocket = await http_client.ws_connect("/ws?clientId=client") + await websocket.send_json({"type": "credential_auth", "data": {"credential_key": "wrong"}}) + assert (await websocket.receive()).type in {WSMsgType.CLOSE, WSMsgType.CLOSED} + + websocket = await http_client.ws_connect("/ws?clientId=client") + await websocket.send_json( + {"type": "credential_auth", "data": {"credential_key": credential_key}} + ) + assert (await websocket.receive_json())["data"]["sid"] == "client" + await websocket.close() + prompt_server.release_api_node_prompt("prompt") + assert "client" not in prompt_server.sockets_metadata + + +@pytest.mark.parametrize("auth_data", [[], "credential_auth", None]) +@pytest.mark.asyncio +async def test_protected_reconnect_rejects_non_object_credential_auth(auth_data, aiohttp_client): + prompt_server = PromptServer(None) + route = next(route for route in prompt_server.routes if route.path == "/ws") + app = web.Application() + app.router.add_get("/ws", route.handler) + http_client = await aiohttp_client(app) + + websocket = await http_client.ws_connect("/ws?clientId=client") + await websocket.receive_json() + api_node_credentials.update("client", "top-secret") + api_node_credentials.bind_prompt("prompt", "client") + await websocket.close() + await asyncio.sleep(0) + + websocket = await http_client.ws_connect("/ws?clientId=client") + await websocket.send_str(json.dumps(auth_data)) + close_message = await websocket.receive() + assert close_message.type == WSMsgType.CLOSE + assert close_message.data == WSCloseCode.POLICY_VIOLATION + + +@pytest.mark.asyncio +async def test_unprotected_reconnect_negotiates_features_after_stale_credential_auth(aiohttp_client): + prompt_server = PromptServer(None) + route = next(route for route in prompt_server.routes if route.path == "/ws") + app = web.Application() + app.router.add_get("/ws", route.handler) + http_client = await aiohttp_client(app) + + websocket = await http_client.ws_connect("/ws?clientId=client") + status = await websocket.receive_json() + stale_credential_key = status["data"]["credential_key"] + await websocket.close() + await asyncio.sleep(0) + assert "client" not in prompt_server.sockets_metadata + + websocket = await http_client.ws_connect("/ws?clientId=client") + await websocket.send_json( + {"type": "credential_auth", "data": {"credential_key": stale_credential_key}} + ) + await websocket.send_json( + {"type": "feature_flags", "data": {"supports_preview_metadata": True}} + ) + + status = await websocket.receive_json() + assert status["type"] == "status" + assert status["data"]["credential_key"] != stale_credential_key + server_features = await websocket.receive_json() + assert server_features["type"] == "feature_flags" + assert server_features["data"]["supports_preview_metadata"] is True + assert prompt_server.sockets_metadata["client"]["feature_flags"] == { + "supports_preview_metadata": True + } + await websocket.close() + + +def test_credentials_are_redacted_from_repr_and_request_logs(): + credential = CredentialRegistry() + credential.update("client", "top-secret") + credential.bind_prompt("prompt", "client") + assert "top-secret" not in repr(credential.get_for_prompt("prompt")) + assert request_logger._redact_headers( + {"Authorization": "Bearer top-secret", "X-API-KEY": "api-secret", "Other": "visible"} + ) == {"Authorization": "***", "X-API-KEY": "***", "Other": "visible"} + assert execution.format_input_data( + { + "auth_token_comfy_org": ["top-secret"], + "api_key_comfy_org": ["api-secret"], + "prompt": ["visible"], + } + ) == { + "auth_token_comfy_org": ["***"], + "api_key_comfy_org": ["***"], + "prompt": ["visible"], + } + + +def test_execution_exception_message_and_traceback_are_redacted(caplog): + extra_data = { + "auth_token_comfy_org": "top-secret", + "api_key_comfy_org": "api-secret", + } + try: + raise RuntimeError("failed with top-secret and api-secret") + except RuntimeError as ex: + _, _, tb = sys.exc_info() + message, traceback_lines = execution.log_execution_exception(ex, tb, extra_data) + + assert "top-secret" not in message + assert "api-secret" not in message + assert "top-secret" not in "".join(traceback_lines) + assert "api-secret" not in "".join(traceback_lines) + assert "top-secret" not in caplog.text + assert "api-secret" not in caplog.text + + +@pytest.mark.asyncio +async def test_poll_retries_once_only_when_generation_changes(aiohttp_client, monkeypatch): + requests = [] + + async def poll(request): + requests.append(request.headers.get("Authorization")) + if len(requests) == 1: + api_node_credentials.update("client", "new-token") + return web.json_response({"error": "expired"}, status=401) + return web.json_response({"status": "done"}) + + app = web.Application() + app.router.add_get("/poll", poll) + http_client = await aiohttp_client(app) + monkeypatch.setattr(api_client, "default_base_url", lambda: str(http_client.make_url("/"))) + monkeypatch.setattr(api_client.request_logger, "log_request_response", lambda **kwargs: None) + api_node_credentials.update("client", "old-token") + api_node_credentials.bind_prompt("prompt", "client") + + with CurrentNodeContext("prompt", "node"): + result = await api_client.poll_op_raw( + _Node, + api_client.ApiEndpoint("/poll"), + status_extractor=lambda response: response["status"], + completed_statuses=["done"], + poll_interval=0, + max_retries_per_poll=0, + ) + + assert result == {"status": "done"} + assert requests == ["Bearer old-token", "Bearer new-token"] + + +@pytest.mark.asyncio +async def test_auth_refresh_retry_does_not_consume_generic_retry_budget(aiohttp_client, monkeypatch): + request_count = 0 + + async def poll(request): + nonlocal request_count + request_count += 1 + if request_count == 1: + api_node_credentials.update("client", "new-token") + return web.json_response({"error": "expired"}, status=401) + if request_count == 2: + return web.json_response({"error": "temporary"}, status=502) + return web.json_response({"status": "done"}) + + app = web.Application() + app.router.add_get("/poll", poll) + http_client = await aiohttp_client(app) + monkeypatch.setattr(api_client, "default_base_url", lambda: str(http_client.make_url("/"))) + monkeypatch.setattr(api_client.request_logger, "log_request_response", lambda **kwargs: None) + api_node_credentials.update("client", "old-token") + api_node_credentials.bind_prompt("prompt", "client") + + with CurrentNodeContext("prompt", "node"): + result = await api_client.poll_op_raw( + _Node, + api_client.ApiEndpoint("/poll"), + status_extractor=lambda response: response["status"], + completed_statuses=["done"], + poll_interval=0, + max_retries_per_poll=1, + retry_delay_per_poll=0, + ) + + assert result == {"status": "done"} + assert request_count == 3 + + +@pytest.mark.asyncio +async def test_401_without_generation_change_is_not_retried(aiohttp_client, monkeypatch): + request_count = 0 + + async def poll(request): + nonlocal request_count + request_count += 1 + return web.json_response({"error": "expired"}, status=401) + + app = web.Application() + app.router.add_get("/poll", poll) + http_client = await aiohttp_client(app) + monkeypatch.setattr(api_client, "default_base_url", lambda: str(http_client.make_url("/"))) + monkeypatch.setattr(api_client.request_logger, "log_request_response", lambda **kwargs: None) + api_node_credentials.update("client", "old-token") + api_node_credentials.bind_prompt("prompt", "client") + + with CurrentNodeContext("prompt", "node"): + with pytest.raises(Exception, match="Unauthorized"): + await api_client.poll_op_raw( + _Node, + api_client.ApiEndpoint("/poll"), + status_extractor=lambda response: response["status"], + poll_interval=0, + max_retries_per_poll=0, + ) + + assert request_count == 1 + + +@pytest.mark.parametrize( + "headers", + [ + {"Authorization": "Bearer endpoint-token"}, + {"X-API-KEY": "endpoint-key"}, + ], +) +@pytest.mark.asyncio +async def test_endpoint_auth_401_does_not_retry_after_registry_change(headers, aiohttp_client, monkeypatch): + request_count = 0 + + async def poll(request): + nonlocal request_count + request_count += 1 + api_node_credentials.update("client", "new-token") + return web.json_response({"error": "expired"}, status=401) + + app = web.Application() + app.router.add_get("/poll", poll) + http_client = await aiohttp_client(app) + monkeypatch.setattr(api_client, "default_base_url", lambda: str(http_client.make_url("/"))) + monkeypatch.setattr(api_client.request_logger, "log_request_response", lambda **kwargs: None) + api_node_credentials.update("client", "old-token") + api_node_credentials.bind_prompt("prompt", "client") + + with CurrentNodeContext("prompt", "node"): + with pytest.raises(Exception, match="Unauthorized"): + await api_client.poll_op_raw( + _Node, + api_client.ApiEndpoint("/poll", headers=headers), + status_extractor=lambda response: response["status"], + poll_interval=0, + max_retries_per_poll=0, + ) + + assert request_count == 1 + + +@pytest.mark.asyncio +async def test_paid_post_is_not_retried_after_token_change(aiohttp_client, monkeypatch): + requests = [] + + async def generate(request): + requests.append(request.headers.get("Authorization")) + api_node_credentials.update("client", "new-token") + return web.json_response({"error": "expired"}, status=401) + + app = web.Application() + app.router.add_post("/generate", generate) + http_client = await aiohttp_client(app) + monkeypatch.setattr(api_client, "default_base_url", lambda: str(http_client.make_url("/"))) + monkeypatch.setattr(api_client.request_logger, "log_request_response", lambda **kwargs: None) + api_node_credentials.update("client", "old-token") + api_node_credentials.bind_prompt("prompt", "client") + + with CurrentNodeContext("prompt", "node"): + with pytest.raises(Exception, match="Unauthorized"): + await api_client.sync_op_raw( + _Node, + api_client.ApiEndpoint("/generate", method="POST"), + max_retries=0, + retry_401_on_auth_change=True, + ) + + assert requests == ["Bearer old-token"] diff --git a/tests-unit/feature_flags_test.py b/tests-unit/feature_flags_test.py index df16df6ab10..4b6940827ca 100644 --- a/tests-unit/feature_flags_test.py +++ b/tests-unit/feature_flags_test.py @@ -36,9 +36,19 @@ def test_get_server_features_contains_expected_flags(self): assert features["supports_preview_metadata"] is True assert "supports_model_type_tags" in features assert features["supports_model_type_tags"] is True + assert features["comfy_api_credentials"] == { + "version": 1, + "endpoint": "/api/credentials", + "websocket_auth_message": "credential_auth", + } assert "max_upload_size" in features assert isinstance(features["max_upload_size"], (int, float)) + def test_get_server_features_can_omit_credentials(self): + features = get_server_features(include_credentials=False) + assert "comfy_api_credentials" not in features + assert features["supports_preview_metadata"] is True + def test_get_connection_feature_with_missing_sid(self): """Test getting feature for non-existent session ID.""" sockets_metadata = {} From 54dd55c9e95c46c4b12f687c1b673487d38aef7b Mon Sep 17 00:00:00 2001 From: dante01yoon Date: Thu, 10 Sep 2026 21:52:32 +0000 Subject: [PATCH 2/2] Address credential lifecycle review findings Preserve websocket ownership state, revalidate sessions at mutation boundaries, and redact request credentials from all HTTP error and logging paths. --- comfy_api/credential_registry.py | 19 +- comfy_api_nodes/util/client.py | 73 ++++- comfy_api_nodes/util/common_exceptions.py | 6 + execution.py | 16 +- server.py | 27 +- .../credential_registry_test.py | 303 +++++++++++++++++- 6 files changed, 406 insertions(+), 38 deletions(-) diff --git a/comfy_api/credential_registry.py b/comfy_api/credential_registry.py index dbd6d8eeda7..04a45d8fb1f 100644 --- a/comfy_api/credential_registry.py +++ b/comfy_api/credential_registry.py @@ -18,7 +18,7 @@ class _ClientCredential: token: str | None = field(repr=False) generation: int updated_at: float - connected: bool = True + connected: bool = False @dataclass @@ -53,9 +53,13 @@ def update(self, client_id: str, token: str | None) -> int: generation = current.generation + 1 else: current.updated_at = now - current.connected = True return current.generation - self._clients[client_id] = _ClientCredential(token, generation, now) + if current is None: + self._clients[client_id] = _ClientCredential(token, generation, now) + else: + current.token = token + current.generation = generation + current.updated_at = now return generation def bind_prompt(self, prompt_id: str, client_id: str) -> bool: @@ -65,15 +69,16 @@ def bind_prompt(self, prompt_id: str, client_id: str) -> bool: if prompt_id in self._prompts: return False self._prompts[prompt_id] = _PromptClient(client_id, now) - current = self._clients.get(client_id) - if current is not None: - current.connected = True return True def connect(self, client_id: str) -> None: + now = self._clock() with self._lock: + self._cleanup(now) current = self._clients.get(client_id) - if current is not None: + if current is None: + self._clients[client_id] = _ClientCredential(None, 0, now, connected=True) + else: current.connected = True def get_for_prompt(self, prompt_id: str) -> Credential | None: diff --git a/comfy_api_nodes/util/client.py b/comfy_api_nodes/util/client.py index 7834b88a0fd..916d84fa7ab 100644 --- a/comfy_api_nodes/util/client.py +++ b/comfy_api_nodes/util/client.py @@ -30,7 +30,7 @@ is_processing_interrupted, sleep_with_interrupt, ) -from .common_exceptions import ApiServerError, LocalNetworkError, ProcessingInterrupted +from .common_exceptions import ApiResponseError, ApiServerError, LocalNetworkError, ProcessingInterrupted M = TypeVar("M", bound=BaseModel) @@ -432,9 +432,11 @@ async def _ticker(): return resp_json if status in failed_states: - msg = f"Task failed: {json.dumps(resp_json)}" + sensitive_values = getattr(resp_json, "sensitive_values", ()) + redacted_response = _redact_sensitive_data(resp_json, sensitive_values) + msg = f"Task failed: {json.dumps(redacted_response)}" logging.error(msg) - raise Exception(msg) + raise ApiResponseError(msg, sensitive_values) try: await sleep_with_interrupt(poll_interval, cls, None, None, None) @@ -464,6 +466,8 @@ async def _ticker(): raise except (LocalNetworkError, ApiServerError): raise + except ApiResponseError: + raise except Exception as e: raise Exception(f"Polling aborted due to error: {e}") from e finally: @@ -648,6 +652,46 @@ def _friendly_http_message(status: int, body: Any) -> str: return f"HTTP {status}: Unknown error" +def _request_auth_values(headers: dict[str, str]) -> tuple[str, ...]: + values = [] + normalized_headers = {name.lower(): value for name, value in headers.items()} + authorization = normalized_headers.get("authorization") + if authorization: + values.append(authorization) + _, _, bearer_token = authorization.partition(" ") + if bearer_token: + values.append(bearer_token) + api_key = normalized_headers.get("x-api-key") + if api_key: + values.append(api_key) + return tuple(dict.fromkeys(values)) + + +def _redact_sensitive_data(data: Any, sensitive_values: tuple[str, ...]) -> Any: + if isinstance(data, str): + for value in sensitive_values: + data = data.replace(value, "***") + return data + if isinstance(data, bytes): + for value in sensitive_values: + data = data.replace(value.encode(), b"***") + return data + if isinstance(data, dict): + return { + _redact_sensitive_data(key, sensitive_values): _redact_sensitive_data(value, sensitive_values) + for key, value in data.items() + } + if isinstance(data, list): + return [_redact_sensitive_data(value, sensitive_values) for value in data] + return data + + +class _AuthenticatedResponse(dict): + def __init__(self, data: dict[str, Any], sensitive_values: tuple[str, ...]): + super().__init__(data) + self.sensitive_values = sensitive_values + + def _generate_operation_id(method: str, path: str, attempt: int) -> str: slug = path.strip("/").replace("/", "_") or "op" return f"{method}_{slug}_try{attempt}_{uuid.uuid4().hex[:8]}" @@ -734,6 +778,7 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): endpoint_header_names = {name.lower() for name in cfg.endpoint.headers} if "authorization" in endpoint_header_names or "x-api-key" in endpoint_header_names: auth_generation = None + request_auth_values = _request_auth_values(payload_headers) payload_kw: dict[str, Any] = {"headers": payload_headers} if method == "GET": @@ -816,6 +861,7 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): body = await resp.json() except (ContentTypeError, json.JSONDecodeError): body = await resp.text() + redacted_body = _redact_sensitive_data(body, request_auth_values) should_retry = False wait_time = 0.0 retry_label = "" @@ -866,8 +912,8 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): request_method=method, request_url=url, response_status_code=resp.status, - response_headers=dict(resp.headers), - response_content=body, + response_headers=_redact_sensitive_data(dict(resp.headers), request_auth_values), + response_content=redacted_body, error_message=f"HTTP {resp.status} ({retry_label}, will retry in {wait_time:.1f}s)", ) await sleep_with_interrupt( @@ -879,17 +925,17 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): display_callback=_display_time_progress if cfg.monitor_progress else None, ) continue - msg = _friendly_http_message(resp.status, body) + msg = _friendly_http_message(resp.status, redacted_body) request_logger.log_request_response( operation_id=operation_id, request_method=method, request_url=url, response_status_code=resp.status, - response_headers=dict(resp.headers), - response_content=body, + response_headers=_redact_sensitive_data(dict(resp.headers), request_auth_values), + response_content=redacted_body, error_message=msg, ) - raise Exception(msg) + raise ApiResponseError(msg, request_auth_values) if expect_binary: buff = bytearray() @@ -921,8 +967,8 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): request_method=method, request_url=url, response_status_code=resp.status, - response_headers=resp_headers, - response_content=bytes_payload, + response_headers=_redact_sensitive_data(resp_headers, request_auth_values), + response_content=_redact_sensitive_data(bytes_payload, request_auth_values), ) return bytes_payload else: @@ -936,6 +982,7 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): except json.JSONDecodeError: payload = {"_raw": text} response_content_to_log = payload if isinstance(payload, dict) else text + response_content_to_log = _redact_sensitive_data(response_content_to_log, request_auth_values) if is_comfy_api_request: _maybe_remember_credits_used(cfg.node_cls, resp.headers.get(PRICE_CREDITS_HEADER)) with contextlib.suppress(Exception): @@ -947,9 +994,11 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float): request_method=method, request_url=url, response_status_code=resp.status, - response_headers=dict(resp.headers), + response_headers=_redact_sensitive_data(dict(resp.headers), request_auth_values), response_content=response_content_to_log, ) + if isinstance(payload, dict) and request_auth_values: + payload = _AuthenticatedResponse(payload, request_auth_values) return payload except ProcessingInterrupted: diff --git a/comfy_api_nodes/util/common_exceptions.py b/comfy_api_nodes/util/common_exceptions.py index 0606a440700..68c21621566 100644 --- a/comfy_api_nodes/util/common_exceptions.py +++ b/comfy_api_nodes/util/common_exceptions.py @@ -10,5 +10,11 @@ class ApiServerError(NetworkError): """Exception raised when the API server is unreachable but internet is working.""" +class ApiResponseError(Exception): + def __init__(self, message: str, sensitive_values: tuple[str, ...] = ()): + super().__init__(message) + self.sensitive_values = sensitive_values + + class ProcessingInterrupted(Exception): """Operation was interrupted by user/runtime via processing_interrupted().""" diff --git a/execution.py b/execution.py index cbd5e18dc55..98498e1dbc8 100644 --- a/execution.py +++ b/execution.py @@ -432,22 +432,30 @@ def format_input_data(input_data_all): return formatted -def redact_sensitive_text(text, extra_data): +def redact_sensitive_text(text, extra_data, additional_sensitive_values=()): sensitive_values = [ value for key in SENSITIVE_EXTRA_DATA_KEYS if isinstance((value := extra_data.get(key)), str) and value ] + sensitive_values.extend( + value for value in additional_sensitive_values if isinstance(value, str) and value + ) for value in sorted(sensitive_values, key=len, reverse=True): text = text.replace(value, "***") return text def log_execution_exception(ex, tb, extra_data): - exception_message = redact_sensitive_text(str(ex), extra_data) - traceback_lines = [redact_sensitive_text(line, extra_data) for line in traceback.format_tb(tb)] + request_sensitive_values = getattr(ex, "sensitive_values", ()) + exception_message = redact_sensitive_text(str(ex), extra_data, request_sensitive_values) + traceback_lines = [ + redact_sensitive_text(line, extra_data, request_sensitive_values) + for line in traceback.format_tb(tb) + ] formatted_exception = "".join( - redact_sensitive_text(line, extra_data) for line in traceback.format_exception(type(ex), ex, tb) + redact_sensitive_text(line, extra_data, request_sensitive_values) + for line in traceback.format_exception(type(ex), ex, tb) ) logging.error("!!! Exception during processing !!! %s", exception_message) logging.error(formatted_exception) diff --git a/server.py b/server.py index 44e17d832f5..e6103d88333 100644 --- a/server.py +++ b/server.py @@ -180,6 +180,16 @@ def _credential_transport_enabled(request: web.Request) -> bool: return False +def _credential_keys_match(presented: object, expected: object) -> bool: + return ( + isinstance(presented, str) + and isinstance(expected, str) + and presented.isascii() + and expected.isascii() + and hmac.compare_digest(presented, expected) + ) + + def _authenticated_client_id(request: web.Request, sockets_metadata: dict) -> str | None: client_id = request.headers.get(_CLIENT_ID_HEADER) credential_key = request.headers.get(_CREDENTIAL_KEY_HEADER) @@ -187,7 +197,9 @@ def _authenticated_client_id(request: web.Request, sockets_metadata: dict) -> st return None metadata = sockets_metadata.get(client_id) expected_key = metadata.get("credential_key") if metadata else None - if not expected_key or not hmac.compare_digest(credential_key, expected_key): + if not _credential_keys_match(credential_key, expected_key): + return None + if not api_node_credentials.is_protected(client_id): return None return client_id @@ -336,11 +348,7 @@ async def websocket_handler(request): else None ) expected_key = existing_metadata.get("credential_key") - if ( - not isinstance(presented_key, str) - or not isinstance(expected_key, str) - or not hmac.compare_digest(presented_key, expected_key) - ): + if not _credential_keys_match(presented_key, expected_key): await ws.close(code=aiohttp.WSCloseCode.POLICY_VIOLATION, message=b"Invalid client credentials") return ws @@ -873,6 +881,8 @@ async def update_api_credentials(request): if token is not None and (not isinstance(token, str) or not token or len(token) > 16384): return web.json_response({"error": "auth_token_comfy_org must be a non-empty string or null"}, status=400) + if _authenticated_client_id(request, self.sockets_metadata) != client_id: + return web.json_response({"error": "Invalid client credentials"}, status=403) generation = api_node_credentials.update(client_id, token) return web.json_response({"generation": generation}) @@ -1266,6 +1276,11 @@ async def post_prompt(request): extra_data["comfy_usage_source"] = usage_source if valid[0]: outputs_to_execute = valid[2] + if ( + authenticated_client is not None + and _authenticated_client_id(request, self.sockets_metadata) != authenticated_client + ): + return web.json_response({"error": "Invalid client credentials"}, status=403) sensitive = {} for sensitive_val in execution.SENSITIVE_EXTRA_DATA_KEYS: if sensitive_val in extra_data: diff --git a/tests-unit/comfy_api_nodes_test/credential_registry_test.py b/tests-unit/comfy_api_nodes_test/credential_registry_test.py index 4d445131b44..b1b3c43dd37 100644 --- a/tests-unit/comfy_api_nodes_test/credential_registry_test.py +++ b/tests-unit/comfy_api_nodes_test/credential_registry_test.py @@ -17,8 +17,9 @@ from comfy_api_nodes.util import _helpers from comfy_api_nodes.util import client as api_client from comfy_api_nodes.util import request_logger +from comfy_api_nodes.util.common_exceptions import ApiResponseError from comfy_execution.utils import CurrentNodeContext -from server import PromptServer, _authenticated_client_id, _credential_transport_enabled +from server import PromptServer, _authenticated_client_id, _credential_keys_match, _credential_transport_enabled class _Node: @@ -86,6 +87,43 @@ def test_registry_cleanup_on_completion_disconnect_and_ttl(): assert registry.get_for_prompt("stale-prompt") is None +def test_registry_placeholder_and_disconnected_ttl_semantics(): + now = [0.0] + registry = CredentialRegistry(credential_ttl=10, prompt_ttl=20, clock=lambda: now[0]) + + registry.connect("connected") + registry.bind_prompt("connected-prompt", "connected") + assert registry.get_for_prompt("connected-prompt").token is None + assert registry.get_for_prompt("connected-prompt").generation == 0 + registry.release_prompt("connected-prompt") + + now[0] = 11 + registry.bind_prompt("still-connected", "connected") + assert registry.get_for_prompt("still-connected").generation == 0 + + registry.update("detached", "secret") + assert registry.is_protected("detached") is True + now[0] = 22 + assert registry.is_protected("detached") is False + + +def test_active_prompt_preserves_disconnected_credential_until_prompt_ttl(): + now = [0.0] + registry = CredentialRegistry(credential_ttl=10, prompt_ttl=20, clock=lambda: now[0]) + registry.connect("client") + registry.update("client", "secret") + registry.bind_prompt("prompt", "client") + assert registry.disconnect("client") is True + + now[0] = 11 + assert registry.get_for_prompt("prompt").token == "secret" + now[0] = 30 + assert registry.get_for_prompt("prompt").token == "secret" + now[0] = 51 + assert registry.get_for_prompt("prompt") is None + assert registry.is_protected("client") is False + + def test_connected_registry_credentials_do_not_expire(): now = [0.0] registry = CredentialRegistry(credential_ttl=10, clock=lambda: now[0]) @@ -99,6 +137,20 @@ def test_connected_registry_credentials_do_not_expire(): assert registry.get_for_prompt("next-prompt").token == "secret" +def test_update_after_disconnect_does_not_restore_connected_state(): + registry = CredentialRegistry() + registry.connect("client") + registry.update("client", "old-token") + registry.bind_prompt("prompt", "client") + assert registry.disconnect("client") is True + + registry.update("client", "new-token") + assert registry.bind_prompt("other-prompt", "client") is True + registry.release_prompt("prompt") + assert registry.release_prompt("other-prompt") == "client" + assert registry.is_protected("client") is False + + def test_auth_header_uses_registry_then_falls_back_and_preserves_api_key(): with CurrentNodeContext("legacy-prompt", "node"): assert _helpers.get_auth_header(_Node) == {"Authorization": "Bearer snapshot-token"} @@ -135,12 +187,31 @@ def test_auth_header_uses_registry_then_falls_back_and_preserves_api_key(): def test_client_request_authentication_requires_matching_session_key(): request = SimpleNamespace(headers={"X-Comfy-Client-Id": "client", "X-Comfy-Credential-Key": "key"}) metadata = {"client": {"credential_key": "key"}} + assert _authenticated_client_id(request, metadata) is None + + api_node_credentials.connect("client") assert _authenticated_client_id(request, metadata) == "client" request.headers["X-Comfy-Credential-Key"] = "wrong" assert _authenticated_client_id(request, metadata) is None +@pytest.mark.parametrize( + ("presented", "expected"), + [ + ("é", "key"), + ("key", "é"), + (b"key", "key"), + ("key", b"key"), + (None, "key"), + ("key", None), + ], +) +def test_malformed_credential_keys_are_safely_rejected(presented, expected): + assert _credential_keys_match(presented, expected) is False + assert _credential_keys_match("key", "key") is True + + @pytest.mark.parametrize("peername", [("127.0.0.1", 8188), ("::1", 8188, 0, 0)]) def test_credential_transport_allows_direct_loopback(peername): request = SimpleNamespace(transport=_Transport(peername=peername), headers={}) @@ -167,6 +238,7 @@ def test_credential_transport_rejects_plaintext_remote_and_ignores_forwarded_pro async def test_credential_endpoint_updates_and_clears_without_returning_secret(aiohttp_client): prompt_server = PromptServer(None) prompt_server.sockets_metadata["client"] = {"credential_key": "key", "feature_flags": {}} + api_node_credentials.connect("client") route = next(route for route in prompt_server.routes if route.path == "/credentials") app = web.Application() app.router.add_post("/api/credentials", route.handler) @@ -199,6 +271,93 @@ async def test_credential_endpoint_updates_and_clears_without_returning_secret(a assert response.status == 403 +@pytest.mark.asyncio +async def test_credential_endpoint_rechecks_session_after_reading_body(aiohttp_client, monkeypatch): + calls = 0 + + def authenticate(*args): + nonlocal calls + calls += 1 + return "client" if calls == 1 else None + + monkeypatch.setattr("server._authenticated_client_id", authenticate) + prompt_server = PromptServer(None) + route = next(route for route in prompt_server.routes if route.path == "/credentials") + app = web.Application() + app.router.add_post("/api/credentials", route.handler) + http_client = await aiohttp_client(app) + + response = await http_client.post( + "/api/credentials", + headers={"X-Comfy-Client-Id": "client", "X-Comfy-Credential-Key": "old-key"}, + json={"auth_token_comfy_org": "stolen"}, + ) + + assert response.status == 403 + assert api_node_credentials.is_protected("client") is False + + +@pytest.mark.asyncio +async def test_websocket_placeholder_updates_and_survives_only_for_running_prompt(aiohttp_client): + prompt_server = PromptServer(None) + routes = {route.path: route for route in prompt_server.routes} + app = web.Application() + app.router.add_get("/ws", routes["/ws"].handler) + app.router.add_post("/api/credentials", routes["/credentials"].handler) + http_client = await aiohttp_client(app) + + websocket = await http_client.ws_connect("/ws?clientId=client") + status = await websocket.receive_json() + credential_key = status["data"]["credential_key"] + headers = {"X-Comfy-Client-Id": "client", "X-Comfy-Credential-Key": credential_key} + assert api_node_credentials.is_protected("client") is True + + response = await http_client.post( + "/api/credentials", + headers=headers, + json={"auth_token_comfy_org": "first-token"}, + ) + assert response.status == 200 + assert await response.json() == {"generation": 1} + + assert api_node_credentials.bind_prompt("running-prompt", "client") is True + await websocket.close() + await asyncio.sleep(0) + assert "client" in prompt_server.sockets_metadata + assert api_node_credentials.is_protected("client") is True + + response = await http_client.post( + "/api/credentials", + headers=headers, + json={"auth_token_comfy_org": "second-token"}, + ) + assert response.status == 200 + assert await response.json() == {"generation": 2} + assert api_node_credentials.get_for_prompt("running-prompt").token == "second-token" + + prompt_server.release_api_node_prompt("running-prompt") + assert api_node_credentials.is_protected("client") is False + assert "client" not in prompt_server.sockets_metadata + + +@pytest.mark.asyncio +async def test_websocket_placeholder_is_removed_on_disconnect_without_prompt(aiohttp_client): + prompt_server = PromptServer(None) + route = next(route for route in prompt_server.routes if route.path == "/ws") + app = web.Application() + app.router.add_get("/ws", route.handler) + http_client = await aiohttp_client(app) + + websocket = await http_client.ws_connect("/ws?clientId=client") + await websocket.receive_json() + assert api_node_credentials.is_protected("client") is True + + await websocket.close() + await asyncio.sleep(0) + assert api_node_credentials.is_protected("client") is False + assert "client" not in prompt_server.sockets_metadata + + @pytest.mark.asyncio async def test_remote_plaintext_rejects_credentials_but_allows_legacy_prompt( aiohttp_client, monkeypatch @@ -394,17 +553,18 @@ def test_execution_exception_message_and_traceback_are_redacted(caplog): "api_key_comfy_org": "api-secret", } try: - raise RuntimeError("failed with top-secret and api-secret") - except RuntimeError as ex: + raise ApiResponseError( + "failed with top-secret, api-secret, and refreshed-token", + ("refreshed-token",), + ) + except ApiResponseError as ex: _, _, tb = sys.exc_info() message, traceback_lines = execution.log_execution_exception(ex, tb, extra_data) - assert "top-secret" not in message - assert "api-secret" not in message - assert "top-secret" not in "".join(traceback_lines) - assert "api-secret" not in "".join(traceback_lines) - assert "top-secret" not in caplog.text - assert "api-secret" not in caplog.text + for secret in ("top-secret", "api-secret", "refreshed-token"): + assert secret not in message + assert secret not in "".join(traceback_lines) + assert secret not in caplog.text @pytest.mark.asyncio @@ -545,6 +705,131 @@ async def poll(request): assert request_count == 1 +@pytest.mark.parametrize("response_kind", ["json", "text"]) +@pytest.mark.parametrize( + ("endpoint_headers", "secret", "header_name"), + [ + ({}, "registry-token", "Authorization"), + ({"Authorization": "Bearer endpoint-token"}, "endpoint-token", "Authorization"), + ({"X-API-KEY": "endpoint-key"}, "endpoint-key", "X-API-KEY"), + ], +) +@pytest.mark.asyncio +async def test_api_error_redacts_request_auth_from_logs_exception_and_execution( + endpoint_headers, + secret, + header_name, + response_kind, + aiohttp_client, + monkeypatch, + tmp_path, + caplog, +): + async def fail(request): + echoed_auth = request.headers[header_name] + message = f"provider echoed {echoed_auth}" + if response_kind == "json": + return web.json_response( + {"error": "provider_error", "message": message}, + headers={"X-Echo": echoed_auth}, + status=400, + ) + return web.Response(text=message, headers={"X-Echo": echoed_auth}, status=400) + + app = web.Application() + app.router.add_get("/fail", fail) + http_client = await aiohttp_client(app) + monkeypatch.setattr(api_client, "default_base_url", lambda: str(http_client.make_url("/"))) + monkeypatch.setattr(request_logger, "get_log_directory", lambda: str(tmp_path)) + api_node_credentials.update("client", "registry-token") + api_node_credentials.bind_prompt("prompt", "client") + + with CurrentNodeContext("prompt", "node"): + with pytest.raises(ApiResponseError) as error: + await api_client.sync_op_raw( + _Node, + api_client.ApiEndpoint("/fail", headers=endpoint_headers), + max_retries=0, + ) + + message, traceback_lines = execution.log_execution_exception( + error.value, + error.value.__traceback__, + {}, + ) + request_logs = "".join(path.read_text() for path in tmp_path.iterdir()) + assert secret not in request_logs + assert secret not in str(error.value) + assert secret not in message + assert secret not in "".join(traceback_lines) + assert secret not in caplog.text + + +@pytest.mark.parametrize("as_binary", [False, True]) +@pytest.mark.asyncio +async def test_success_response_log_redacts_echoed_endpoint_auth( + as_binary, + aiohttp_client, + monkeypatch, + tmp_path, +): + secret = "endpoint-key" + + async def echo(request): + echoed = request.headers["X-API-KEY"] + if as_binary: + return web.Response( + body=f"echoed {echoed}".encode(), + headers={"X-Echo": echoed}, + content_type="application/octet-stream", + ) + return web.json_response({"echoed": echoed}, headers={"X-Echo": echoed}) + + app = web.Application() + app.router.add_get("/echo", echo) + http_client = await aiohttp_client(app) + monkeypatch.setattr(api_client, "default_base_url", lambda: str(http_client.make_url("/"))) + monkeypatch.setattr(request_logger, "get_log_directory", lambda: str(tmp_path)) + + response = await api_client.sync_op_raw( + _Node, + api_client.ApiEndpoint("/echo", headers={"X-API-KEY": secret}), + max_retries=0, + as_binary=as_binary, + ) + + assert secret in str(response) + request_logs = "".join(path.read_text() for path in tmp_path.iterdir()) + assert secret not in request_logs + + +@pytest.mark.asyncio +async def test_failed_poll_redacts_auth_echoed_in_success_response(aiohttp_client, monkeypatch, caplog): + async def poll(request): + token = request.headers["Authorization"].removeprefix("Bearer ") + return web.json_response({"status": "failed", "message": f"echoed {token}"}) + + app = web.Application() + app.router.add_get("/poll", poll) + http_client = await aiohttp_client(app) + monkeypatch.setattr(api_client, "default_base_url", lambda: str(http_client.make_url("/"))) + monkeypatch.setattr(api_client.request_logger, "log_request_response", lambda **kwargs: None) + api_node_credentials.update("client", "refreshed-token") + api_node_credentials.bind_prompt("prompt", "client") + + with CurrentNodeContext("prompt", "node"): + with pytest.raises(ApiResponseError) as error: + await api_client.poll_op_raw( + _Node, + api_client.ApiEndpoint("/poll"), + status_extractor=lambda response: response["status"], + poll_interval=0, + ) + + assert "refreshed-token" not in str(error.value) + assert "refreshed-token" not in caplog.text + + @pytest.mark.asyncio async def test_paid_post_is_not_retried_after_token_change(aiohttp_client, monkeypatch): requests = []