diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e68e6caf..39e5cb61b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ history. `asset_upload_refused`; new codes `asset_upload_invalid`, `asset_upload_failed` and `asset_upload_timeout` cover the other ways it can end. + - `comfy knowledge resolve` suggests canonical model ids for an incomplete family/version name such as `Qwen 2.1`. The command still fails with `knowledge_unknown_model` and exit 1, but `error.details.candidate_ids` lists @@ -60,9 +61,18 @@ history. ### Fixed -- `comfy assets library ls` reports a 401 or 403 as `cloud_unauthorized` even - when the server drops the connection partway through its error body, instead - of crashing with `IncompleteRead`. +- Parallel build and deploy requests rejected on an old access token now adopt + a peer's refreshed token under the shared refresh lock instead of rotating it + again. Each rejected request retries at most once. If session recovery still + fails while a refreshable sign-in remains stored, the watch reports that the + remote work may continue instead of telling you to sign in again. + `comfy build release create --watch`, `comfy deploy up`, `promote`, + `rollback` and `comfy deploy status --watch` now end with the new code + `watch_auth_lost` and exit code 75, as `deploy_watch_lost` does. Its hint + names the command that reads the release or deployment again. A 401 before + anything was accepted, on a workspace API key, or when no refreshable sign-in + remains still reports `build_not_signed_in` or `deploy_not_signed_in`. + - A typed `comfy model download` with no `--relative-path` now saves into the folder the running local ComfyUI scans for that model type, and says where. On Comfy Desktop that is the shared library, not the install's `models/`. @@ -112,6 +122,10 @@ history. value. Saves with a real marker have one more value than sockets and still go through the filter. +- `comfy assets library ls` reports a 401 or 403 as `cloud_unauthorized` even + when the server drops the connection partway through its error body, instead + of crashing with `IncompleteRead`. + ## [1.24.0] - 2026-10-29 [Full notes](https://github.com/Comfy-Org/comfy-cli/releases/tag/v1.24.0) · 25 commits since v1.23.0. diff --git a/comfy_cli/builder_api.py b/comfy_cli/builder_api.py index 6bad57a92..1d7cd0ea4 100644 --- a/comfy_cli/builder_api.py +++ b/comfy_cli/builder_api.py @@ -137,6 +137,7 @@ def __init__( # handed a token (COMFY_BUILDER_TOKEN, COMFY_CLOUD_AUTH_TOKEN) or a key keeps # it and lets a 401 surface. self._refreshes_on_401 = False + self._sign_in_base_url: str | None = None # What to replace once the builder refuses this client's credential. self._refused_hint = ( credentials.refused_key_hint(api_key) @@ -175,9 +176,15 @@ def from_session(cls, base_url: str) -> BuilderClient: raise BuilderAuthError("not signed in: run `comfy cloud login` or set COMFY_CLOUD_API_KEY") client = cls(base_url, session.access_token) client._refreshes_on_401 = True + client._sign_in_base_url = session.base_url client._refused_hint = "run `comfy cloud login` first" return client + @property + def sign_in_base_url(self) -> str | None: + """The environment of the stored sign-in this client's token came from; None for a key or a handed token.""" + return self._sign_in_base_url if self._refreshes_on_401 else None + def _send(self, url: str, **kwargs) -> tuple[int, dict | list | None]: """Every builder request goes through here, so a long wait survives its token expiring. diff --git a/comfy_cli/cloud/oauth.py b/comfy_cli/cloud/oauth.py index cb6029c62..68e4f7ed2 100644 --- a/comfy_cli/cloud/oauth.py +++ b/comfy_cli/cloud/oauth.py @@ -782,6 +782,7 @@ def ensure_fresh_session( force: bool = False, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ): """Return the stored cloud session, refreshing it first when the access token is expired (or within ``leeway_s`` of expiring) and a refresh token @@ -803,6 +804,10 @@ def ensure_fresh_session( ``expires_at`` recorded — still thinks it is valid. The refresh token is still required; without one this is a no-op. + ``rejected_access_token`` identifies the token that received the 401. If a + peer already replaced it, adopt that session instead of refreshing again. + This also covers a late 401 that arrives after the peer released the lock. + ``allow_clear=False`` keeps the stored session on disk even when a refresh hits a fatal token error (reuse-detection / ``invalid_grant``). Background, read-mostly callers (the detached ``comfy run`` watcher) pass this so a @@ -855,6 +860,7 @@ def ensure_fresh_session( force=force, allow_clear=allow_clear, expect=expect, + rejected_access_token=rejected_access_token, ) except TimeoutError: # A peer has held the lock longer than a refresh should take. Don't @@ -875,6 +881,7 @@ def _locked_refresh( force: bool, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ): """Decide → POST → persist, executed while holding the secrets lock. @@ -890,11 +897,25 @@ def _locked_refresh( if session is None or not session.refresh_token: return session + # The rejected request may have been in flight while a peer finished its + # refresh, before even our first store read. Don't rotate that peer's token + # again just because this client's older request was refused. + if ( + rejected_access_token is not None + and session.access_token != rejected_access_token + and session.expires_at is not None + and not session.is_expired(leeway_s=leeway_s) + ): + return session + # Double-check after acquiring the lock. A peer may have refreshed while we # blocked: if the stored refresh token rotated away from what we observed # before locking, the family already advanced — use it, never re-send the # consumed token (that is exactly what trips reuse-detection). - if session.refresh_token != observed_refresh_token: + # A usable reactive peer already returned above. Otherwise refresh its + # current family, never replaying the older token observed before locking. + # Ordinary proactive callers still coalesce on any peer rotation. + if session.refresh_token != observed_refresh_token and rejected_access_token is None: return session if not force and not session.is_expired(leeway_s=leeway_s): # Peer advanced the access token's expiry without us noticing a token diff --git a/comfy_cli/command/build.py b/comfy_cli/command/build.py index 695a3df9e..147ff225c 100644 --- a/comfy_cli/command/build.py +++ b/comfy_cli/command/build.py @@ -3139,6 +3139,40 @@ def _selected_release_id(renderer, client, scope: _BuildScope, release: str | No return _newest_release_id(renderer, client, _resolve_build_id(renderer, client, scope)) +#: The exit a watch that lost its sign-in ends with, as `comfy deploy`'s watches do +#: (``deploy_runtime.EXIT_WATCH_LOST``): only the watch ended, so it must not exit 1, +#: which says the release failed. 75 is EX_TEMPFAIL in sysexits.h. +_EXIT_WATCH_LOST: Final = 75 + + +def _watched_release(renderer, client, release_id: str) -> dict: + """One read of a release this command has already cut. + + The builder accepted the cut with this sign-in, so a 401 session recovery did + not clear ends the watch while a refreshable sign-in remains stored. The + release may continue remotely. A key, a handed token, or a sign-in without + a refresh token keeps ``build_not_signed_in``. + """ + from comfy_cli import credentials + from comfy_cli.builder_api import BuilderCredentialRefused + + try: + return client.get_release(release_id) + except BuilderCredentialRefused as error: + if not credentials.sign_in_still_stored(getattr(client, "sign_in_base_url", None)): + raise + renderer.error( + code="watch_auth_lost", + message=f"the builder refused the sign-in (401) after attempting session recovery; " + f"release {release_id} may still be building on the server", + hint=f"read it with `comfy build release show {release_id}` until its status is complete; " + "the stored sign-in was preserved", + details={"releaseId": release_id}, + exit_code=_EXIT_WATCH_LOST, + ) + raise typer.Exit(code=_EXIT_WATCH_LOST) from error + + def _poll_call(renderer, fn, usable=None): """``_builder_call`` for one poll of a watch, which a single timed-out or dropped request must not end: the release keeps building either way. A @@ -3196,7 +3230,7 @@ def _poll_status(renderer, client, release_id: str) -> dict: word and ends the watch at the caller.""" return _poll_call( renderer, - lambda: client.get_release(release_id), + lambda: _watched_release(renderer, client, release_id), usable=lambda release: isinstance(release, dict) and isinstance(release.get("status"), str), ) diff --git a/comfy_cli/command/deploy_runtime.py b/comfy_cli/command/deploy_runtime.py index 9f477cf51..13759d5b6 100644 --- a/comfy_cli/command/deploy_runtime.py +++ b/comfy_cli/command/deploy_runtime.py @@ -12,6 +12,7 @@ import typer from typing_extensions import assert_never +from comfy_cli import credentials from comfy_cli.builder_api import BuilderClient, BuilderCredentialRefused from comfy_cli.command.build import DEFAULT_BUILDER_URL from comfy_cli.command.build_paths import BuildSpecNotFoundError, resolve_build_paths @@ -121,6 +122,8 @@ def terminal_status_error(deployment_id: str, status: str) -> JsonObject: class DeployWatchLostError(Exception): """The deploy API left a watch's reads unanswered for the whole retry window.""" + code = "deploy_watch_lost" + def __init__(self, deployment_id: str, cause: Exception | None = None, *, message: str | None = None) -> None: super().__init__( message or f"the deploy API stopped answering while following deployment {deployment_id} ({cause})" @@ -128,6 +131,23 @@ def __init__(self, deployment_id: str, cause: Exception | None = None, *, messag self.deployment_id = deployment_id +class DeployWatchAuthLostError(DeployWatchLostError): + """The deploy API refused a watch's read while a refreshable sign-in remains stored. + + The client has already attempted shared session recovery and at most one + retry. The deployment may continue remotely, so it is the watch that ended. + """ + + code = "watch_auth_lost" + + def __init__(self, deployment_id: str) -> None: + super().__init__( + deployment_id, + message=f"the deploy API refused the sign-in (401) while following deployment {deployment_id}, " + "after attempting session recovery", + ) + + def _unanswered(error: Exception) -> bool: """A read the service never answered: a 5xx, or no response at all. @@ -139,12 +159,26 @@ def _unanswered(error: Exception) -> bool: return error.code == "deploy_server_error" and (error.status is None or error.status >= 500) +def _sign_in_refused(client: DeployUpClient, error: Exception) -> bool: + """A 401 on a watch's read, from a client whose refreshable sign-in remains. + + A key, a handed token, or a sign-in without a refresh token keeps the + sign-in code, because changing that credential is the fix. + """ + if not isinstance(error, DeployAPIError) or error.status != 401: + return False + return credentials.sign_in_still_stored(getattr(client, "sign_in_base_url", None)) + + def exit_watch_lost(renderer: Renderer, error: DeployWatchLostError, hint: str | None = None) -> NoReturn: """Say it is the watch that gave out, not the deployment, and how to pick it up again.""" + hint = hint or f"run `{reattach_hint(error.deployment_id)}` to watch it again" + if isinstance(error, DeployWatchAuthLostError): + hint = f"{hint}; the stored sign-in was preserved" renderer.error( - code="deploy_watch_lost", + code=error.code, message=f"{error}; the deployment may still be coming up", - hint=hint or f"run `{reattach_hint(error.deployment_id)}` to watch it again", + hint=hint, details={"deployment_id": error.deployment_id}, exit_code=EXIT_WATCH_LOST, ) @@ -172,7 +206,9 @@ def poll_deployment( The progress a watcher shows rides the same read the loop already makes, so watching costs the service nothing it was not already answering. A read the service does not answer is retried, and the first of each run of them goes to - ``on_unanswered``; past the window this raises ``DeployWatchLostError``. + ``on_unanswered``; past the window this raises ``DeployWatchLostError``. A + 401 session recovery did not clear, with a refreshable sign-in still stored, + raises ``DeployWatchAuthLostError`` at once. """ failures, since, started = 0, 0.0, clock() while True: @@ -186,6 +222,8 @@ def poll_deployment( # The client maps a timeout, but a connection dropped before the status # line or partway through a 200's body escapes it unmapped. except (DeployAPIError, ConnectionError, http.client.HTTPException) as error: + if _sign_in_refused(client, error): + raise DeployWatchAuthLostError(deployment_id) from error if not _unanswered(error): raise if failures == 0: diff --git a/comfy_cli/credentials.py b/comfy_cli/credentials.py index 9cf75efd4..bcba23d50 100644 --- a/comfy_cli/credentials.py +++ b/comfy_cli/credentials.py @@ -92,6 +92,7 @@ def get_session( force: bool = False, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ) -> CloudSession | None: """Read the stored Comfy Cloud OAuth session. @@ -104,6 +105,8 @@ def get_session( ``force=True`` (implies the refresh path) refreshes unconditionally, ignoring the local expiry check. Reserved for the *reactive* path: after a server 401, the token is known-rejected even if our clock disagrees. + ``rejected_access_token`` lets that path adopt a peer's newer access token + instead of rotating it again; the comparison happens under the refresh lock. ``allow_clear=False`` forwards to ``ensure_fresh_session`` so a fatal refresh failure does NOT clear the stored session. Background watchers pass @@ -117,9 +120,12 @@ def get_session( if refresh or force: from comfy_cli.cloud import oauth + options = {} if expect is not None: - return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear, expect=expect) - return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear) + options["expect"] = expect + if rejected_access_token is not None: + options["rejected_access_token"] = rejected_access_token + return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear, **options) from comfy_cli.auth import store as auth_store return auth_store.get_cloud_session() @@ -140,14 +146,52 @@ def refreshed_access_token( :func:`get_session`. """ if expect is not None: - session = get_session(refresh=True, force=True, allow_clear=allow_clear, expect=expect) + session = get_session( + refresh=True, force=True, allow_clear=allow_clear, expect=expect, rejected_access_token=rejected + ) else: - session = get_session(refresh=True, force=True, allow_clear=allow_clear) + session = get_session(refresh=True, force=True, allow_clear=allow_clear, rejected_access_token=rejected) if session is None or not session.access_token or session.access_token == rejected: return None return session.access_token +def sign_in_still_stored(base_url: str | None) -> bool: + """Whether a refreshable sign-in for the refused client's environment remains. + + A 401 that outlives :func:`refreshed_access_token` and one retry does not show + that nobody is signed in: a peer process can hold the refresh lock past this + one's timeout, a refresh that fails on the network keeps the old token, and + the server can refuse a token it minted a moment ago. A sign-in without a + refresh token cannot recover this way and retains the sign-in guidance. + ``base_url`` is the environment the client's sign-in came from; ``None``, + for a key or a handed token, never matches. + """ + if base_url is None: + return False + from comfy_cli import locking + from comfy_cli.auth import store as auth_store + + try: + # Do not spend another lock budget after session recovery timed out. + # The store's nested read reuses this thread's lock reentrantly. + with locking.file_lock(auth_store.lock_path(), timeout=0): + session = get_session(refresh=False) + except TimeoutError: + # Contention cannot prove the held sign-in disappeared. End only the + # watch, rather than hanging or prescribing another login. + return True + except OSError as error: + _log.warning("could not re-read the stored sign-in after a 401: %s", error) + return False + return ( + session is not None + and bool(session.access_token) + and bool(session.refresh_token) + and session.base_url == base_url + ) + + class DataPlaneToken: """The bearer one long command sends to a deployment endpoint, kept fresh. diff --git a/comfy_cli/error_codes.py b/comfy_cli/error_codes.py index ff11c8db7..2bac298dd 100644 --- a/comfy_cli/error_codes.py +++ b/comfy_cli/error_codes.py @@ -1751,6 +1751,19 @@ class ErrorCode: "after a move, read `comfy deploy show --deployment ` instead: the move has landed once it shows no " "pendingUpdate and its releaseId is the release asked for", ), + ErrorCode( + "watch_auth_lost", + "A watch over work the server had already accepted (`comfy build release create --watch`, `comfy deploy " + "up`, `promote` and `rollback`, `comfy deploy status --watch`) was refused with HTTP 401 despite " + "serialized session recovery and at most one retry, while a refreshable sign-in remains stored. " + "Only the watch ended: the release may still be building and the deployment may still be coming up. " + "The exit code is 75, not 1. `details.releaseId` (build) or " + "`details.deployment_id` (deploy) names what is still running. A 401 before anything was accepted, on a " + "workspace API key, or with no refreshable sign-in remaining stays `build_not_signed_in` or " + "`deploy_not_signed_in`.", + "the stored sign-in was preserved: re-read the work with `comfy build " + "release show ` or `comfy deploy status --deployment --watch`", + ), ErrorCode( "deploy_update_failed", "`comfy deploy up`, `promote` or `rollback` moved a deployment onto another release, and the watch saw " diff --git a/comfy_cli/skills/comfy-build/SKILL.md b/comfy_cli/skills/comfy-build/SKILL.md index 83a2e85ce..57758174a 100644 --- a/comfy_cli/skills/comfy-build/SKILL.md +++ b/comfy_cli/skills/comfy-build/SKILL.md @@ -76,7 +76,10 @@ blob ls (hidden; workspace private blobs that reaches the builder need it; a plain scan and a plain `validate` do not. A CI job sets a workspace API key in `COMFY_CLOUD_API_KEY` instead, and that key wins over a stored sign-in, so when the hint says that key was refused, - signing in cannot help: replace the key. + signing in cannot help: replace the key. `watch_auth_lost` (exit 75) means + `release create --watch` was refused after the cut despite session recovery, + while a refreshable sign-in remains stored. The release may still be building. + Read it with `comfy build release show ` before considering a new sign-in. On `build_not_enabled` the platform is in limited beta and this account is not enabled — stop and say so. diff --git a/comfy_cli/skills/comfy-deploy-failures/SKILL.md b/comfy_cli/skills/comfy-deploy-failures/SKILL.md index cb44d581b..f0f68c1dc 100644 --- a/comfy_cli/skills/comfy-deploy-failures/SKILL.md +++ b/comfy_cli/skills/comfy-deploy-failures/SKILL.md @@ -55,6 +55,7 @@ argument you pass. | `deploy_update_replaced` | A watched `up`, `promote` or `rollback` was replaced by a newer update; `details.replacing_release_id` names its release | Follow it with `comfy deploy status --deployment `, or end it with `comfy deploy cancel --deployment ` | | `deploy_updates_unavailable` | `promote`, `rollback`, `history`, `cancel` or `events --release` on a deployment with no `revision`, or `cancel` against a deploy service too old for it | `promote` or `rollback`: `up --create` on the release instead. `history`: read `events`. `cancel`: wait for the update to land or fail. `events`: drop `--release`; those events do not say which release made them | | `deploy_watch_lost` | The watch's reads went unanswered for a minute, or a move by `up`, `promote` or `rollback` did not land within an hour (exit 75); the deployment may still be coming up | `comfy deploy status --deployment --watch`, or after a move `comfy deploy show --deployment ` until it shows no `pendingUpdate`; do not redeploy | +| `watch_auth_lost` | A watch's read was refused with a 401 despite session recovery, and a refreshable sign-in remains stored (exit 75); the deployment may still be coming up | Reattach as for `deploy_watch_lost` before considering a new sign-in; do not redeploy | | `deploy_delete_needs_confirm` | `delete` without `--yes` non-interactively | Confirm with the user, then pass `--yes` | ### Submitting a workflow diff --git a/comfy_cli/skills/comfy-deploy/SKILL.md b/comfy_cli/skills/comfy-deploy/SKILL.md index 46893eceb..df9b5dbab 100644 --- a/comfy_cli/skills/comfy-deploy/SKILL.md +++ b/comfy_cli/skills/comfy-deploy/SKILL.md @@ -176,6 +176,10 @@ stops answering (a 5xx or a dropped connection, as during its own rollout) is retried for about a minute; if it is still down, the wait ends with `deploy_watch_lost` and **exit 75**, not 1. That is not a failed deployment: attach again with the command in the hint rather than redeploying. +`watch_auth_lost`, also exit 75, is a read refused with a 401 despite session +recovery, while a refreshable sign-in remains stored. Parallel requests adopt a +peer's refreshed token and retry at most once. Attach again the same way before +considering a new sign-in; the deployment may still be coming up. `status` also reports **why** a deployment stopped, as `stopReason`: `user`, `credits`, or `policy`. `credits` is a billing problem and not something a retry diff --git a/docs/json-output.md b/docs/json-output.md index 523082a5f..ef463790b 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -572,6 +572,14 @@ watch ends with the error code `deploy_watch_lost` and exit code **75**, never 1 only the watch ended, and the deployment may still be coming up. Its `hint` carries the command that re-attaches and `details.deployment_id` the deployment. +A 401 on a watch's read triggers serialized session recovery: the client adopts +a peer's newer token or refreshes the rejected one, then retries at most once. +If recovery fails while a refreshable sign-in for the same environment remains +stored, the watch ends with `watch_auth_lost`, again with exit code **75** and +the same `details`. Its `hint` reattaches the watch without a re-login prompt. +The remote work may continue. A 401 with no refreshable sign-in remaining, or +on a workspace API key, stays `deploy_not_signed_in`. + ## Success envelope On `--wait` success, `data` carries: diff --git a/tests/comfy_cli/auth/test_oauth.py b/tests/comfy_cli/auth/test_oauth.py index c7064763b..bfdb821fb 100644 --- a/tests/comfy_cli/auth/test_oauth.py +++ b/tests/comfy_cli/auth/test_oauth.py @@ -703,7 +703,8 @@ def worker(): # (b) Double-check: if a peer already rotated the token between our pre-lock # read and the post-lock re-read, we adopt it and do NOT refresh again. - def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, monkeypatch): + @pytest.mark.parametrize("peer_expiry", [None, 1, 4102444800], ids=["unknown-expiry", "expired", "known-unexpired"]) + def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, monkeypatch, peer_expiry): self._persist_expired(refresh_token="RT0") # Simulate a peer that rotated RT0 → RT1 while we waited for the lock: @@ -716,7 +717,7 @@ def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, m access_token="PEER_NEW", refresh_token="RT1", token_type="Bearer", - expires_at=int(time.time()) + 3600, + expires_at=peer_expiry, saved_at="2026-01-01T00:00:02+00:00", ) reads = [self._stale(), fresh_peer] @@ -727,6 +728,65 @@ def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, m assert result.refresh_token == "RT1" assert result.access_token == "PEER_NEW" + def test_a_rejected_older_token_does_not_adopt_an_already_expired_peer_token(self, persisted, monkeypatch): + from comfy_cli.credentials import refreshed_access_token + + self._persist_expired() + monkeypatch.setattr(oauth, "refresh_tokens", lambda **_: self._token_set(access="NEW", refresh="RT1")) + + assert refreshed_access_token("BEFORE_OLD") == "NEW" + assert auth_store.get_cloud_session().refresh_token == "RT1" + + @pytest.mark.parametrize( + ("peer_access", "peer_expiry"), + [("PEER_UNUSABLE", None), ("PEER_UNUSABLE", 1), ("REJECTED_OLDER", 4102444800)], + ids=["unknown-expiry", "expired", "same-rejected-token"], + ) + @pytest.mark.parametrize("peer_timing", ["already-stored", "after-first-read"]) + def test_a_peer_token_that_cannot_be_retried_uses_its_available_refresh( + self, persisted, monkeypatch, peer_timing, peer_access, peer_expiry + ): + from comfy_cli.credentials import refreshed_access_token + + def save(access, refresh, expiry): + return auth_store.save_cloud_session( + base_url="https://c", + resource="https://c/api", + client_id="cid", + scope="s", + access_token=access, + refresh_token=refresh, + token_type="Bearer", + expires_at=expiry, + ) + + save("REJECTED_OLDER", "RT0", 4102444800) + if peer_timing == "already-stored": + save(peer_access, "RT1", peer_expiry) + else: + real_read = auth_store.get_cloud_session + + def read_then_peer_rotates(): + session = real_read() + if session.refresh_token == "RT0": + save(peer_access, "RT1", peer_expiry) + return session + + monkeypatch.setattr(auth_store, "get_cloud_session", read_then_peer_rotates) + refreshes = [] + + def refresh(**kwargs): + refreshes.append(kwargs["refresh_token"]) + return self._token_set(access="NEW", refresh="RT2") + + monkeypatch.setattr(oauth, "refresh_tokens", refresh) + + assert refreshed_access_token("REJECTED_OLDER") == "NEW" + stored = auth_store.get_cloud_session() + assert stored.access_token == "NEW" + assert stored.refresh_token == "RT2" + assert refreshes == ["RT1"] + def _stale(self) -> auth_store.CloudSession: return auth_store.CloudSession( base_url="https://c", diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 718655009..68e9306e3 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -48,8 +48,22 @@ def reset_singleton(): reset_renderer_for_testing() +@pytest.fixture(scope="module") +def root_cli_ready(): + """Load the actual CLI before a test's target fake can bind into lazy modules.""" + from click import Context + from typer.main import get_command + + from comfy_cli.cmdline import app as root_app + + root = get_command(root_app) + with Context(root) as context: + for name in root.list_commands(context): + root.get_command(context, name) + + @pytest.fixture -def cloud_target(monkeypatch: pytest.MonkeyPatch): +def cloud_target(monkeypatch: pytest.MonkeyPatch, root_cli_ready): from comfy_cli.target import Target fake = Target( @@ -1148,7 +1162,7 @@ def signed_in(monkeypatch: pytest.MonkeyPatch): session=SimpleNamespace(access_token="old-token", base_url="https://cloud.example.com"), forced=0 ) - def get_session(*, refresh=True, force=False, allow_clear=True, expect=None): + def get_session(*, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: state.forced += 1 state.session = SimpleNamespace(access_token="new-token", base_url="https://cloud.example.com") @@ -1261,7 +1275,7 @@ def stored_login(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv(var, raising=False) state = SimpleNamespace(login=("a", 1), forced=0) - def get_session(*, refresh=True, force=False, allow_clear=True, expect=None): + def get_session(*, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: state.forced += 1 state.login = (state.login[0], state.login[1] + 1) diff --git a/tests/comfy_cli/command/test_deploy_progress.py b/tests/comfy_cli/command/test_deploy_progress.py index 5aaaf1927..051c62da3 100644 --- a/tests/comfy_cli/command/test_deploy_progress.py +++ b/tests/comfy_cli/command/test_deploy_progress.py @@ -683,7 +683,14 @@ class _TerminalRenderer: def __init__(self, width: int) -> None: from rich.console import Console - self._console = Console(file=io.StringIO(), width=width, force_terminal=True, color_system=None) + # Emulate the promised terminal even when the caller exports TERM=dumb. + self._console = Console( + file=io.StringIO(), + width=width, + force_terminal=True, + color_system=None, + _environ={"TERM": "xterm-256color"}, + ) def is_pretty(self) -> bool: return True diff --git a/tests/comfy_cli/command/test_deploy_run.py b/tests/comfy_cli/command/test_deploy_run.py index f2629f77f..c67768daf 100644 --- a/tests/comfy_cli/command/test_deploy_run.py +++ b/tests/comfy_cli/command/test_deploy_run.py @@ -613,7 +613,7 @@ class _RotatingSignIn: def __init__(self) -> None: self.token = "token-at-start" - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): return SimpleNamespace(access_token=self.token, base_url="https://cloud.comfy.org") @@ -673,7 +673,7 @@ def test_an_unreadable_sign_in_at_cancel_still_cancels_with_the_token_the_run_ho monkeypatch.setattr(deploy_run, "resolve_partner_credential", lambda: None) store = {"readable": True} - def get_session(*, refresh=True, force=False, allow_clear=True): + def get_session(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if not store["readable"]: raise OSError("auth store lock unavailable") return SimpleNamespace(access_token="cloud-secret", base_url="https://cloud.comfy.org") diff --git a/tests/comfy_cli/command/test_deploy_status.py b/tests/comfy_cli/command/test_deploy_status.py index 0076d559d..86b9d5a4d 100644 --- a/tests/comfy_cli/command/test_deploy_status.py +++ b/tests/comfy_cli/command/test_deploy_status.py @@ -4,6 +4,7 @@ import json from datetime import datetime, timedelta, timezone from pathlib import Path +from types import SimpleNamespace import jsonschema import pytest @@ -801,6 +802,34 @@ def get_deployment(self, deployment_id: str) -> JsonObject: assert len(client.get_calls) == 7 +def test_a_watch_refused_with_the_sign_in_still_stored_ends_as_watch_auth_lost(tmp_path, monkeypatch) -> None: + # Given a watch whose read is refused after the client's refresh and retry + class Refused(RecordingDeploy): + sign_in_base_url = "https://cloud.comfy.org" + + def get_deployment(self, deployment_id: str) -> JsonObject: + self.get_calls.append(deployment_id) + raise DeployAPIError("deploy_not_signed_in", "invalid auth token", status=401) + + client = Refused([_status_deployment(status="provisioning")]) + sleeps: list[float] = [] + _install_clients(monkeypatch, FakeBuilder(), client, sleeps) + stored = SimpleNamespace(access_token="token", refresh_token="rt", base_url="https://cloud.comfy.org") + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: stored) + + # When + result = _invoke_json(write_spec(tmp_path), "--watch") + + # Then + assert result.exit_code == 75, result.stderr + error = _json_envelope(result)["error"] + assert error["code"] == "watch_auth_lost" + assert "comfy deploy status --deployment dep-status --watch" in error["hint"] + assert "comfy cloud login" not in error["hint"] + assert client.get_calls == ["dep-status"] + assert sleeps == [] + + def test_a_watch_that_gives_up_takes_the_live_line_down_before_the_error_prints(tmp_path, monkeypatch) -> None: """An error printed while the live line is up lands under a spinner that is still turning.""" # Given a watch that draws its line, then never hears back. CliRunner's diff --git a/tests/comfy_cli/command/test_deploy_up.py b/tests/comfy_cli/command/test_deploy_up.py index d10e6927d..5df61ef8c 100644 --- a/tests/comfy_cli/command/test_deploy_up.py +++ b/tests/comfy_cli/command/test_deploy_up.py @@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace import jsonschema import pytest @@ -814,6 +814,93 @@ def test_a_4xx_on_the_follow_read_still_fails_at_once(tmp_path, monkeypatch) -> assert client.get_calls == 2 +_CLOUD = "https://cloud.comfy.org" + + +def _refused() -> DeployAPIError: + # What the client raises once a 401 outlives its forced refresh and its one retry. + return DeployAPIError( + "deploy_not_signed_in", + "invalid auth token", + status=401, + details={"operation": "get", "status": 401, "server_code": "UNAUTHORIZED"}, + ) + + +def _stored_sign_in(monkeypatch, base_url: str | None) -> None: + session = ( + SimpleNamespace(access_token="token", refresh_token="rt", base_url=base_url) if base_url is not None else None + ) + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: session) + + +def test_a_401_on_the_follow_read_with_the_sign_in_still_stored_ends_the_watch_not_the_deployment( + tmp_path, monkeypatch +) -> None: + """Parallel watches can each be refused after the shared refresh. The deployment + was accepted with that same sign-in and keeps coming up, so the envelope must + not send an agent to sign in, nor report a failure that did not happen.""" + # Given a deployment the service accepted, then a follow read it refuses + client = FlakyFollow([_refused()], get_statuses=["queued", "ready"]) + client.sign_in_base_url = _CLOUD # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, _CLOUD) + sleeps: list[float] = [] + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, sleeps) + + # Then + assert result.exit_code == 75, result.stderr + error = _json_envelope(result)["error"] + assert error["code"] == "watch_auth_lost" + assert "may still be coming up" in error["message"] + assert error["hint"].startswith("run `comfy deploy status --deployment dep-1 --watch` to watch it again") + assert "comfy cloud login" not in error["hint"] + assert error["details"] == {"deployment_id": "dep-1"} + assert sleeps == [] + assert len(client.create_keys) == 1 + + +@pytest.mark.parametrize( + ("client_sign_in", "stored_sign_in"), + [(_CLOUD, None), (_CLOUD, "https://staging.comfy.test"), (None, _CLOUD)], + ids=["refresh_signed_the_user_out", "signed_in_to_another_environment", "workspace_key"], +) +def test_a_401_on_the_follow_read_with_no_sign_in_to_keep_still_says_to_sign_in( + tmp_path, monkeypatch, client_sign_in, stored_sign_in +) -> None: + # Given a follow read refused, and no sign-in this watch's client could keep using + client = FlakyFollow([_refused()], get_statuses=["queued", "ready"]) + client.sign_in_base_url = client_sign_in # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, stored_sign_in) + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, []) + + # Then + assert result.exit_code == 1 + assert _json_envelope(result)["error"]["code"] == "deploy_not_signed_in" + + +class _RefusesTheCreate(FakeDeploy): + def create_deployment(self, release_id: str, compute_config: JsonObject, **kwargs) -> JsonObject: + raise _refused() + + +def test_a_401_on_the_create_keeps_the_sign_in_code(tmp_path, monkeypatch) -> None: + # Given a sign-in still stored, and a service that refuses the create itself + client = _RefusesTheCreate() + client.sign_in_base_url = _CLOUD # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, _CLOUD) + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, []) + + # Then nothing was accepted, so nothing runs on, and the refusal is the sign-in's + assert result.exit_code == 1 + assert _json_envelope(result)["error"]["code"] == "deploy_not_signed_in" + + def test_a_second_outage_in_one_watch_is_announced_and_backed_off_afresh(tmp_path, monkeypatch) -> None: """A read that is answered ends the outage, so the next one is new: it is announced again and gets the whole retry schedule, not what was left of the first.""" diff --git a/tests/comfy_cli/command/test_pretty_print_sanitize.py b/tests/comfy_cli/command/test_pretty_print_sanitize.py index ed8f70792..7e1b00407 100644 --- a/tests/comfy_cli/command/test_pretty_print_sanitize.py +++ b/tests/comfy_cli/command/test_pretty_print_sanitize.py @@ -708,7 +708,8 @@ def test_status_glyph_keeps_its_own_tags_live(pretty): from comfy_cli.output.glyphs import status_glyph - console = Console(file=pretty, force_terminal=True, width=80) + # This tests authored styling, independently of the caller's terminal type. + console = Console(file=pretty, force_terminal=True, width=80, _environ={"TERM": "xterm-256color"}) console.print(status_glyph("success")) out = pretty.getvalue() diff --git a/tests/comfy_cli/command/test_run.py b/tests/comfy_cli/command/test_run.py index 9731c0233..3af335823 100644 --- a/tests/comfy_cli/command/test_run.py +++ b/tests/comfy_cli/command/test_run.py @@ -2363,14 +2363,29 @@ def test_markup_in_node_name_does_not_crash_confirm(self, monkeypatch): class TestExecuteUiWorkflow: + @pytest.fixture(autouse=True) + def local_preflight_catalog(self, monkeypatch): + """Validate against the complete conversion fixture without a live server.""" + from copy import deepcopy + + monkeypatch.setattr("comfy_cli.command.run._fetch_object_info", lambda *a, **k: deepcopy(self.OBJECT_INFO)) + + def unexpected_catalog_request(*args, **kwargs): + pytest.fail("unexpected real catalog request; mock the specific object_info transport") + + monkeypatch.setattr("comfy_cli.http._PLAIN_OPENER.open", unexpected_catalog_request) + monkeypatch.setattr("comfy_cli.cql.engine._load_from_target", unexpected_catalog_request) + + # The conversion success fixture must also be an executable graph: + # PreviewImage consumes IMAGE, not an EmptyLatentImage's LATENT output. UI = { "nodes": [ { "id": 1, - "type": "EmptyLatentImage", + "type": "EmptyImage", "inputs": [], - "outputs": [{"name": "LATENT", "type": "LATENT", "links": [10]}], - "widgets_values": [512, 512, 1], + "outputs": [{"name": "IMAGE", "type": "IMAGE", "links": [10]}], + "widgets_values": [512, 512, 1, 0], "mode": 0, }, { @@ -2384,19 +2399,20 @@ class TestExecuteUiWorkflow: "links": [[10, 1, 0, 2, 0, "IMAGE"]], } OBJECT_INFO = { - "EmptyLatentImage": { + "EmptyImage": { "input": { "required": { "width": ["INT", {"default": 512}], "height": ["INT", {"default": 512}], "batch_size": ["INT", {"default": 1}], + "color": ["INT", {"default": 0}], } }, - "input_order": {"required": ["width", "height", "batch_size"]}, - "output": ["LATENT"], - "output_name": ["LATENT"], + "input_order": {"required": ["width", "height", "batch_size", "color"]}, + "output": ["IMAGE"], + "output_name": ["IMAGE"], "output_node": False, - "display_name": "Empty Latent Image", + "display_name": "Empty Image", }, "PreviewImage": { "input": {"required": {"images": ["IMAGE"]}}, @@ -2435,7 +2451,7 @@ def test_ui_workflow_is_converted_then_executed(self, ui_workflow_file): assert mock_fetch.call_args.args == ("127.0.0.1", 8188, 30) api_workflow = MockExec.call_args.args[0] assert set(api_workflow) == {"1", "2"} - assert api_workflow["1"]["class_type"] == "EmptyLatentImage" + assert api_workflow["1"]["class_type"] == "EmptyImage" assert api_workflow["2"]["inputs"]["images"] == ["1", 0] mock_exec.queue.assert_called_once() diff --git a/tests/comfy_cli/command/test_run_json.py b/tests/comfy_cli/command/test_run_json.py index a6ba3f478..eca122763 100644 --- a/tests/comfy_cli/command/test_run_json.py +++ b/tests/comfy_cli/command/test_run_json.py @@ -52,13 +52,30 @@ def ndjson_renderer(): reset_renderer_for_testing() +@pytest.fixture(autouse=True) +def local_preflight_catalog(monkeypatch): + """Keep event/queue tests independent of any real server on localhost. + + Validation still runs against complete schemas. Tests for conversion HTTP + and deliberately invalid catalogs override their specific boundary below. + """ + from copy import deepcopy + + monkeypatch.setattr("comfy_cli.command.run._fetch_object_info", lambda *a, **k: deepcopy(OBJECT_INFO)) + + def unexpected_catalog_request(*args, **kwargs): + pytest.fail("unexpected real catalog request; mock the specific object_info transport") + + monkeypatch.setattr("comfy_cli.http._PLAIN_OPENER.open", unexpected_catalog_request) + + @pytest.fixture def simple_workflow(): return { "1": { - "class_type": "EmptyLatentImage", - "inputs": {"width": 64, "height": 64, "batch_size": 1}, - "_meta": {"title": "Latent"}, + "class_type": "EmptyImage", + "inputs": {"width": 64, "height": 64, "batch_size": 1, "color": 0}, + "_meta": {"title": "Image"}, }, "2": { "class_type": "SaveImage", @@ -524,8 +541,8 @@ def test_queued_nodes_manifest_from_workflow(self, workflow_file, capsys, simple nodes = queued["nodes"] assert len(nodes) == 2 by_id = {n["node_id"]: n for n in nodes} - assert by_id["1"]["class_type"] == "EmptyLatentImage" - assert by_id["1"]["title"] == "Latent" # _meta.title wins + assert by_id["1"]["class_type"] == "EmptyImage" + assert by_id["1"]["title"] == "Image" # _meta.title wins assert by_id["2"]["class_type"] == "SaveImage" assert by_id["2"]["title"] == "Save" @@ -621,7 +638,7 @@ def test_execution_error(self, workflow_file, capsys): "data": { "prompt_id": "p", "node_id": "1", - "node_type": "EmptyLatentImage", + "node_type": "EmptyImage", "exception_type": "RuntimeError", "exception_message": "boom", "traceback": [' File "x.py"\n', " raise RuntimeError\n"], @@ -635,12 +652,12 @@ def test_execution_error(self, workflow_file, capsys): assert any(e["type"] == "execution_error" for e in _events(lines)) env = _envelope(lines) assert env["error"]["code"] == "execution_error" - assert env["error"]["message"] == "EmptyLatentImage (node 1): boom" + assert env["error"]["message"] == "EmptyImage (node 1): boom" details = env["error"]["details"] assert details["node_id"] == "1" - assert details["class_type"] == "EmptyLatentImage" + assert details["class_type"] == "EmptyImage" assert details["exception_type"] == "RuntimeError" - assert details["title"] == "Latent" # from _meta.title + assert details["title"] == "Image" # from _meta.title # The envelope carries only the traceback tail; the full traceback # stays on the execution_error event. assert isinstance(details["traceback_tail"], list) @@ -658,7 +675,7 @@ def test_execution_error_node_id_coerced_to_str(self, workflow_file, capsys): "data": { "prompt_id": "p", "node_id": 7, - "node_type": "EmptyLatentImage", + "node_type": "EmptyImage", "exception_type": "RuntimeError", "exception_message": "boom", "traceback": [], @@ -797,8 +814,8 @@ def test_node_ids_coerced_to_str(self, simple_workflow, capsys): assert isinstance(ev["node"], str), f"{ev['type']} node is {type(ev['node']).__name__}" def test_title_falls_back_to_class_type(self): - ex = _make_workflow_execution({"1": {"class_type": "EmptyLatentImage", "inputs": {}}}) - assert ex.get_node_title("1") == "EmptyLatentImage" + ex = _make_workflow_execution({"1": {"class_type": "EmptyImage", "inputs": {}}}) + assert ex.get_node_title("1") == "EmptyImage" def test_title_falls_back_to_node_id_for_unknown(self): ex = _make_workflow_execution({}) @@ -809,10 +826,10 @@ def test_title_falls_back_to_node_id_for_unknown(self): "nodes": [ { "id": 1, - "type": "EmptyLatentImage", + "type": "EmptyImage", "inputs": [], - "outputs": [{"name": "LATENT", "type": "LATENT", "links": [10]}], - "widgets_values": [64, 64, 1], + "outputs": [{"name": "IMAGE", "type": "IMAGE", "links": [10]}], + "widgets_values": [64, 64, 1, 0], "mode": 0, }, { @@ -827,17 +844,26 @@ def test_title_falls_back_to_node_id_for_unknown(self): } OBJECT_INFO = { - "EmptyLatentImage": { + "EmptyImage": { "input": { "required": { "width": ["INT", {"default": 512}], "height": ["INT", {"default": 512}], "batch_size": ["INT", {"default": 1}], + "color": ["INT", {"default": 0}], } }, - "input_order": {"required": ["width", "height", "batch_size"]}, + "input_order": {"required": ["width", "height", "batch_size", "color"]}, + "output": ["IMAGE"], + "output_name": ["IMAGE"], "output_node": False, - "display_name": "Empty Latent Image", + "display_name": "Empty Image", + }, + "SaveImage": { + "input": {"required": {"filename_prefix": ["STRING", {"default": "ComfyUI"}], "images": ["IMAGE"]}}, + "input_order": {"required": ["filename_prefix", "images"]}, + "output": [], + "output_node": True, }, "PreviewImage": { "input": {"required": {"images": ["IMAGE"]}}, @@ -970,7 +996,7 @@ def test_api_input_emits_prompt_preview_before_queued(self, workflow_file, capsy assert types[0] == "prompt_preview" assert "queued" in types assert types.index("prompt_preview") < types.index("queued") - assert _events(lines)[0]["prompt"]["1"]["class_type"] == "EmptyLatentImage" + assert _events(lines)[0]["prompt"]["1"]["class_type"] == "EmptyImage" def test_ui_input_emits_converted_then_prompt_preview_then_queued(self, ui_workflow_file, capsys): with ( @@ -1035,7 +1061,7 @@ def test_api_input_emits_prompt_preview_and_envelope_only(self, workflow_file, c assert [e["type"] for e in _events(lines)] == ["prompt_preview"] preview = _events(lines)[0] assert preview["schema"] == "event/1" - assert preview["prompt"]["1"]["class_type"] == "EmptyLatentImage" + assert preview["prompt"]["1"]["class_type"] == "EmptyImage" env = _envelope(lines) assert env["ok"] is True assert env["data"]["status"] == "preview" @@ -1092,7 +1118,7 @@ def test_print_prompt_text_mode_pretty_prints_json(self, workflow_file, capsys): out, _err = capsys.readouterr() parsed = json.loads(out) assert "1" in parsed - assert parsed["1"]["class_type"] == "EmptyLatentImage" + assert parsed["1"]["class_type"] == "EmptyImage" def test_print_prompt_does_not_post_when_workflow_invalid(self, capsys): # Pre-flight failures (workflow_not_found, workflow_not_api_format) @@ -1240,10 +1266,10 @@ def test_execution_cached_event_shape(self, workflow_file, capsys): cached_events = [e for e in _events(lines) if e["type"] == "execution_cached"] assert len(cached_events) == 2 - # Node 1 has _meta.title="Latent"; class_type=EmptyLatentImage + # Node 1 has _meta.title="Image"; class_type=EmptyImage n1 = next(e for e in cached_events if e["node"] == "1") - assert n1["class_type"] == "EmptyLatentImage" - assert n1["title"] == "Latent" + assert n1["class_type"] == "EmptyImage" + assert n1["title"] == "Image" # Node 2 has _meta.title="Save"; class_type=SaveImage n2 = next(e for e in cached_events if e["node"] == "2") assert n2["class_type"] == "SaveImage" @@ -1380,9 +1406,9 @@ class TestErrorPathCoverage: def _make_workflow(self): return { "1": { - "class_type": "EmptyLatentImage", - "inputs": {"width": 64, "height": 64, "batch_size": 1}, - "_meta": {"title": "Latent"}, + "class_type": "EmptyImage", + "inputs": {"width": 64, "height": 64, "batch_size": 1, "color": 0}, + "_meta": {"title": "Image"}, }, "2": { "class_type": "SaveImage", @@ -1399,10 +1425,10 @@ def test_object_info_timeout_routes_to_connection_error(self, capsys): "nodes": [ { "id": 1, - "type": "EmptyLatentImage", + "type": "EmptyImage", "inputs": [], "outputs": [], - "widgets_values": [64, 64, 1], + "widgets_values": [64, 64, 1, 0], "mode": 0, } ], @@ -1414,7 +1440,7 @@ def test_object_info_timeout_routes_to_connection_error(self, capsys): try: with ( patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), - patch("comfy_cli.http._AUTHED_OPENER.open", side_effect=TimeoutError("timed out")), + patch("comfy_cli.http._PLAIN_OPENER.open", side_effect=TimeoutError("timed out")), ): lines, exit_code = _run_execute_capture(path, capsys) assert _envelope(lines)["error"]["code"] == "connection_error" @@ -1427,10 +1453,10 @@ def test_object_info_non_json_body_routes_to_object_info_unavailable(self, capsy "nodes": [ { "id": 1, - "type": "EmptyLatentImage", + "type": "EmptyImage", "inputs": [], "outputs": [], - "widgets_values": [64, 64, 1], + "widgets_values": [64, 64, 1, 0], "mode": 0, } ], @@ -1876,7 +1902,7 @@ def test_api_workflow_async_stream_matches_documented_archetype( assert isinstance(queued["client_id"], str) and queued["client_id"] assert queued["validation_warnings"] == [] assert queued["nodes"] == [ - {"node_id": "1", "class_type": "EmptyLatentImage", "title": "Latent"}, + {"node_id": "1", "class_type": "EmptyImage", "title": "Image"}, {"node_id": "2", "class_type": "SaveImage", "title": "Save"}, ] assert queued["base_url"] == "https://api.comfy.org" diff --git a/tests/comfy_cli/command/test_run_prompt.py b/tests/comfy_cli/command/test_run_prompt.py index c67139558..da6f3c557 100644 --- a/tests/comfy_cli/command/test_run_prompt.py +++ b/tests/comfy_cli/command/test_run_prompt.py @@ -9,6 +9,8 @@ from __future__ import annotations +import json +from pathlib import Path from unittest.mock import MagicMock, patch import pytest @@ -32,11 +34,15 @@ class TestExecuteSubmitsPreloadedGraph: """`preloaded` short-circuits file loading; the in-memory graph is what reaches the submit path (WorkflowExecution).""" + @pytest.fixture(autouse=True) + def preloaded_catalog(self): + """Validate the injected graph against a frozen catalog, without a server.""" + catalog = json.loads((Path(__file__).resolve().parents[1] / "fixtures/sd15_object_info.json").read_text()) + with patch("comfy_cli.command.run._fetch_object_info", return_value=catalog): + yield + def test_preloaded_graph_is_submitted(self): - injected = { - "6": {"class_type": "CLIPTextEncode", "inputs": {"text": "a red fox in snow"}}, - "9": {"class_type": "SaveImage", "inputs": {"images": ["8", 0]}}, - } + injected = build_default_workflow(prompt="a red fox in snow") with ( patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), patch("comfy_cli.command.run.ExecutionProgress"), @@ -63,7 +69,7 @@ def test_preloaded_graph_is_submitted(self): def test_preloaded_skips_file_loading(self): """A non-existent path never triggers workflow_not_found when preloaded.""" - injected = {"9": {"class_type": "SaveImage", "inputs": {}}} + injected = build_default_workflow(prompt="fox") with ( patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), patch("comfy_cli.command.run.ExecutionProgress"), diff --git a/tests/comfy_cli/command/test_transfer_download.py b/tests/comfy_cli/command/test_transfer_download.py index 889515bf5..0dc887921 100644 --- a/tests/comfy_cli/command/test_transfer_download.py +++ b/tests/comfy_cli/command/test_transfer_download.py @@ -1428,7 +1428,7 @@ def __init__(self, base_url: str, first: str) -> None: self.token = first self.refreshing_reads: list[bool] = [] - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): from types import SimpleNamespace if refresh: diff --git a/tests/comfy_cli/output/test_error_envelope_where.py b/tests/comfy_cli/output/test_error_envelope_where.py index e84ae3a06..ed0ae837f 100644 --- a/tests/comfy_cli/output/test_error_envelope_where.py +++ b/tests/comfy_cli/output/test_error_envelope_where.py @@ -324,15 +324,19 @@ def _workflow(tmp_path): def test_cloud(self, tmp_path): err = LoadError("no object_info", details={"hint": "sign in"}) - with patch("comfy_cli.cql.engine.Graph.load", side_effect=err): + with patch("comfy_cli.cql.loader.resilient_load_object_info", side_effect=err) as load: env = _run(["--json", "validate", "--workflow", self._workflow(tmp_path), "--where", "cloud"]) + load.assert_called_once() + assert load.call_args.kwargs["mode"] == "cloud" assert env["error"]["code"] == "cql_no_graph" assert env["where"] == "cloud" def test_local(self, tmp_path): err = LoadError("no object_info", details={"hint": "comfy launch"}) - with patch("comfy_cli.cql.engine.Graph.load", side_effect=err): + with patch("comfy_cli.cql.loader.resilient_load_object_info", side_effect=err) as load: env = _run(["--json", "validate", "--workflow", self._workflow(tmp_path), "--where", "local"]) + load.assert_called_once() + assert load.call_args.kwargs["mode"] == "local" assert env["error"]["code"] == "cql_no_graph" assert env["where"] == "local" diff --git a/tests/comfy_cli/test_data_plane_token.py b/tests/comfy_cli/test_data_plane_token.py index 9f138686c..82323c836 100644 --- a/tests/comfy_cli/test_data_plane_token.py +++ b/tests/comfy_cli/test_data_plane_token.py @@ -27,7 +27,7 @@ def __init__(self, proactive: str, forced: str | None) -> None: self.forced = forced self.forced_calls = 0 - def __call__(self, *, refresh=True, force=False, allow_clear=True, expect=None): + def __call__(self, *, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: self.forced_calls += 1 if not self.forced: @@ -181,7 +181,7 @@ def test_a_sign_in_to_another_environment_mid_run_is_never_sent_or_refreshed(mon refreshing: list[bool] = [] - def get_session(*, refresh=True, force=False, allow_clear=True): + def get_session(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if refresh or force: refreshing.append(allow_clear) return stored @@ -382,3 +382,29 @@ def get_session(**kwargs): # Then: the refresh is asked exactly as before, with no expectation attached assert calls and all("expect" not in kwargs for kwargs in calls) + + +def test_an_account_bound_holder_adopts_the_same_accounts_peer_refresh(login_race) -> None: + # A peer finishes its refresh while the held token's rejected request is in flight. + held = _save_login("a", expires_at=_LATER).access_token + token = DataPlaneToken(held, refreshes=True, base_url=_CLOUD, same_account=True) + peer = _jwt("a", 2) + from comfy_cli.auth import store + + stored = store.get_cloud_session() + store.save_cloud_session( + base_url=stored.base_url, + resource=stored.resource, + client_id=stored.client_id, + scope=stored.scope, + access_token=peer, + refresh_token="refresh-a-peer", + token_type="Bearer", + expires_at=_LATER, + ) + + assert token.after_rejection() is True + assert token.current() == peer + assert login_race.refreshed == [] + assert store.get_cloud_session().refresh_token == "refresh-a-peer" + assert token.account_replaced is False diff --git a/tests/comfy_cli/test_http.py b/tests/comfy_cli/test_http.py index 703199d85..20adb7027 100644 --- a/tests/comfy_cli/test_http.py +++ b/tests/comfy_cli/test_http.py @@ -819,7 +819,6 @@ def test_an_unloadable_supplement_falls_through_to_the_platform_roots(tmp_path, monkeypatch.setitem(sys.modules, "certifi", types.SimpleNamespace(where=lambda: str(broken))) real = ssl.create_default_context - populated = len(pathlib.Path(certifi.where()).read_text().split("-----BEGIN CERTIFICATE-----")) - 1 def platform_with_roots(*args, **kwargs): context = real(*args, **kwargs) @@ -827,13 +826,15 @@ def platform_with_roots(*args, **kwargs): context.load_verify_locations(cafile=certifi.where()) return context + expected_roots = set(platform_with_roots().get_ca_certs(binary_form=True)) + assert expected_roots, "the platform fixture must contain trust roots" monkeypatch.setattr(ssl, "create_default_context", platform_with_roots) # When context = http_mod.ssl_context() # Then - assert context.cert_store_stats()["x509_ca"] == populated, ( + assert set(context.get_ca_certs(binary_form=True)) == expected_roots, ( "the platform roots must survive a supplement that would not load" ) diff --git a/tests/comfy_cli/test_platform_api_key.py b/tests/comfy_cli/test_platform_api_key.py index 64a14f17d..3c6c2b1d8 100644 --- a/tests/comfy_cli/test_platform_api_key.py +++ b/tests/comfy_cli/test_platform_api_key.py @@ -41,7 +41,7 @@ def __call__(self, url, target, **kwargs): def _install(monkeypatch, server: _Server, key: Credential | None, sign_in: str | None = "signed-in-token") -> list: refreshes: list = [] - def sessions(*, refresh=True, force=False, allow_clear=True): + def sessions(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if force: refreshes.append(force) return SimpleNamespace(access_token=sign_in, base_url="https://cloud.comfy.org") if sign_in else None @@ -187,7 +187,7 @@ def test_a_refreshed_sign_in_the_builder_still_refuses_names_the_login(monkeypat server = _Server(status=401) refreshed = iter(["refreshed-token"]) - def sessions(*, refresh=True, force=False, allow_clear=True): + def sessions(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): return SimpleNamespace( access_token=next(refreshed) if force else "signed-in-token", base_url="https://cloud.comfy.org" ) diff --git a/tests/comfy_cli/test_session_refresh.py b/tests/comfy_cli/test_session_refresh.py index a0db03ba4..027d285bd 100644 --- a/tests/comfy_cli/test_session_refresh.py +++ b/tests/comfy_cli/test_session_refresh.py @@ -10,12 +10,16 @@ import http.client import io +import os +import threading +import time import urllib.error from types import SimpleNamespace import pytest from comfy_cli.builder_api import BuilderClient +from comfy_cli.credentials import Credential from comfy_cli.deploy_api import DeployAPIError, DeployClient _DEPLOY_URL = "https://deploy.test/deploy" @@ -38,18 +42,27 @@ def __call__(self, url, target, **kwargs): class _Sessions: - """Stands in for the stored sign-in: hands out ``first`` until forced, then ``after``.""" + """Stands in for the stored sign-in: hands out ``first`` until forced, then ``after``. + + A forced refresh that yields no token clears the store, as a fatal refresh does. + """ def __init__(self, first: str, after: str | None) -> None: self.first = first self.after = after self.forced = 0 + self.cleared = False - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if force: self.forced += 1 - return SimpleNamespace(access_token=self.after, base_url="https://cloud.comfy.org") if self.after else None - return SimpleNamespace(access_token=self.first, base_url="https://cloud.comfy.org") + if self.after is None: + self.cleared = allow_clear + return None + return SimpleNamespace(access_token=self.after, refresh_token="rt", base_url="https://cloud.comfy.org") + if self.cleared: + return None + return SimpleNamespace(access_token=self.first, refresh_token="rt", base_url="https://cloud.comfy.org") def _install(monkeypatch, server: _Server, sessions: _Sessions) -> None: @@ -75,6 +88,63 @@ def _install(monkeypatch, server: _Server, sessions: _Sessions) -> None: } +@pytest.mark.parametrize("lagging", ["builder", "deploy"]) +def test_overlapping_clients_reuse_the_peer_refresh_before_retrying(monkeypatch, tmp_path, lagging) -> None: + """A late 401 for OLD must not rotate the NEW token a peer just obtained.""" + from comfy_cli.auth import store + from comfy_cli.cloud import oauth + + monkeypatch.setattr(store, "secrets_path", lambda: tmp_path / "secrets.json") + store.save_cloud_session( + base_url="https://cloud.comfy.org", + resource="https://cloud.comfy.org/api", + client_id="cli", + scope="s", + access_token="OLD", + refresh_token="RT0", + token_type="Bearer", + expires_at=int(time.time()) + 3600, + ) + refreshes = [] + + def refresh(**kwargs): + refreshes.append(kwargs["refresh_token"]) + generation = len(refreshes) + return oauth.TokenSet( + access_token=f"NEW{generation}", + refresh_token=f"RT{generation}", + token_type="Bearer", + expires_in=3600, + expires_at=int(time.time()) + 3600, + scope="s", + ) + + monkeypatch.setattr(oauth, "refresh_tokens", refresh) + clients = {"builder": BuilderClient.from_session(_BUILDER_URL), "deploy": DeployClient.from_session(_DEPLOY_URL)} + peer = "deploy" if lagging == "builder" else "builder" + calls = { + "builder": lambda: clients["builder"].get_release("r1"), + "deploy": lambda: clients["deploy"].get_deployment("dep-1"), + } + requests = [] + + def send(url, target, **kwargs): + name = "builder" if url.startswith(_BUILDER_URL) else "deploy" + requests.append((name, target.auth_token)) + if target.auth_token == "OLD": + if name == lagging: + calls[peer]() # Peer finishes its refresh while this stale request is in flight. + raise urllib.error.HTTPError(url, 401, "expired", http.client.HTTPMessage(), io.BytesIO(b"{}")) + return 200, dict(_ANSWER) + + monkeypatch.setattr("comfy_cli.builder_api.request_json", send) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", send) + + assert calls[lagging]()["status"] == "ready" + assert refreshes == ["RT0"] + assert requests == [(lagging, "OLD"), (peer, "OLD"), (peer, "NEW1"), (lagging, "NEW1")] + + @pytest.mark.parametrize("call", _DEPLOY_CALLS.values(), ids=_DEPLOY_CALLS.keys()) def test_deploy_client_retries_once_with_a_refreshed_token_after_a_401(monkeypatch, call) -> None: # Given @@ -260,3 +330,348 @@ def test_a_refused_injected_token_says_to_replace_it_not_to_sign_in(monkeypatch) assert "COMFY_BUILDER_TOKEN" in envelope["error"]["hint"] assert "comfy cloud login" not in envelope["error"]["hint"] assert server.tokens == ["injected"] + + +# --- A 401 that outlives the refresh during a watch ----------------------------- +# +# Several commands polling at once can each be refused after the shared refresh: +# a peer can hold the refresh lock past this process's timeout, and then this one +# retries nothing. The build or deployment was accepted with the same sign-in a +# moment earlier and runs on, so while that sign-in is still stored, signing in +# again is not the fix. A 401 before anything was submitted keeps its sign-in code. + + +class _AnswersTheCut(_Server): + """Answers the cut whatever token it carries, then refuses every read the watch makes.""" + + def __call__(self, url, target, **kwargs): + if kwargs.get("method") == "POST": + self.tokens.append(target.auth_token) + return 200, dict(_ANSWER) + return super().__call__(url, target, **kwargs) + + +def _watched_cut(monkeypatch, server: _Server, sessions: _Sessions): + import json + + from typer.testing import CliRunner + + from comfy_cli.cmdline import app + + _install(monkeypatch, server, sessions) + monkeypatch.delenv("COMFY_BUILDER_TOKEN", raising=False) + monkeypatch.delenv("COMFY_CLOUD_API_KEY", raising=False) + result = CliRunner().invoke( + app, + ["--json", "build", "release", "create", "--id", "b1", "--target", "linux/nvidia", "--watch"], + env={"COMFY_BUILDER_URL": _BUILDER_URL}, + ) + envelope = json.loads([line for line in result.stdout.splitlines() if line.strip()][-1]) + return result, envelope["error"] + + +def test_a_release_watch_refused_after_the_refresh_says_the_release_keeps_building(monkeypatch) -> None: + # Given a cut the builder accepts, then a watch it refuses even after the refresh + server = _AnswersTheCut(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then only the watch ended: its own code and exit status, and no sign-in prompt + assert result.exit_code == 75 + assert error["code"] == "watch_auth_lost" + assert "may still be building" in error["message"] + assert "comfy build release show r1" in error["hint"] + assert "comfy cloud login" not in error["hint"] + assert error["details"] == {"releaseId": "r1"} + assert server.tokens == ["signed-in", "signed-in", "refreshed"] + assert sessions.forced == 1 + + +@pytest.mark.parametrize("misses", [1, 4]) +@pytest.mark.parametrize("refreshed", ["refreshed", None], ids=["sign-in-retained", "signed-out"]) +@pytest.mark.parametrize("later_poll", [False, True], ids=["initial-poll", "later-poll"]) +def test_a_release_watch_transport_retry_preserves_the_following_auth_refusal( + monkeypatch, misses, refreshed, later_poll +) -> None: + from comfy_cli.command import build + + class TimeoutThenRefuse(_AnswersTheCut): + remaining = misses + pending_building = later_poll + + def __call__(self, url, target, **kwargs): + if kwargs.get("method") != "POST" and self.pending_building: + self.pending_building = False + return 200, {"id": "r1", "status": "building"} + if kwargs.get("method") != "POST" and self.remaining: + self.remaining -= 1 + raise TimeoutError("timed out") + return super().__call__(url, target, **kwargs) + + server = TimeoutThenRefuse(valid=set()) + sessions = _Sessions(first="signed-in", after=refreshed) + sleeps = [] + monkeypatch.setattr(build.time, "sleep", sleeps.append) + + result, error = _watched_cut(monkeypatch, server, sessions) + + assert result.exit_code == (75 if refreshed else 1), result.output + assert error["code"] == ("watch_auth_lost" if refreshed else "build_not_signed_in") + assert sleeps == ([build._RELEASE_POLL_SECONDS] if later_poll else []) + [4.0, 8.0, 16.0, 30.0][:misses] + assert server.tokens == (["signed-in", "signed-in", "refreshed"] if refreshed else ["signed-in", "signed-in"]) + assert sessions.forced == 1 + assert sessions.cleared is (refreshed is None) + assert server.remaining == 0 + if refreshed: + assert error["details"] == {"releaseId": "r1"} + assert "may still be building" in error["message"] + assert "comfy build release show r1" in error["hint"] + assert "comfy cloud login" not in error["hint"] + + +def test_a_release_watch_whose_refresh_signed_the_user_out_still_says_to_sign_in(monkeypatch) -> None: + # Given a watch refused, and a refresh that cleared the sign-in + server = _AnswersTheCut(valid=set()) + sessions = _Sessions(first="signed-in", after=None) + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then nobody is signed in any more, so signing in is the move + assert result.exit_code == 1 + assert error["code"] == "build_not_signed_in" + assert "comfy cloud login" in error["hint"] + + +def test_a_cut_refused_before_anything_was_made_keeps_the_sign_in_code(monkeypatch) -> None: + # Given a builder that refuses the cut itself, with the sign-in still stored + server = _Server(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then nothing is running, so the refusal is the sign-in's + assert result.exit_code == 1 + assert error["code"] == "build_not_signed_in" + assert server.tokens == ["signed-in", "refreshed"] + + +def test_a_deploy_watch_refused_after_the_refresh_ends_the_watch_not_the_deployment(monkeypatch) -> None: + # Given + from comfy_cli.command.deploy_runtime import DeployWatchAuthLostError, poll_deployment + + server = _Server(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + _install(monkeypatch, server, sessions) + sleeps: list[float] = [] + + # When + with pytest.raises(DeployWatchAuthLostError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", sleeps.append) + + # Then the read was refused after one refresh and one retry, and never retried again + assert exc_info.value.deployment_id == "dep-1" + assert exc_info.value.code == "watch_auth_lost" + assert server.tokens == ["signed-in", "refreshed"] + assert sleeps == [] + + +def test_a_watch_with_an_unrefreshable_stored_token_requires_sign_in(monkeypatch) -> None: + from comfy_cli.command.deploy_runtime import poll_deployment + + # A token without a refresh token can remain on disk after its lifetime ends. + session = SimpleNamespace(access_token="expired", refresh_token=None, base_url="https://cloud.comfy.org") + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: session) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", _Server(valid=set())) + + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", lambda _: None) + + assert exc_info.value.code == "deploy_not_signed_in" + + +class _SignedInElsewhere(_Sessions): + """A sign-in to another environment, stored while the watch ran.""" + + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): + session = super().__call__(refresh=refresh, force=force, allow_clear=allow_clear) + if self.forced and not force: + return SimpleNamespace(access_token=self.after, refresh_token="rt", base_url="https://staging.comfy.test") + return session + + +@pytest.mark.parametrize( + "sessions", + [_Sessions(first="signed-in", after=None), _SignedInElsewhere(first="signed-in", after="staging")], + ids=["refresh_signed_the_user_out", "signed_in_to_another_environment"], +) +def test_a_deploy_watch_with_no_sign_in_left_to_keep_says_to_sign_in(monkeypatch, sessions) -> None: + # Given + from comfy_cli.command.deploy_runtime import poll_deployment + + _install(monkeypatch, _Server(valid=set()), sessions) + + # When + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", lambda _: None) + + # Then + assert exc_info.value.code == "deploy_not_signed_in" + + +@pytest.mark.parametrize( + "build", + [ + lambda: DeployClient(_DEPLOY_URL, "injected"), + lambda: DeployClient( + _DEPLOY_URL, api_key=Credential(kind="api_key", value="k", source="env:COMFY_CLOUD_API_KEY") + ), + ], + ids=["handed_token", "workspace_key"], +) +def test_a_deploy_watch_on_a_credential_it_cannot_refresh_keeps_the_sign_in_code(monkeypatch, build) -> None: + # Given a sign-in stored beside a client that was never built from it + from comfy_cli.command.deploy_runtime import poll_deployment + + server = _Server(valid=set()) + _install(monkeypatch, server, _Sessions(first="signed-in", after="refreshed")) + + # When + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(build(), "dep-1", lambda _: None) + + # Then the credential the client sent is the one to replace + assert exc_info.value.code == "deploy_not_signed_in" + + +@pytest.fixture +def peer_takes_refresh_lock_after_first_read(monkeypatch, tmp_path): + """Own a real secrets lock; compress only its contended acquisition attempts.""" + from comfy_cli import locking + from comfy_cli.auth import store + from comfy_cli.cloud import oauth + + monkeypatch.setenv("COMFY_SECRETS_PATH", str(tmp_path / "secrets.json")) + store.save_cloud_session( + base_url="https://cloud.comfy.org", + resource="https://cloud.comfy.org/api", + client_id="cli", + scope="s", + access_token="signed-in", + refresh_token="RT0", + token_type="Bearer", + expires_at=4102444800, + ) + initial = store.secrets_path().read_bytes() + state = SimpleNamespace(armed=False, started=False, contended=False, timeouts=[], errors=[]) + held, release = threading.Event(), threading.Event() + real_acquire, real_read = locking._acquire, store.get_cloud_session + lock_stat = store.lock_path().stat() + + def peer(): + try: + with locking.file_lock(store.lock_path(), timeout=0): + state.contended = True + held.set() + if not release.wait(5): + raise AssertionError("test did not release its owned secrets lock") + state.contended = False + except BaseException as error: + state.errors.append(error) + held.set() + + owner = threading.Thread(target=peer) + + def acquire(fd, *, timeout): + info = os.fstat(fd) + owns_path = (info.st_dev, info.st_ino) == (lock_stat.st_dev, lock_stat.st_ino) + if owns_path and state.contended: + state.timeouts.append(timeout) + if timeout is None: + pytest.fail("unbounded secrets read after the refresh lock timed out") + # Exercise the actual contended OS lock without waiting out the + # production refresh budget. Classification already requests zero. + return real_acquire(fd, timeout=0) + return real_acquire(fd, timeout=timeout) + + def read(): + session = real_read() + if state.armed and not state.started: + state.started = True + owner.start() + assert held.wait(5), "test peer never acquired its secrets lock" + assert not state.errors + return session + + monkeypatch.setattr(locking, "_acquire", acquire) + monkeypatch.setattr(store, "get_cloud_session", read) + monkeypatch.setattr(oauth, "refresh_tokens", lambda **_: pytest.fail("must not refresh outside the held lock")) + + def release_peer(): + release.set() + if owner.ident is not None: + # Acquisition was nonblocking; the only possible wait is our own + # event, released above even when a regression assertion fails. + owner.join() + assert not state.errors + + state.release_peer = release_peer + try: + yield state + finally: + release_peer() + assert store.secrets_path().read_bytes() == initial + + +@pytest.mark.parametrize("kind", ["builder", "deploy"]) +def test_a_watch_does_not_wait_again_when_a_peer_outlives_the_refresh_lock( + monkeypatch, capsys, peer_takes_refresh_lock_after_first_read, kind +): + import json + + import typer + + from comfy_cli import credentials + from comfy_cli.caller import Caller + from comfy_cli.command.build import _watched_release + from comfy_cli.command.deploy_runtime import DeployWatchAuthLostError, exit_watch_lost, poll_deployment + from comfy_cli.output.renderer import Renderer + + state = peer_takes_refresh_lock_after_first_read + client = BuilderClient.from_session(_BUILDER_URL) if kind == "builder" else DeployClient.from_session(_DEPLOY_URL) + requests = [] + + def refused(url, target, **kwargs): + requests.append(target.auth_token) + state.armed = True + raise urllib.error.HTTPError(url, 401, "refused", http.client.HTTPMessage(), io.BytesIO(b"{}")) + + monkeypatch.setattr("comfy_cli.builder_api.request_json", refused) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", refused) + renderer = Renderer.resolve( + is_stdout_tty=False, env={}, caller=Caller(kind="user", agentic=False, source_env=None), json_flag=True + ) + with pytest.raises(typer.Exit) as stopped: + if kind == "builder": + _watched_release(renderer, client, "r1") + else: + try: + poll_deployment(client, "dep-1", lambda _: pytest.fail("must not sleep after the refused read")) + except DeployWatchAuthLostError as error: + exit_watch_lost(renderer, error) + + envelope = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert stopped.value.exit_code == 75 + assert envelope["error"]["code"] == "watch_auth_lost" + assert "comfy cloud login" not in envelope["error"]["hint"] + assert requests == ["signed-in"] + assert state.timeouts == [90.0, 0] + assert state.started is True + + # With the same real store readable again, the nested lock is reentrant. + state.release_peer() + assert credentials.sign_in_still_stored("https://cloud.comfy.org") is True + assert credentials.sign_in_still_stored(None) is False