From 9edb6d275751f6f48fd3d3dcc0fb92d63ec105e6 Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Wed, 4 Nov 2026 18:46:06 -0500 Subject: [PATCH 1/3] fix(auth): share one refresh across processes and keep watches alive on a 401 A request refused on an old access token now adopts a peer's refreshed token under the shared refresh lock, instead of rotating the family again, and retries at most once. A peer token is adopted only when its expiry is known and still ahead; otherwise the current stored family is refreshed. A build or deploy watch that still gets a 401 while a refreshable sign-in remains stored ends with the new code `watch_auth_lost` and exit 75: the release or deployment may still be running, the hint names the command that reads it again, and the stored sign-in is kept. The re-read that decides this does not wait on the lock again. A 401 before the work was accepted, on an API key, or with no refreshable sign-in left keeps `build_not_signed_in` / `deploy_not_signed_in`. Records the decision in ADR-AUTH-0007. Carries over #1044 by @wealthwiselabs, rebased on main. --- CHANGELOG.md | 2 + comfy_cli/builder_api.py | 7 + comfy_cli/cloud/oauth.py | 23 +- comfy_cli/command/build.py | 36 +- comfy_cli/command/deploy_runtime.py | 44 +- comfy_cli/credentials.py | 52 ++- comfy_cli/error_codes.py | 13 + comfy_cli/skills/comfy-build/SKILL.md | 5 +- .../skills/comfy-deploy-failures/SKILL.md | 1 + comfy_cli/skills/comfy-deploy/SKILL.md | 4 + ...ted-sign-in-once-and-keep-watches-alive.md | 217 +++++++++ docs/json-output.md | 8 + tests/comfy_cli/auth/test_oauth.py | 64 ++- .../comfy_cli/command/test_assets_library.py | 4 +- tests/comfy_cli/command/test_deploy_run.py | 4 +- tests/comfy_cli/command/test_deploy_status.py | 29 ++ tests/comfy_cli/command/test_deploy_up.py | 89 +++- .../command/test_transfer_download.py | 2 +- tests/comfy_cli/test_data_plane_token.py | 30 +- tests/comfy_cli/test_platform_api_key.py | 4 +- tests/comfy_cli/test_session_refresh.py | 423 +++++++++++++++++- 21 files changed, 1035 insertions(+), 26 deletions(-) create mode 100644 docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ade79741..0e1d304b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,8 @@ history. ### Fixed +- Parallel build and deploy requests rejected on an old access token now adopt a peer's refreshed token under the shared refresh lock instead of rotating it again. Each rejected request retries at most once. If session recovery still fails while a refreshable sign-in remains stored, the watch reports that the remote work may continue instead of telling you to sign in again. `comfy build release create --watch`, `comfy deploy up`, `promote`, `rollback` and `comfy deploy status --watch` now end with the new code `watch_auth_lost` and exit code 75, as `deploy_watch_lost` does. Its hint names the command that reads the release or deployment again. A 401 before anything was accepted, on a workspace API key, or when no refreshable sign-in remains still reports `build_not_signed_in` or `deploy_not_signed_in`. + - UI→API conversion expands a `COMFY_DYNAMICCOMBO_V3` selector's sub-inputs in the order the workflow was SAVED with, when the saved graph records one, instead of the option's current order. An option that has since gained a diff --git a/comfy_cli/builder_api.py b/comfy_cli/builder_api.py index 6bad57a92..1d7cd0ea4 100644 --- a/comfy_cli/builder_api.py +++ b/comfy_cli/builder_api.py @@ -137,6 +137,7 @@ def __init__( # handed a token (COMFY_BUILDER_TOKEN, COMFY_CLOUD_AUTH_TOKEN) or a key keeps # it and lets a 401 surface. self._refreshes_on_401 = False + self._sign_in_base_url: str | None = None # What to replace once the builder refuses this client's credential. self._refused_hint = ( credentials.refused_key_hint(api_key) @@ -175,9 +176,15 @@ def from_session(cls, base_url: str) -> BuilderClient: raise BuilderAuthError("not signed in: run `comfy cloud login` or set COMFY_CLOUD_API_KEY") client = cls(base_url, session.access_token) client._refreshes_on_401 = True + client._sign_in_base_url = session.base_url client._refused_hint = "run `comfy cloud login` first" return client + @property + def sign_in_base_url(self) -> str | None: + """The environment of the stored sign-in this client's token came from; None for a key or a handed token.""" + return self._sign_in_base_url if self._refreshes_on_401 else None + def _send(self, url: str, **kwargs) -> tuple[int, dict | list | None]: """Every builder request goes through here, so a long wait survives its token expiring. diff --git a/comfy_cli/cloud/oauth.py b/comfy_cli/cloud/oauth.py index cb6029c62..68e4f7ed2 100644 --- a/comfy_cli/cloud/oauth.py +++ b/comfy_cli/cloud/oauth.py @@ -782,6 +782,7 @@ def ensure_fresh_session( force: bool = False, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ): """Return the stored cloud session, refreshing it first when the access token is expired (or within ``leeway_s`` of expiring) and a refresh token @@ -803,6 +804,10 @@ def ensure_fresh_session( ``expires_at`` recorded — still thinks it is valid. The refresh token is still required; without one this is a no-op. + ``rejected_access_token`` identifies the token that received the 401. If a + peer already replaced it, adopt that session instead of refreshing again. + This also covers a late 401 that arrives after the peer released the lock. + ``allow_clear=False`` keeps the stored session on disk even when a refresh hits a fatal token error (reuse-detection / ``invalid_grant``). Background, read-mostly callers (the detached ``comfy run`` watcher) pass this so a @@ -855,6 +860,7 @@ def ensure_fresh_session( force=force, allow_clear=allow_clear, expect=expect, + rejected_access_token=rejected_access_token, ) except TimeoutError: # A peer has held the lock longer than a refresh should take. Don't @@ -875,6 +881,7 @@ def _locked_refresh( force: bool, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ): """Decide → POST → persist, executed while holding the secrets lock. @@ -890,11 +897,25 @@ def _locked_refresh( if session is None or not session.refresh_token: return session + # The rejected request may have been in flight while a peer finished its + # refresh, before even our first store read. Don't rotate that peer's token + # again just because this client's older request was refused. + if ( + rejected_access_token is not None + and session.access_token != rejected_access_token + and session.expires_at is not None + and not session.is_expired(leeway_s=leeway_s) + ): + return session + # Double-check after acquiring the lock. A peer may have refreshed while we # blocked: if the stored refresh token rotated away from what we observed # before locking, the family already advanced — use it, never re-send the # consumed token (that is exactly what trips reuse-detection). - if session.refresh_token != observed_refresh_token: + # A usable reactive peer already returned above. Otherwise refresh its + # current family, never replaying the older token observed before locking. + # Ordinary proactive callers still coalesce on any peer rotation. + if session.refresh_token != observed_refresh_token and rejected_access_token is None: return session if not force and not session.is_expired(leeway_s=leeway_s): # Peer advanced the access token's expiry without us noticing a token diff --git a/comfy_cli/command/build.py b/comfy_cli/command/build.py index 695a3df9e..147ff225c 100644 --- a/comfy_cli/command/build.py +++ b/comfy_cli/command/build.py @@ -3139,6 +3139,40 @@ def _selected_release_id(renderer, client, scope: _BuildScope, release: str | No return _newest_release_id(renderer, client, _resolve_build_id(renderer, client, scope)) +#: The exit a watch that lost its sign-in ends with, as `comfy deploy`'s watches do +#: (``deploy_runtime.EXIT_WATCH_LOST``): only the watch ended, so it must not exit 1, +#: which says the release failed. 75 is EX_TEMPFAIL in sysexits.h. +_EXIT_WATCH_LOST: Final = 75 + + +def _watched_release(renderer, client, release_id: str) -> dict: + """One read of a release this command has already cut. + + The builder accepted the cut with this sign-in, so a 401 session recovery did + not clear ends the watch while a refreshable sign-in remains stored. The + release may continue remotely. A key, a handed token, or a sign-in without + a refresh token keeps ``build_not_signed_in``. + """ + from comfy_cli import credentials + from comfy_cli.builder_api import BuilderCredentialRefused + + try: + return client.get_release(release_id) + except BuilderCredentialRefused as error: + if not credentials.sign_in_still_stored(getattr(client, "sign_in_base_url", None)): + raise + renderer.error( + code="watch_auth_lost", + message=f"the builder refused the sign-in (401) after attempting session recovery; " + f"release {release_id} may still be building on the server", + hint=f"read it with `comfy build release show {release_id}` until its status is complete; " + "the stored sign-in was preserved", + details={"releaseId": release_id}, + exit_code=_EXIT_WATCH_LOST, + ) + raise typer.Exit(code=_EXIT_WATCH_LOST) from error + + def _poll_call(renderer, fn, usable=None): """``_builder_call`` for one poll of a watch, which a single timed-out or dropped request must not end: the release keeps building either way. A @@ -3196,7 +3230,7 @@ def _poll_status(renderer, client, release_id: str) -> dict: word and ends the watch at the caller.""" return _poll_call( renderer, - lambda: client.get_release(release_id), + lambda: _watched_release(renderer, client, release_id), usable=lambda release: isinstance(release, dict) and isinstance(release.get("status"), str), ) diff --git a/comfy_cli/command/deploy_runtime.py b/comfy_cli/command/deploy_runtime.py index 9f477cf51..13759d5b6 100644 --- a/comfy_cli/command/deploy_runtime.py +++ b/comfy_cli/command/deploy_runtime.py @@ -12,6 +12,7 @@ import typer from typing_extensions import assert_never +from comfy_cli import credentials from comfy_cli.builder_api import BuilderClient, BuilderCredentialRefused from comfy_cli.command.build import DEFAULT_BUILDER_URL from comfy_cli.command.build_paths import BuildSpecNotFoundError, resolve_build_paths @@ -121,6 +122,8 @@ def terminal_status_error(deployment_id: str, status: str) -> JsonObject: class DeployWatchLostError(Exception): """The deploy API left a watch's reads unanswered for the whole retry window.""" + code = "deploy_watch_lost" + def __init__(self, deployment_id: str, cause: Exception | None = None, *, message: str | None = None) -> None: super().__init__( message or f"the deploy API stopped answering while following deployment {deployment_id} ({cause})" @@ -128,6 +131,23 @@ def __init__(self, deployment_id: str, cause: Exception | None = None, *, messag self.deployment_id = deployment_id +class DeployWatchAuthLostError(DeployWatchLostError): + """The deploy API refused a watch's read while a refreshable sign-in remains stored. + + The client has already attempted shared session recovery and at most one + retry. The deployment may continue remotely, so it is the watch that ended. + """ + + code = "watch_auth_lost" + + def __init__(self, deployment_id: str) -> None: + super().__init__( + deployment_id, + message=f"the deploy API refused the sign-in (401) while following deployment {deployment_id}, " + "after attempting session recovery", + ) + + def _unanswered(error: Exception) -> bool: """A read the service never answered: a 5xx, or no response at all. @@ -139,12 +159,26 @@ def _unanswered(error: Exception) -> bool: return error.code == "deploy_server_error" and (error.status is None or error.status >= 500) +def _sign_in_refused(client: DeployUpClient, error: Exception) -> bool: + """A 401 on a watch's read, from a client whose refreshable sign-in remains. + + A key, a handed token, or a sign-in without a refresh token keeps the + sign-in code, because changing that credential is the fix. + """ + if not isinstance(error, DeployAPIError) or error.status != 401: + return False + return credentials.sign_in_still_stored(getattr(client, "sign_in_base_url", None)) + + def exit_watch_lost(renderer: Renderer, error: DeployWatchLostError, hint: str | None = None) -> NoReturn: """Say it is the watch that gave out, not the deployment, and how to pick it up again.""" + hint = hint or f"run `{reattach_hint(error.deployment_id)}` to watch it again" + if isinstance(error, DeployWatchAuthLostError): + hint = f"{hint}; the stored sign-in was preserved" renderer.error( - code="deploy_watch_lost", + code=error.code, message=f"{error}; the deployment may still be coming up", - hint=hint or f"run `{reattach_hint(error.deployment_id)}` to watch it again", + hint=hint, details={"deployment_id": error.deployment_id}, exit_code=EXIT_WATCH_LOST, ) @@ -172,7 +206,9 @@ def poll_deployment( The progress a watcher shows rides the same read the loop already makes, so watching costs the service nothing it was not already answering. A read the service does not answer is retried, and the first of each run of them goes to - ``on_unanswered``; past the window this raises ``DeployWatchLostError``. + ``on_unanswered``; past the window this raises ``DeployWatchLostError``. A + 401 session recovery did not clear, with a refreshable sign-in still stored, + raises ``DeployWatchAuthLostError`` at once. """ failures, since, started = 0, 0.0, clock() while True: @@ -186,6 +222,8 @@ def poll_deployment( # The client maps a timeout, but a connection dropped before the status # line or partway through a 200's body escapes it unmapped. except (DeployAPIError, ConnectionError, http.client.HTTPException) as error: + if _sign_in_refused(client, error): + raise DeployWatchAuthLostError(deployment_id) from error if not _unanswered(error): raise if failures == 0: diff --git a/comfy_cli/credentials.py b/comfy_cli/credentials.py index 9cf75efd4..bcba23d50 100644 --- a/comfy_cli/credentials.py +++ b/comfy_cli/credentials.py @@ -92,6 +92,7 @@ def get_session( force: bool = False, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ) -> CloudSession | None: """Read the stored Comfy Cloud OAuth session. @@ -104,6 +105,8 @@ def get_session( ``force=True`` (implies the refresh path) refreshes unconditionally, ignoring the local expiry check. Reserved for the *reactive* path: after a server 401, the token is known-rejected even if our clock disagrees. + ``rejected_access_token`` lets that path adopt a peer's newer access token + instead of rotating it again; the comparison happens under the refresh lock. ``allow_clear=False`` forwards to ``ensure_fresh_session`` so a fatal refresh failure does NOT clear the stored session. Background watchers pass @@ -117,9 +120,12 @@ def get_session( if refresh or force: from comfy_cli.cloud import oauth + options = {} if expect is not None: - return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear, expect=expect) - return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear) + options["expect"] = expect + if rejected_access_token is not None: + options["rejected_access_token"] = rejected_access_token + return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear, **options) from comfy_cli.auth import store as auth_store return auth_store.get_cloud_session() @@ -140,14 +146,52 @@ def refreshed_access_token( :func:`get_session`. """ if expect is not None: - session = get_session(refresh=True, force=True, allow_clear=allow_clear, expect=expect) + session = get_session( + refresh=True, force=True, allow_clear=allow_clear, expect=expect, rejected_access_token=rejected + ) else: - session = get_session(refresh=True, force=True, allow_clear=allow_clear) + session = get_session(refresh=True, force=True, allow_clear=allow_clear, rejected_access_token=rejected) if session is None or not session.access_token or session.access_token == rejected: return None return session.access_token +def sign_in_still_stored(base_url: str | None) -> bool: + """Whether a refreshable sign-in for the refused client's environment remains. + + A 401 that outlives :func:`refreshed_access_token` and one retry does not show + that nobody is signed in: a peer process can hold the refresh lock past this + one's timeout, a refresh that fails on the network keeps the old token, and + the server can refuse a token it minted a moment ago. A sign-in without a + refresh token cannot recover this way and retains the sign-in guidance. + ``base_url`` is the environment the client's sign-in came from; ``None``, + for a key or a handed token, never matches. + """ + if base_url is None: + return False + from comfy_cli import locking + from comfy_cli.auth import store as auth_store + + try: + # Do not spend another lock budget after session recovery timed out. + # The store's nested read reuses this thread's lock reentrantly. + with locking.file_lock(auth_store.lock_path(), timeout=0): + session = get_session(refresh=False) + except TimeoutError: + # Contention cannot prove the held sign-in disappeared. End only the + # watch, rather than hanging or prescribing another login. + return True + except OSError as error: + _log.warning("could not re-read the stored sign-in after a 401: %s", error) + return False + return ( + session is not None + and bool(session.access_token) + and bool(session.refresh_token) + and session.base_url == base_url + ) + + class DataPlaneToken: """The bearer one long command sends to a deployment endpoint, kept fresh. diff --git a/comfy_cli/error_codes.py b/comfy_cli/error_codes.py index ff11c8db7..2bac298dd 100644 --- a/comfy_cli/error_codes.py +++ b/comfy_cli/error_codes.py @@ -1751,6 +1751,19 @@ class ErrorCode: "after a move, read `comfy deploy show --deployment ` instead: the move has landed once it shows no " "pendingUpdate and its releaseId is the release asked for", ), + ErrorCode( + "watch_auth_lost", + "A watch over work the server had already accepted (`comfy build release create --watch`, `comfy deploy " + "up`, `promote` and `rollback`, `comfy deploy status --watch`) was refused with HTTP 401 despite " + "serialized session recovery and at most one retry, while a refreshable sign-in remains stored. " + "Only the watch ended: the release may still be building and the deployment may still be coming up. " + "The exit code is 75, not 1. `details.releaseId` (build) or " + "`details.deployment_id` (deploy) names what is still running. A 401 before anything was accepted, on a " + "workspace API key, or with no refreshable sign-in remaining stays `build_not_signed_in` or " + "`deploy_not_signed_in`.", + "the stored sign-in was preserved: re-read the work with `comfy build " + "release show ` or `comfy deploy status --deployment --watch`", + ), ErrorCode( "deploy_update_failed", "`comfy deploy up`, `promote` or `rollback` moved a deployment onto another release, and the watch saw " diff --git a/comfy_cli/skills/comfy-build/SKILL.md b/comfy_cli/skills/comfy-build/SKILL.md index 83a2e85ce..57758174a 100644 --- a/comfy_cli/skills/comfy-build/SKILL.md +++ b/comfy_cli/skills/comfy-build/SKILL.md @@ -76,7 +76,10 @@ blob ls (hidden; workspace private blobs that reaches the builder need it; a plain scan and a plain `validate` do not. A CI job sets a workspace API key in `COMFY_CLOUD_API_KEY` instead, and that key wins over a stored sign-in, so when the hint says that key was refused, - signing in cannot help: replace the key. + signing in cannot help: replace the key. `watch_auth_lost` (exit 75) means + `release create --watch` was refused after the cut despite session recovery, + while a refreshable sign-in remains stored. The release may still be building. + Read it with `comfy build release show ` before considering a new sign-in. On `build_not_enabled` the platform is in limited beta and this account is not enabled — stop and say so. diff --git a/comfy_cli/skills/comfy-deploy-failures/SKILL.md b/comfy_cli/skills/comfy-deploy-failures/SKILL.md index cb44d581b..f0f68c1dc 100644 --- a/comfy_cli/skills/comfy-deploy-failures/SKILL.md +++ b/comfy_cli/skills/comfy-deploy-failures/SKILL.md @@ -55,6 +55,7 @@ argument you pass. | `deploy_update_replaced` | A watched `up`, `promote` or `rollback` was replaced by a newer update; `details.replacing_release_id` names its release | Follow it with `comfy deploy status --deployment `, or end it with `comfy deploy cancel --deployment ` | | `deploy_updates_unavailable` | `promote`, `rollback`, `history`, `cancel` or `events --release` on a deployment with no `revision`, or `cancel` against a deploy service too old for it | `promote` or `rollback`: `up --create` on the release instead. `history`: read `events`. `cancel`: wait for the update to land or fail. `events`: drop `--release`; those events do not say which release made them | | `deploy_watch_lost` | The watch's reads went unanswered for a minute, or a move by `up`, `promote` or `rollback` did not land within an hour (exit 75); the deployment may still be coming up | `comfy deploy status --deployment --watch`, or after a move `comfy deploy show --deployment ` until it shows no `pendingUpdate`; do not redeploy | +| `watch_auth_lost` | A watch's read was refused with a 401 despite session recovery, and a refreshable sign-in remains stored (exit 75); the deployment may still be coming up | Reattach as for `deploy_watch_lost` before considering a new sign-in; do not redeploy | | `deploy_delete_needs_confirm` | `delete` without `--yes` non-interactively | Confirm with the user, then pass `--yes` | ### Submitting a workflow diff --git a/comfy_cli/skills/comfy-deploy/SKILL.md b/comfy_cli/skills/comfy-deploy/SKILL.md index 46893eceb..df9b5dbab 100644 --- a/comfy_cli/skills/comfy-deploy/SKILL.md +++ b/comfy_cli/skills/comfy-deploy/SKILL.md @@ -176,6 +176,10 @@ stops answering (a 5xx or a dropped connection, as during its own rollout) is retried for about a minute; if it is still down, the wait ends with `deploy_watch_lost` and **exit 75**, not 1. That is not a failed deployment: attach again with the command in the hint rather than redeploying. +`watch_auth_lost`, also exit 75, is a read refused with a 401 despite session +recovery, while a refreshable sign-in remains stored. Parallel requests adopt a +peer's refreshed token and retry at most once. Attach again the same way before +considering a new sign-in; the deployment may still be coming up. `status` also reports **why** a deployment stopped, as `stopReason`: `user`, `credits`, or `policy`. `credits` is a billing problem and not something a retry diff --git a/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md b/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md new file mode 100644 index 000000000..06a00af8c --- /dev/null +++ b/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md @@ -0,0 +1,217 @@ +# ADR-AUTH-0007: Recover a Rejected Sign-In Once, and Keep Watches Alive + +Date: 2026-11-04 + +## Status + +Proposed + +Relation: Builds on [ADR-AUTOMATION-0003](AUTOMATION-0003-one-versioned-machine-output-dialect.md). +One envelope per command, append-only error codes and human diagnostics off +stdout all stay as they are. This ADR adds one error code, `watch_auth_lost`, +and fixes what it means. + +## Context + +A Comfy Cloud sign-in is one stored OAuth session: an access token that lasts +about fifteen minutes and a refresh token that rotates on every use. The +server tracks each refresh token as part of a family. Sending a refresh token +that was already used is treated as theft: the server kills the whole family, +and the user has to sign in again. + +Several `comfy` processes can share that one stored session at once: a +`comfy build release create --watch` in one terminal, a `comfy deploy status +--watch` in another, an agent running `comfy deploy up` in a third. Each of +them can get a 401 when its access token expires mid-wait. Before this +decision, two failure modes followed from that: + +- Two processes rejected on the same old access token could both refresh. The + second one either replayed a consumed refresh token (and killed the family) + or rotated the token the first had just stored, so every peer refreshed in + turn. +- A watch over work the server had already accepted (a release that is + building, a deployment that is coming up) ended with `build_not_signed_in` + or `deploy_not_signed_in` and exit 1 when its recovery failed. That told the + user to sign in again and suggested the work had failed, while the release + or deployment was still running, and the stored sign-in was often fine. + +Who owns the token family, when a process may refresh it, and what a watch +reports when it loses its sign-in are contracts that later auth or watch code +must keep. They belong in an ADR, not only in code comments. + +## Decision + +### One owner of the token family at a time + +- Only a process holding the cross-process secrets lock + (`auth_store.lock_path()`) may refresh or write the stored session. The lock + wait is bounded at 90 seconds (`_REFRESH_LOCK_TIMEOUT_S`). +- Under the lock the process re-reads the store and decides from what is + stored now, never from what it read before locking. It never sends a + refresh token it observed before taking the lock if the store has since + moved on. +- If the lock wait times out, the process does not refresh and does not + re-read the store. It returns the session it held before locking and + leaves the stored session untouched. + +### What a process rejected with a 401 does + +A request refused with a 401 passes the access token that was refused +(`rejected_access_token`) into the shared refresh. Under the lock: + +1. **Adopt a usable peer session.** If the stored access token differs from + the rejected one and has a known, unexpired `expires_at` (outside the usual + leeway), a peer already refreshed. Use that session and do not refresh. +2. **Otherwise refresh the current family.** A stored token with no + `expires_at`, an expired one, or the rejected token itself is not usable. + The process refreshes with the refresh token stored now. A token whose expiry + is unknown is never adopted, because the one retry would just get another 401. +3. **Proactive callers** (a process refreshing before expiry, with no 401) keep + the earlier rule: any peer rotation since their pre-lock read means the + family already moved, and they use the stored session. + +### When the stored session is cleared + +- Only an error that means the family is dead (`invalid_grant`, + `invalid_token`, or a reuse-detection signal) clears the stored session, and + only if the caller did not pass `allow_clear=False`. Background read-mostly + callers, such as the detached `comfy run` watcher, pass that. +- Any other refresh failure (network, 5xx, timeout, `invalid_client`) never + clears it. The old session stays and the caller's request surfaces its 401. + +### One retry per rejected request + +`BuilderClient._send` and the deploy API client retry a 401 at most once, and +only for a client built from the stored sign-in. The order is: 401, shared +recovery, one retry with the new token, and if that is refused too, the +command reports the refusal. A client handed a token (`COMFY_BUILDER_TOKEN`, +`COMFY_CLOUD_AUTH_TOKEN`) or a workspace API key never refreshes and never +retries. The server refuses a 401 before acting, so the retry is safe for +every method; a retried create carries the same idempotency key. + +### What a watch reports when it loses its sign-in + +A watch that reads work the server already accepted ends with `watch_auth_lost` +and exit code **75** (`EX_TEMPFAIL`, the same exit as `deploy_watch_lost`), +not 1, when all of these hold: + +- the read got a 401 after shared recovery and the one retry; +- the client was built from the stored sign-in (it has a `sign_in_base_url`); +- a sign-in that can still recover remains stored: it has an access token, a + refresh token, and the same `base_url` as the client's. + +That re-read (`credentials.sign_in_still_stored`) takes the secrets lock with +`timeout=0`. If a peer holds the lock, the answer is "still stored": being +locked out does not prove the sign-in is gone, and the watch must not hang +for another lock budget after recovery already timed out. If the store +cannot be read at all (`OSError`), the answer is "not stored". + +The envelope says only the watch ended: the message says the release may still +be building or the deployment may still be coming up. `details.releaseId` +(build) or `details.deployment_id` (deploy) names the work. The hint names the +command that reads it again (`comfy build release show `, or `comfy deploy +status --deployment --watch`) and says the stored sign-in was preserved. +The watch never clears the stored sign-in on this path. + +Every other 401 keeps its sign-in code and its "sign in again" or "replace +the key" hint: a 401 before the server accepted anything, a 401 on an API key +or a handed token, or a 401 with no recoverable sign-in left. +`build_not_signed_in` and `deploy_not_signed_in` keep their meaning. + +## Alternatives considered + +- **Always refresh after a 401.** This is the simplest rule, and it is the bug: + N processes rejected on the same old token rotate the family N times, and a + late one replays a consumed token and kills the family. +- **Adopt any stored token that differs from the rejected one.** Rejected + because a peer token with no recorded expiry, or one that has since expired, + gets a second 401 on the only retry. This case was raised in review and is + handled by the expiry check in rule 1. +- **Retry until a request succeeds.** Rejected because a server that keeps + refusing a valid-looking token would loop forever, and each loop spends a + refresh. One retry covers the real case, a token that expired mid-wait. +- **Wait on the lock again to re-read the store before classifying a watch + 401.** Rejected because after a 90 second timeout a second wait can hang the + watch for as long again, or forever with no timeout. A non-blocking read + answers the only question the classification needs. +- **Reuse `deploy_watch_lost`, or add a separate code for build.** Rejected + because `deploy_watch_lost` means the service stopped answering, which needs a + different next step. One `watch_auth_lost` covers both build and deploy + watches, with the same exit code. +- **Keep `build_not_signed_in` / exit 1 for a lost watch.** Rejected because it + makes callers and agents resubmit work that is still running, and sign in + again when the stored sign-in is fine. + +## What we reject + +- Refreshing, or writing the stored session, outside the secrets lock. +- Sending a refresh token observed before the lock after the store has moved on. +- Treating a token with unknown expiry as usable for a reactive adoption. +- More than one retry per rejected request, or any retry for a key or a handed + token. +- Clearing the stored session on a refresh failure that does not mean the + family is dead, or from a watch that ends with `watch_auth_lost`. +- Waiting on the secrets lock with no timeout on any path that runs after a + 401. +- Exit 1 for a watch that ended while the server-side work may still be + running. +- Reusing `watch_auth_lost` for a failure before the server accepted the work. + +## Consequences + +### Positive + +- Parallel commands on one sign-in refresh once, not once each, and a late 401 + no longer kills the token family. +- A watch that loses its sign-in tells the truth: the work may still be + running, here is how to read it, and the sign-in was kept. Exit 75 lets + scripts tell "watch ended" from "work failed". +- Build and deploy watches report the same code and exit for the same + situation. + +### Negative + +- `watch_auth_lost` is a new public error code. Under ADR-AUTOMATION-0003 it + is append-only once released and cannot be renamed or repurposed. +- A watch can end with `watch_auth_lost` when a peer held the lock and the + sign-in was in fact about to be lost. The next command then reports the + sign-in code. That is the cost of not hanging. +- Callers that matched on `build_not_signed_in` / `deploy_not_signed_in` to + detect a lost watch now see a different code and exit 75. + +### Known limits + +- The lock is a file lock on one machine. Two machines sharing one stored + session (a synced home directory) are not coordinated. +- The tests cover real temporary credential storage and a real OS lock, with + mocked HTTP. They do not cover a live OAuth server's reuse detection under + concurrency. + +## Validation + +- `tests/comfy_cli/auth/test_oauth.py`: a rejected older token adopts a peer's + unexpired token; it does not adopt an expired peer token or one with no + expiry, and refreshes with the stored family instead. +- `tests/comfy_cli/test_session_refresh.py`: overlapping clients reuse the peer + refresh before retrying; one retry, then a second 401 surfaces; handed tokens + never swap; the retried create keeps its idempotency key; build and deploy + watches end as `watch_auth_lost` with the sign-in kept; a watch does not wait + again when a peer outlives the refresh lock; unrefreshable, signed-out and + pre-accept 401s keep the sign-in code. +- `tests/comfy_cli/command/test_deploy_up.py` and `test_deploy_status.py`: the + follow read's 401 with and without a stored sign-in, and a 401 on the create. +- The error-code registry tests keep `watch_auth_lost` registered and emitted + from the same places. + +## Revisit triggers + +- The server starts returning a distinct status or error for "token expired" + versus "token revoked". The adopt-or-refresh rule could then read it + directly. +- Access-token lifetime moves well past fifteen minutes, or the server stops + rotating refresh tokens. One retry and the family-ownership rule may then + be more than is needed. +- Field reports of `watch_auth_lost` followed by a work item that had in fact + failed, or of family-kill re-logins with this rule in place. +- Session storage moves off the local file store (for example to an OS + keychain with its own locking). diff --git a/docs/json-output.md b/docs/json-output.md index 523082a5f..ef463790b 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -572,6 +572,14 @@ watch ends with the error code `deploy_watch_lost` and exit code **75**, never 1 only the watch ended, and the deployment may still be coming up. Its `hint` carries the command that re-attaches and `details.deployment_id` the deployment. +A 401 on a watch's read triggers serialized session recovery: the client adopts +a peer's newer token or refreshes the rejected one, then retries at most once. +If recovery fails while a refreshable sign-in for the same environment remains +stored, the watch ends with `watch_auth_lost`, again with exit code **75** and +the same `details`. Its `hint` reattaches the watch without a re-login prompt. +The remote work may continue. A 401 with no refreshable sign-in remaining, or +on a workspace API key, stays `deploy_not_signed_in`. + ## Success envelope On `--wait` success, `data` carries: diff --git a/tests/comfy_cli/auth/test_oauth.py b/tests/comfy_cli/auth/test_oauth.py index c7064763b..bfdb821fb 100644 --- a/tests/comfy_cli/auth/test_oauth.py +++ b/tests/comfy_cli/auth/test_oauth.py @@ -703,7 +703,8 @@ def worker(): # (b) Double-check: if a peer already rotated the token between our pre-lock # read and the post-lock re-read, we adopt it and do NOT refresh again. - def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, monkeypatch): + @pytest.mark.parametrize("peer_expiry", [None, 1, 4102444800], ids=["unknown-expiry", "expired", "known-unexpired"]) + def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, monkeypatch, peer_expiry): self._persist_expired(refresh_token="RT0") # Simulate a peer that rotated RT0 → RT1 while we waited for the lock: @@ -716,7 +717,7 @@ def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, m access_token="PEER_NEW", refresh_token="RT1", token_type="Bearer", - expires_at=int(time.time()) + 3600, + expires_at=peer_expiry, saved_at="2026-01-01T00:00:02+00:00", ) reads = [self._stale(), fresh_peer] @@ -727,6 +728,65 @@ def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, m assert result.refresh_token == "RT1" assert result.access_token == "PEER_NEW" + def test_a_rejected_older_token_does_not_adopt_an_already_expired_peer_token(self, persisted, monkeypatch): + from comfy_cli.credentials import refreshed_access_token + + self._persist_expired() + monkeypatch.setattr(oauth, "refresh_tokens", lambda **_: self._token_set(access="NEW", refresh="RT1")) + + assert refreshed_access_token("BEFORE_OLD") == "NEW" + assert auth_store.get_cloud_session().refresh_token == "RT1" + + @pytest.mark.parametrize( + ("peer_access", "peer_expiry"), + [("PEER_UNUSABLE", None), ("PEER_UNUSABLE", 1), ("REJECTED_OLDER", 4102444800)], + ids=["unknown-expiry", "expired", "same-rejected-token"], + ) + @pytest.mark.parametrize("peer_timing", ["already-stored", "after-first-read"]) + def test_a_peer_token_that_cannot_be_retried_uses_its_available_refresh( + self, persisted, monkeypatch, peer_timing, peer_access, peer_expiry + ): + from comfy_cli.credentials import refreshed_access_token + + def save(access, refresh, expiry): + return auth_store.save_cloud_session( + base_url="https://c", + resource="https://c/api", + client_id="cid", + scope="s", + access_token=access, + refresh_token=refresh, + token_type="Bearer", + expires_at=expiry, + ) + + save("REJECTED_OLDER", "RT0", 4102444800) + if peer_timing == "already-stored": + save(peer_access, "RT1", peer_expiry) + else: + real_read = auth_store.get_cloud_session + + def read_then_peer_rotates(): + session = real_read() + if session.refresh_token == "RT0": + save(peer_access, "RT1", peer_expiry) + return session + + monkeypatch.setattr(auth_store, "get_cloud_session", read_then_peer_rotates) + refreshes = [] + + def refresh(**kwargs): + refreshes.append(kwargs["refresh_token"]) + return self._token_set(access="NEW", refresh="RT2") + + monkeypatch.setattr(oauth, "refresh_tokens", refresh) + + assert refreshed_access_token("REJECTED_OLDER") == "NEW" + stored = auth_store.get_cloud_session() + assert stored.access_token == "NEW" + assert stored.refresh_token == "RT2" + assert refreshes == ["RT1"] + def _stale(self) -> auth_store.CloudSession: return auth_store.CloudSession( base_url="https://c", diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 718655009..4a99b58b8 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -1148,7 +1148,7 @@ def signed_in(monkeypatch: pytest.MonkeyPatch): session=SimpleNamespace(access_token="old-token", base_url="https://cloud.example.com"), forced=0 ) - def get_session(*, refresh=True, force=False, allow_clear=True, expect=None): + def get_session(*, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: state.forced += 1 state.session = SimpleNamespace(access_token="new-token", base_url="https://cloud.example.com") @@ -1261,7 +1261,7 @@ def stored_login(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv(var, raising=False) state = SimpleNamespace(login=("a", 1), forced=0) - def get_session(*, refresh=True, force=False, allow_clear=True, expect=None): + def get_session(*, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: state.forced += 1 state.login = (state.login[0], state.login[1] + 1) diff --git a/tests/comfy_cli/command/test_deploy_run.py b/tests/comfy_cli/command/test_deploy_run.py index f2629f77f..c67768daf 100644 --- a/tests/comfy_cli/command/test_deploy_run.py +++ b/tests/comfy_cli/command/test_deploy_run.py @@ -613,7 +613,7 @@ class _RotatingSignIn: def __init__(self) -> None: self.token = "token-at-start" - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): return SimpleNamespace(access_token=self.token, base_url="https://cloud.comfy.org") @@ -673,7 +673,7 @@ def test_an_unreadable_sign_in_at_cancel_still_cancels_with_the_token_the_run_ho monkeypatch.setattr(deploy_run, "resolve_partner_credential", lambda: None) store = {"readable": True} - def get_session(*, refresh=True, force=False, allow_clear=True): + def get_session(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if not store["readable"]: raise OSError("auth store lock unavailable") return SimpleNamespace(access_token="cloud-secret", base_url="https://cloud.comfy.org") diff --git a/tests/comfy_cli/command/test_deploy_status.py b/tests/comfy_cli/command/test_deploy_status.py index 0076d559d..86b9d5a4d 100644 --- a/tests/comfy_cli/command/test_deploy_status.py +++ b/tests/comfy_cli/command/test_deploy_status.py @@ -4,6 +4,7 @@ import json from datetime import datetime, timedelta, timezone from pathlib import Path +from types import SimpleNamespace import jsonschema import pytest @@ -801,6 +802,34 @@ def get_deployment(self, deployment_id: str) -> JsonObject: assert len(client.get_calls) == 7 +def test_a_watch_refused_with_the_sign_in_still_stored_ends_as_watch_auth_lost(tmp_path, monkeypatch) -> None: + # Given a watch whose read is refused after the client's refresh and retry + class Refused(RecordingDeploy): + sign_in_base_url = "https://cloud.comfy.org" + + def get_deployment(self, deployment_id: str) -> JsonObject: + self.get_calls.append(deployment_id) + raise DeployAPIError("deploy_not_signed_in", "invalid auth token", status=401) + + client = Refused([_status_deployment(status="provisioning")]) + sleeps: list[float] = [] + _install_clients(monkeypatch, FakeBuilder(), client, sleeps) + stored = SimpleNamespace(access_token="token", refresh_token="rt", base_url="https://cloud.comfy.org") + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: stored) + + # When + result = _invoke_json(write_spec(tmp_path), "--watch") + + # Then + assert result.exit_code == 75, result.stderr + error = _json_envelope(result)["error"] + assert error["code"] == "watch_auth_lost" + assert "comfy deploy status --deployment dep-status --watch" in error["hint"] + assert "comfy cloud login" not in error["hint"] + assert client.get_calls == ["dep-status"] + assert sleeps == [] + + def test_a_watch_that_gives_up_takes_the_live_line_down_before_the_error_prints(tmp_path, monkeypatch) -> None: """An error printed while the live line is up lands under a spinner that is still turning.""" # Given a watch that draws its line, then never hears back. CliRunner's diff --git a/tests/comfy_cli/command/test_deploy_up.py b/tests/comfy_cli/command/test_deploy_up.py index d10e6927d..5df61ef8c 100644 --- a/tests/comfy_cli/command/test_deploy_up.py +++ b/tests/comfy_cli/command/test_deploy_up.py @@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace import jsonschema import pytest @@ -814,6 +814,93 @@ def test_a_4xx_on_the_follow_read_still_fails_at_once(tmp_path, monkeypatch) -> assert client.get_calls == 2 +_CLOUD = "https://cloud.comfy.org" + + +def _refused() -> DeployAPIError: + # What the client raises once a 401 outlives its forced refresh and its one retry. + return DeployAPIError( + "deploy_not_signed_in", + "invalid auth token", + status=401, + details={"operation": "get", "status": 401, "server_code": "UNAUTHORIZED"}, + ) + + +def _stored_sign_in(monkeypatch, base_url: str | None) -> None: + session = ( + SimpleNamespace(access_token="token", refresh_token="rt", base_url=base_url) if base_url is not None else None + ) + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: session) + + +def test_a_401_on_the_follow_read_with_the_sign_in_still_stored_ends_the_watch_not_the_deployment( + tmp_path, monkeypatch +) -> None: + """Parallel watches can each be refused after the shared refresh. The deployment + was accepted with that same sign-in and keeps coming up, so the envelope must + not send an agent to sign in, nor report a failure that did not happen.""" + # Given a deployment the service accepted, then a follow read it refuses + client = FlakyFollow([_refused()], get_statuses=["queued", "ready"]) + client.sign_in_base_url = _CLOUD # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, _CLOUD) + sleeps: list[float] = [] + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, sleeps) + + # Then + assert result.exit_code == 75, result.stderr + error = _json_envelope(result)["error"] + assert error["code"] == "watch_auth_lost" + assert "may still be coming up" in error["message"] + assert error["hint"].startswith("run `comfy deploy status --deployment dep-1 --watch` to watch it again") + assert "comfy cloud login" not in error["hint"] + assert error["details"] == {"deployment_id": "dep-1"} + assert sleeps == [] + assert len(client.create_keys) == 1 + + +@pytest.mark.parametrize( + ("client_sign_in", "stored_sign_in"), + [(_CLOUD, None), (_CLOUD, "https://staging.comfy.test"), (None, _CLOUD)], + ids=["refresh_signed_the_user_out", "signed_in_to_another_environment", "workspace_key"], +) +def test_a_401_on_the_follow_read_with_no_sign_in_to_keep_still_says_to_sign_in( + tmp_path, monkeypatch, client_sign_in, stored_sign_in +) -> None: + # Given a follow read refused, and no sign-in this watch's client could keep using + client = FlakyFollow([_refused()], get_statuses=["queued", "ready"]) + client.sign_in_base_url = client_sign_in # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, stored_sign_in) + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, []) + + # Then + assert result.exit_code == 1 + assert _json_envelope(result)["error"]["code"] == "deploy_not_signed_in" + + +class _RefusesTheCreate(FakeDeploy): + def create_deployment(self, release_id: str, compute_config: JsonObject, **kwargs) -> JsonObject: + raise _refused() + + +def test_a_401_on_the_create_keeps_the_sign_in_code(tmp_path, monkeypatch) -> None: + # Given a sign-in still stored, and a service that refuses the create itself + client = _RefusesTheCreate() + client.sign_in_base_url = _CLOUD # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, _CLOUD) + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, []) + + # Then nothing was accepted, so nothing runs on, and the refusal is the sign-in's + assert result.exit_code == 1 + assert _json_envelope(result)["error"]["code"] == "deploy_not_signed_in" + + def test_a_second_outage_in_one_watch_is_announced_and_backed_off_afresh(tmp_path, monkeypatch) -> None: """A read that is answered ends the outage, so the next one is new: it is announced again and gets the whole retry schedule, not what was left of the first.""" diff --git a/tests/comfy_cli/command/test_transfer_download.py b/tests/comfy_cli/command/test_transfer_download.py index 889515bf5..0dc887921 100644 --- a/tests/comfy_cli/command/test_transfer_download.py +++ b/tests/comfy_cli/command/test_transfer_download.py @@ -1428,7 +1428,7 @@ def __init__(self, base_url: str, first: str) -> None: self.token = first self.refreshing_reads: list[bool] = [] - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): from types import SimpleNamespace if refresh: diff --git a/tests/comfy_cli/test_data_plane_token.py b/tests/comfy_cli/test_data_plane_token.py index 9f138686c..82323c836 100644 --- a/tests/comfy_cli/test_data_plane_token.py +++ b/tests/comfy_cli/test_data_plane_token.py @@ -27,7 +27,7 @@ def __init__(self, proactive: str, forced: str | None) -> None: self.forced = forced self.forced_calls = 0 - def __call__(self, *, refresh=True, force=False, allow_clear=True, expect=None): + def __call__(self, *, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: self.forced_calls += 1 if not self.forced: @@ -181,7 +181,7 @@ def test_a_sign_in_to_another_environment_mid_run_is_never_sent_or_refreshed(mon refreshing: list[bool] = [] - def get_session(*, refresh=True, force=False, allow_clear=True): + def get_session(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if refresh or force: refreshing.append(allow_clear) return stored @@ -382,3 +382,29 @@ def get_session(**kwargs): # Then: the refresh is asked exactly as before, with no expectation attached assert calls and all("expect" not in kwargs for kwargs in calls) + + +def test_an_account_bound_holder_adopts_the_same_accounts_peer_refresh(login_race) -> None: + # A peer finishes its refresh while the held token's rejected request is in flight. + held = _save_login("a", expires_at=_LATER).access_token + token = DataPlaneToken(held, refreshes=True, base_url=_CLOUD, same_account=True) + peer = _jwt("a", 2) + from comfy_cli.auth import store + + stored = store.get_cloud_session() + store.save_cloud_session( + base_url=stored.base_url, + resource=stored.resource, + client_id=stored.client_id, + scope=stored.scope, + access_token=peer, + refresh_token="refresh-a-peer", + token_type="Bearer", + expires_at=_LATER, + ) + + assert token.after_rejection() is True + assert token.current() == peer + assert login_race.refreshed == [] + assert store.get_cloud_session().refresh_token == "refresh-a-peer" + assert token.account_replaced is False diff --git a/tests/comfy_cli/test_platform_api_key.py b/tests/comfy_cli/test_platform_api_key.py index 64a14f17d..3c6c2b1d8 100644 --- a/tests/comfy_cli/test_platform_api_key.py +++ b/tests/comfy_cli/test_platform_api_key.py @@ -41,7 +41,7 @@ def __call__(self, url, target, **kwargs): def _install(monkeypatch, server: _Server, key: Credential | None, sign_in: str | None = "signed-in-token") -> list: refreshes: list = [] - def sessions(*, refresh=True, force=False, allow_clear=True): + def sessions(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if force: refreshes.append(force) return SimpleNamespace(access_token=sign_in, base_url="https://cloud.comfy.org") if sign_in else None @@ -187,7 +187,7 @@ def test_a_refreshed_sign_in_the_builder_still_refuses_names_the_login(monkeypat server = _Server(status=401) refreshed = iter(["refreshed-token"]) - def sessions(*, refresh=True, force=False, allow_clear=True): + def sessions(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): return SimpleNamespace( access_token=next(refreshed) if force else "signed-in-token", base_url="https://cloud.comfy.org" ) diff --git a/tests/comfy_cli/test_session_refresh.py b/tests/comfy_cli/test_session_refresh.py index a0db03ba4..027d285bd 100644 --- a/tests/comfy_cli/test_session_refresh.py +++ b/tests/comfy_cli/test_session_refresh.py @@ -10,12 +10,16 @@ import http.client import io +import os +import threading +import time import urllib.error from types import SimpleNamespace import pytest from comfy_cli.builder_api import BuilderClient +from comfy_cli.credentials import Credential from comfy_cli.deploy_api import DeployAPIError, DeployClient _DEPLOY_URL = "https://deploy.test/deploy" @@ -38,18 +42,27 @@ def __call__(self, url, target, **kwargs): class _Sessions: - """Stands in for the stored sign-in: hands out ``first`` until forced, then ``after``.""" + """Stands in for the stored sign-in: hands out ``first`` until forced, then ``after``. + + A forced refresh that yields no token clears the store, as a fatal refresh does. + """ def __init__(self, first: str, after: str | None) -> None: self.first = first self.after = after self.forced = 0 + self.cleared = False - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if force: self.forced += 1 - return SimpleNamespace(access_token=self.after, base_url="https://cloud.comfy.org") if self.after else None - return SimpleNamespace(access_token=self.first, base_url="https://cloud.comfy.org") + if self.after is None: + self.cleared = allow_clear + return None + return SimpleNamespace(access_token=self.after, refresh_token="rt", base_url="https://cloud.comfy.org") + if self.cleared: + return None + return SimpleNamespace(access_token=self.first, refresh_token="rt", base_url="https://cloud.comfy.org") def _install(monkeypatch, server: _Server, sessions: _Sessions) -> None: @@ -75,6 +88,63 @@ def _install(monkeypatch, server: _Server, sessions: _Sessions) -> None: } +@pytest.mark.parametrize("lagging", ["builder", "deploy"]) +def test_overlapping_clients_reuse_the_peer_refresh_before_retrying(monkeypatch, tmp_path, lagging) -> None: + """A late 401 for OLD must not rotate the NEW token a peer just obtained.""" + from comfy_cli.auth import store + from comfy_cli.cloud import oauth + + monkeypatch.setattr(store, "secrets_path", lambda: tmp_path / "secrets.json") + store.save_cloud_session( + base_url="https://cloud.comfy.org", + resource="https://cloud.comfy.org/api", + client_id="cli", + scope="s", + access_token="OLD", + refresh_token="RT0", + token_type="Bearer", + expires_at=int(time.time()) + 3600, + ) + refreshes = [] + + def refresh(**kwargs): + refreshes.append(kwargs["refresh_token"]) + generation = len(refreshes) + return oauth.TokenSet( + access_token=f"NEW{generation}", + refresh_token=f"RT{generation}", + token_type="Bearer", + expires_in=3600, + expires_at=int(time.time()) + 3600, + scope="s", + ) + + monkeypatch.setattr(oauth, "refresh_tokens", refresh) + clients = {"builder": BuilderClient.from_session(_BUILDER_URL), "deploy": DeployClient.from_session(_DEPLOY_URL)} + peer = "deploy" if lagging == "builder" else "builder" + calls = { + "builder": lambda: clients["builder"].get_release("r1"), + "deploy": lambda: clients["deploy"].get_deployment("dep-1"), + } + requests = [] + + def send(url, target, **kwargs): + name = "builder" if url.startswith(_BUILDER_URL) else "deploy" + requests.append((name, target.auth_token)) + if target.auth_token == "OLD": + if name == lagging: + calls[peer]() # Peer finishes its refresh while this stale request is in flight. + raise urllib.error.HTTPError(url, 401, "expired", http.client.HTTPMessage(), io.BytesIO(b"{}")) + return 200, dict(_ANSWER) + + monkeypatch.setattr("comfy_cli.builder_api.request_json", send) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", send) + + assert calls[lagging]()["status"] == "ready" + assert refreshes == ["RT0"] + assert requests == [(lagging, "OLD"), (peer, "OLD"), (peer, "NEW1"), (lagging, "NEW1")] + + @pytest.mark.parametrize("call", _DEPLOY_CALLS.values(), ids=_DEPLOY_CALLS.keys()) def test_deploy_client_retries_once_with_a_refreshed_token_after_a_401(monkeypatch, call) -> None: # Given @@ -260,3 +330,348 @@ def test_a_refused_injected_token_says_to_replace_it_not_to_sign_in(monkeypatch) assert "COMFY_BUILDER_TOKEN" in envelope["error"]["hint"] assert "comfy cloud login" not in envelope["error"]["hint"] assert server.tokens == ["injected"] + + +# --- A 401 that outlives the refresh during a watch ----------------------------- +# +# Several commands polling at once can each be refused after the shared refresh: +# a peer can hold the refresh lock past this process's timeout, and then this one +# retries nothing. The build or deployment was accepted with the same sign-in a +# moment earlier and runs on, so while that sign-in is still stored, signing in +# again is not the fix. A 401 before anything was submitted keeps its sign-in code. + + +class _AnswersTheCut(_Server): + """Answers the cut whatever token it carries, then refuses every read the watch makes.""" + + def __call__(self, url, target, **kwargs): + if kwargs.get("method") == "POST": + self.tokens.append(target.auth_token) + return 200, dict(_ANSWER) + return super().__call__(url, target, **kwargs) + + +def _watched_cut(monkeypatch, server: _Server, sessions: _Sessions): + import json + + from typer.testing import CliRunner + + from comfy_cli.cmdline import app + + _install(monkeypatch, server, sessions) + monkeypatch.delenv("COMFY_BUILDER_TOKEN", raising=False) + monkeypatch.delenv("COMFY_CLOUD_API_KEY", raising=False) + result = CliRunner().invoke( + app, + ["--json", "build", "release", "create", "--id", "b1", "--target", "linux/nvidia", "--watch"], + env={"COMFY_BUILDER_URL": _BUILDER_URL}, + ) + envelope = json.loads([line for line in result.stdout.splitlines() if line.strip()][-1]) + return result, envelope["error"] + + +def test_a_release_watch_refused_after_the_refresh_says_the_release_keeps_building(monkeypatch) -> None: + # Given a cut the builder accepts, then a watch it refuses even after the refresh + server = _AnswersTheCut(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then only the watch ended: its own code and exit status, and no sign-in prompt + assert result.exit_code == 75 + assert error["code"] == "watch_auth_lost" + assert "may still be building" in error["message"] + assert "comfy build release show r1" in error["hint"] + assert "comfy cloud login" not in error["hint"] + assert error["details"] == {"releaseId": "r1"} + assert server.tokens == ["signed-in", "signed-in", "refreshed"] + assert sessions.forced == 1 + + +@pytest.mark.parametrize("misses", [1, 4]) +@pytest.mark.parametrize("refreshed", ["refreshed", None], ids=["sign-in-retained", "signed-out"]) +@pytest.mark.parametrize("later_poll", [False, True], ids=["initial-poll", "later-poll"]) +def test_a_release_watch_transport_retry_preserves_the_following_auth_refusal( + monkeypatch, misses, refreshed, later_poll +) -> None: + from comfy_cli.command import build + + class TimeoutThenRefuse(_AnswersTheCut): + remaining = misses + pending_building = later_poll + + def __call__(self, url, target, **kwargs): + if kwargs.get("method") != "POST" and self.pending_building: + self.pending_building = False + return 200, {"id": "r1", "status": "building"} + if kwargs.get("method") != "POST" and self.remaining: + self.remaining -= 1 + raise TimeoutError("timed out") + return super().__call__(url, target, **kwargs) + + server = TimeoutThenRefuse(valid=set()) + sessions = _Sessions(first="signed-in", after=refreshed) + sleeps = [] + monkeypatch.setattr(build.time, "sleep", sleeps.append) + + result, error = _watched_cut(monkeypatch, server, sessions) + + assert result.exit_code == (75 if refreshed else 1), result.output + assert error["code"] == ("watch_auth_lost" if refreshed else "build_not_signed_in") + assert sleeps == ([build._RELEASE_POLL_SECONDS] if later_poll else []) + [4.0, 8.0, 16.0, 30.0][:misses] + assert server.tokens == (["signed-in", "signed-in", "refreshed"] if refreshed else ["signed-in", "signed-in"]) + assert sessions.forced == 1 + assert sessions.cleared is (refreshed is None) + assert server.remaining == 0 + if refreshed: + assert error["details"] == {"releaseId": "r1"} + assert "may still be building" in error["message"] + assert "comfy build release show r1" in error["hint"] + assert "comfy cloud login" not in error["hint"] + + +def test_a_release_watch_whose_refresh_signed_the_user_out_still_says_to_sign_in(monkeypatch) -> None: + # Given a watch refused, and a refresh that cleared the sign-in + server = _AnswersTheCut(valid=set()) + sessions = _Sessions(first="signed-in", after=None) + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then nobody is signed in any more, so signing in is the move + assert result.exit_code == 1 + assert error["code"] == "build_not_signed_in" + assert "comfy cloud login" in error["hint"] + + +def test_a_cut_refused_before_anything_was_made_keeps_the_sign_in_code(monkeypatch) -> None: + # Given a builder that refuses the cut itself, with the sign-in still stored + server = _Server(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then nothing is running, so the refusal is the sign-in's + assert result.exit_code == 1 + assert error["code"] == "build_not_signed_in" + assert server.tokens == ["signed-in", "refreshed"] + + +def test_a_deploy_watch_refused_after_the_refresh_ends_the_watch_not_the_deployment(monkeypatch) -> None: + # Given + from comfy_cli.command.deploy_runtime import DeployWatchAuthLostError, poll_deployment + + server = _Server(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + _install(monkeypatch, server, sessions) + sleeps: list[float] = [] + + # When + with pytest.raises(DeployWatchAuthLostError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", sleeps.append) + + # Then the read was refused after one refresh and one retry, and never retried again + assert exc_info.value.deployment_id == "dep-1" + assert exc_info.value.code == "watch_auth_lost" + assert server.tokens == ["signed-in", "refreshed"] + assert sleeps == [] + + +def test_a_watch_with_an_unrefreshable_stored_token_requires_sign_in(monkeypatch) -> None: + from comfy_cli.command.deploy_runtime import poll_deployment + + # A token without a refresh token can remain on disk after its lifetime ends. + session = SimpleNamespace(access_token="expired", refresh_token=None, base_url="https://cloud.comfy.org") + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: session) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", _Server(valid=set())) + + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", lambda _: None) + + assert exc_info.value.code == "deploy_not_signed_in" + + +class _SignedInElsewhere(_Sessions): + """A sign-in to another environment, stored while the watch ran.""" + + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): + session = super().__call__(refresh=refresh, force=force, allow_clear=allow_clear) + if self.forced and not force: + return SimpleNamespace(access_token=self.after, refresh_token="rt", base_url="https://staging.comfy.test") + return session + + +@pytest.mark.parametrize( + "sessions", + [_Sessions(first="signed-in", after=None), _SignedInElsewhere(first="signed-in", after="staging")], + ids=["refresh_signed_the_user_out", "signed_in_to_another_environment"], +) +def test_a_deploy_watch_with_no_sign_in_left_to_keep_says_to_sign_in(monkeypatch, sessions) -> None: + # Given + from comfy_cli.command.deploy_runtime import poll_deployment + + _install(monkeypatch, _Server(valid=set()), sessions) + + # When + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", lambda _: None) + + # Then + assert exc_info.value.code == "deploy_not_signed_in" + + +@pytest.mark.parametrize( + "build", + [ + lambda: DeployClient(_DEPLOY_URL, "injected"), + lambda: DeployClient( + _DEPLOY_URL, api_key=Credential(kind="api_key", value="k", source="env:COMFY_CLOUD_API_KEY") + ), + ], + ids=["handed_token", "workspace_key"], +) +def test_a_deploy_watch_on_a_credential_it_cannot_refresh_keeps_the_sign_in_code(monkeypatch, build) -> None: + # Given a sign-in stored beside a client that was never built from it + from comfy_cli.command.deploy_runtime import poll_deployment + + server = _Server(valid=set()) + _install(monkeypatch, server, _Sessions(first="signed-in", after="refreshed")) + + # When + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(build(), "dep-1", lambda _: None) + + # Then the credential the client sent is the one to replace + assert exc_info.value.code == "deploy_not_signed_in" + + +@pytest.fixture +def peer_takes_refresh_lock_after_first_read(monkeypatch, tmp_path): + """Own a real secrets lock; compress only its contended acquisition attempts.""" + from comfy_cli import locking + from comfy_cli.auth import store + from comfy_cli.cloud import oauth + + monkeypatch.setenv("COMFY_SECRETS_PATH", str(tmp_path / "secrets.json")) + store.save_cloud_session( + base_url="https://cloud.comfy.org", + resource="https://cloud.comfy.org/api", + client_id="cli", + scope="s", + access_token="signed-in", + refresh_token="RT0", + token_type="Bearer", + expires_at=4102444800, + ) + initial = store.secrets_path().read_bytes() + state = SimpleNamespace(armed=False, started=False, contended=False, timeouts=[], errors=[]) + held, release = threading.Event(), threading.Event() + real_acquire, real_read = locking._acquire, store.get_cloud_session + lock_stat = store.lock_path().stat() + + def peer(): + try: + with locking.file_lock(store.lock_path(), timeout=0): + state.contended = True + held.set() + if not release.wait(5): + raise AssertionError("test did not release its owned secrets lock") + state.contended = False + except BaseException as error: + state.errors.append(error) + held.set() + + owner = threading.Thread(target=peer) + + def acquire(fd, *, timeout): + info = os.fstat(fd) + owns_path = (info.st_dev, info.st_ino) == (lock_stat.st_dev, lock_stat.st_ino) + if owns_path and state.contended: + state.timeouts.append(timeout) + if timeout is None: + pytest.fail("unbounded secrets read after the refresh lock timed out") + # Exercise the actual contended OS lock without waiting out the + # production refresh budget. Classification already requests zero. + return real_acquire(fd, timeout=0) + return real_acquire(fd, timeout=timeout) + + def read(): + session = real_read() + if state.armed and not state.started: + state.started = True + owner.start() + assert held.wait(5), "test peer never acquired its secrets lock" + assert not state.errors + return session + + monkeypatch.setattr(locking, "_acquire", acquire) + monkeypatch.setattr(store, "get_cloud_session", read) + monkeypatch.setattr(oauth, "refresh_tokens", lambda **_: pytest.fail("must not refresh outside the held lock")) + + def release_peer(): + release.set() + if owner.ident is not None: + # Acquisition was nonblocking; the only possible wait is our own + # event, released above even when a regression assertion fails. + owner.join() + assert not state.errors + + state.release_peer = release_peer + try: + yield state + finally: + release_peer() + assert store.secrets_path().read_bytes() == initial + + +@pytest.mark.parametrize("kind", ["builder", "deploy"]) +def test_a_watch_does_not_wait_again_when_a_peer_outlives_the_refresh_lock( + monkeypatch, capsys, peer_takes_refresh_lock_after_first_read, kind +): + import json + + import typer + + from comfy_cli import credentials + from comfy_cli.caller import Caller + from comfy_cli.command.build import _watched_release + from comfy_cli.command.deploy_runtime import DeployWatchAuthLostError, exit_watch_lost, poll_deployment + from comfy_cli.output.renderer import Renderer + + state = peer_takes_refresh_lock_after_first_read + client = BuilderClient.from_session(_BUILDER_URL) if kind == "builder" else DeployClient.from_session(_DEPLOY_URL) + requests = [] + + def refused(url, target, **kwargs): + requests.append(target.auth_token) + state.armed = True + raise urllib.error.HTTPError(url, 401, "refused", http.client.HTTPMessage(), io.BytesIO(b"{}")) + + monkeypatch.setattr("comfy_cli.builder_api.request_json", refused) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", refused) + renderer = Renderer.resolve( + is_stdout_tty=False, env={}, caller=Caller(kind="user", agentic=False, source_env=None), json_flag=True + ) + with pytest.raises(typer.Exit) as stopped: + if kind == "builder": + _watched_release(renderer, client, "r1") + else: + try: + poll_deployment(client, "dep-1", lambda _: pytest.fail("must not sleep after the refused read")) + except DeployWatchAuthLostError as error: + exit_watch_lost(renderer, error) + + envelope = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert stopped.value.exit_code == 75 + assert envelope["error"]["code"] == "watch_auth_lost" + assert "comfy cloud login" not in envelope["error"]["hint"] + assert requests == ["signed-in"] + assert state.timeouts == [90.0, 0] + assert state.started is True + + # With the same real store readable again, the nested lock is reentrant. + state.release_peer() + assert credentials.sign_in_still_stored("https://cloud.comfy.org") is True + assert credentials.sign_in_still_stored(None) is False From c63690e6febd65d46daf8516e7f6ed51628b1739 Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Wed, 4 Nov 2026 19:55:08 -0500 Subject: [PATCH 2/3] docs(adr): AUTH-0007 bounds only the refresh-lock wait The first store read in the shared refresh takes the secrets lock with no timeout, as before this change. Say so in the decision, the rejected list and Known limits instead of promising a bounded wait on every path after a 401. --- ...jected-sign-in-once-and-keep-watches-alive.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md b/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md index 06a00af8c..24b9aeb26 100644 --- a/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md +++ b/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md @@ -44,8 +44,11 @@ must keep. They belong in an ADR, not only in code comments. ### One owner of the token family at a time - Only a process holding the cross-process secrets lock - (`auth_store.lock_path()`) may refresh or write the stored session. The lock - wait is bounded at 90 seconds (`_REFRESH_LOCK_TIMEOUT_S`). + (`auth_store.lock_path()`) may refresh or write the stored session. The wait + for the lock that guards the refresh is bounded at 90 seconds + (`_REFRESH_LOCK_TIMEOUT_S`). The first read of the store, before a refresh + is decided (`auth_store.get_cloud_session()`), takes the same lock with no + timeout, as it did before this decision; see Known limits. - Under the lock the process re-reads the store and decides from what is stored now, never from what it read before locking. It never sends a refresh token it observed before taking the lock if the store has since @@ -151,8 +154,8 @@ or a handed token, or a 401 with no recoverable sign-in left. token. - Clearing the stored session on a refresh failure that does not mean the family is dead, or from a watch that ends with `watch_auth_lost`. -- Waiting on the secrets lock with no timeout on any path that runs after a - 401. +- Waiting with no timeout for the lock that guards the refresh, on any path + that runs after a 401. The first store read is not covered; see Known limits. - Exit 1 for a watch that ended while the server-side work may still be running. - Reusing `watch_auth_lost` for a failure before the server accepted the work. @@ -183,6 +186,11 @@ or a handed token, or a 401 with no recoverable sign-in left. - The lock is a file lock on one machine. Two machines sharing one stored session (a synced home directory) are not coordinated. +- Only the refresh wait is bounded. The first store read in the shared refresh + (`auth_store.get_cloud_session()`) waits for the secrets lock with no + timeout, so a peer that already holds the lock when recovery starts can + keep a request or a watch waiting until it lets go. This is unchanged from + before; bounding that read is a separate change to the secret store. - The tests cover real temporary credential storage and a real OS lock, with mocked HTTP. They do not cover a live OAuth server's reuse detection under concurrency. From 06fafbb7cb8464645e3c169594531cfa7eaf0d3e Mon Sep 17 00:00:00 2001 From: Vinh Trinh Date: Wed, 4 Nov 2026 20:00:51 -0500 Subject: [PATCH 3/3] docs(adr): rename AUTH-0007 to AUTHENTICATION-0007 so the id does not read as a ticket id --- ...7-recover-a-rejected-sign-in-once-and-keep-watches-alive.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/adr/{AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md => AUTHENTICATION-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md} (99%) diff --git a/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md b/docs/adr/AUTHENTICATION-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md similarity index 99% rename from docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md rename to docs/adr/AUTHENTICATION-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md index 24b9aeb26..af9ae4bab 100644 --- a/docs/adr/AUTH-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md +++ b/docs/adr/AUTHENTICATION-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md @@ -1,4 +1,4 @@ -# ADR-AUTH-0007: Recover a Rejected Sign-In Once, and Keep Watches Alive +# ADR-AUTHENTICATION-0007: Recover a Rejected Sign-In Once, and Keep Watches Alive Date: 2026-11-04