diff --git a/CHANGELOG.md b/CHANGELOG.md index 0084d81b5..cb5d95528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,8 @@ history. ### Fixed +- Parallel build and deploy requests rejected on an old access token now adopt a peer's refreshed token under the shared refresh lock instead of rotating it again. Each rejected request retries at most once. If session recovery still fails while a refreshable sign-in remains stored, the watch reports that the remote work may continue instead of telling you to sign in again. `comfy build release create --watch`, `comfy deploy up`, `promote`, `rollback` and `comfy deploy status --watch` now end with the new code `watch_auth_lost` and exit code 75, as `deploy_watch_lost` does. Its hint names the command that reads the release or deployment again. A 401 before anything was accepted, on a workspace API key, or when no refreshable sign-in remains still reports `build_not_signed_in` or `deploy_not_signed_in`. + - `comfy update` no longer dies with a `FileNotFoundError` traceback when the configured workspace directory does not exist (never installed, or deleted). It now fails with a `not_in_workspace` error that points at `comfy install`, diff --git a/comfy_cli/builder_api.py b/comfy_cli/builder_api.py index 6bad57a92..1d7cd0ea4 100644 --- a/comfy_cli/builder_api.py +++ b/comfy_cli/builder_api.py @@ -137,6 +137,7 @@ def __init__( # handed a token (COMFY_BUILDER_TOKEN, COMFY_CLOUD_AUTH_TOKEN) or a key keeps # it and lets a 401 surface. self._refreshes_on_401 = False + self._sign_in_base_url: str | None = None # What to replace once the builder refuses this client's credential. self._refused_hint = ( credentials.refused_key_hint(api_key) @@ -175,9 +176,15 @@ def from_session(cls, base_url: str) -> BuilderClient: raise BuilderAuthError("not signed in: run `comfy cloud login` or set COMFY_CLOUD_API_KEY") client = cls(base_url, session.access_token) client._refreshes_on_401 = True + client._sign_in_base_url = session.base_url client._refused_hint = "run `comfy cloud login` first" return client + @property + def sign_in_base_url(self) -> str | None: + """The environment of the stored sign-in this client's token came from; None for a key or a handed token.""" + return self._sign_in_base_url if self._refreshes_on_401 else None + def _send(self, url: str, **kwargs) -> tuple[int, dict | list | None]: """Every builder request goes through here, so a long wait survives its token expiring. diff --git a/comfy_cli/cloud/oauth.py b/comfy_cli/cloud/oauth.py index cb6029c62..68e4f7ed2 100644 --- a/comfy_cli/cloud/oauth.py +++ b/comfy_cli/cloud/oauth.py @@ -782,6 +782,7 @@ def ensure_fresh_session( force: bool = False, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ): """Return the stored cloud session, refreshing it first when the access token is expired (or within ``leeway_s`` of expiring) and a refresh token @@ -803,6 +804,10 @@ def ensure_fresh_session( ``expires_at`` recorded — still thinks it is valid. The refresh token is still required; without one this is a no-op. + ``rejected_access_token`` identifies the token that received the 401. If a + peer already replaced it, adopt that session instead of refreshing again. + This also covers a late 401 that arrives after the peer released the lock. + ``allow_clear=False`` keeps the stored session on disk even when a refresh hits a fatal token error (reuse-detection / ``invalid_grant``). Background, read-mostly callers (the detached ``comfy run`` watcher) pass this so a @@ -855,6 +860,7 @@ def ensure_fresh_session( force=force, allow_clear=allow_clear, expect=expect, + rejected_access_token=rejected_access_token, ) except TimeoutError: # A peer has held the lock longer than a refresh should take. Don't @@ -875,6 +881,7 @@ def _locked_refresh( force: bool, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ): """Decide → POST → persist, executed while holding the secrets lock. @@ -890,11 +897,25 @@ def _locked_refresh( if session is None or not session.refresh_token: return session + # The rejected request may have been in flight while a peer finished its + # refresh, before even our first store read. Don't rotate that peer's token + # again just because this client's older request was refused. + if ( + rejected_access_token is not None + and session.access_token != rejected_access_token + and session.expires_at is not None + and not session.is_expired(leeway_s=leeway_s) + ): + return session + # Double-check after acquiring the lock. A peer may have refreshed while we # blocked: if the stored refresh token rotated away from what we observed # before locking, the family already advanced — use it, never re-send the # consumed token (that is exactly what trips reuse-detection). - if session.refresh_token != observed_refresh_token: + # A usable reactive peer already returned above. Otherwise refresh its + # current family, never replaying the older token observed before locking. + # Ordinary proactive callers still coalesce on any peer rotation. + if session.refresh_token != observed_refresh_token and rejected_access_token is None: return session if not force and not session.is_expired(leeway_s=leeway_s): # Peer advanced the access token's expiry without us noticing a token diff --git a/comfy_cli/command/build.py b/comfy_cli/command/build.py index 695a3df9e..147ff225c 100644 --- a/comfy_cli/command/build.py +++ b/comfy_cli/command/build.py @@ -3139,6 +3139,40 @@ def _selected_release_id(renderer, client, scope: _BuildScope, release: str | No return _newest_release_id(renderer, client, _resolve_build_id(renderer, client, scope)) +#: The exit a watch that lost its sign-in ends with, as `comfy deploy`'s watches do +#: (``deploy_runtime.EXIT_WATCH_LOST``): only the watch ended, so it must not exit 1, +#: which says the release failed. 75 is EX_TEMPFAIL in sysexits.h. +_EXIT_WATCH_LOST: Final = 75 + + +def _watched_release(renderer, client, release_id: str) -> dict: + """One read of a release this command has already cut. + + The builder accepted the cut with this sign-in, so a 401 session recovery did + not clear ends the watch while a refreshable sign-in remains stored. The + release may continue remotely. A key, a handed token, or a sign-in without + a refresh token keeps ``build_not_signed_in``. + """ + from comfy_cli import credentials + from comfy_cli.builder_api import BuilderCredentialRefused + + try: + return client.get_release(release_id) + except BuilderCredentialRefused as error: + if not credentials.sign_in_still_stored(getattr(client, "sign_in_base_url", None)): + raise + renderer.error( + code="watch_auth_lost", + message=f"the builder refused the sign-in (401) after attempting session recovery; " + f"release {release_id} may still be building on the server", + hint=f"read it with `comfy build release show {release_id}` until its status is complete; " + "the stored sign-in was preserved", + details={"releaseId": release_id}, + exit_code=_EXIT_WATCH_LOST, + ) + raise typer.Exit(code=_EXIT_WATCH_LOST) from error + + def _poll_call(renderer, fn, usable=None): """``_builder_call`` for one poll of a watch, which a single timed-out or dropped request must not end: the release keeps building either way. A @@ -3196,7 +3230,7 @@ def _poll_status(renderer, client, release_id: str) -> dict: word and ends the watch at the caller.""" return _poll_call( renderer, - lambda: client.get_release(release_id), + lambda: _watched_release(renderer, client, release_id), usable=lambda release: isinstance(release, dict) and isinstance(release.get("status"), str), ) diff --git a/comfy_cli/command/deploy_runtime.py b/comfy_cli/command/deploy_runtime.py index 9f477cf51..13759d5b6 100644 --- a/comfy_cli/command/deploy_runtime.py +++ b/comfy_cli/command/deploy_runtime.py @@ -12,6 +12,7 @@ import typer from typing_extensions import assert_never +from comfy_cli import credentials from comfy_cli.builder_api import BuilderClient, BuilderCredentialRefused from comfy_cli.command.build import DEFAULT_BUILDER_URL from comfy_cli.command.build_paths import BuildSpecNotFoundError, resolve_build_paths @@ -121,6 +122,8 @@ def terminal_status_error(deployment_id: str, status: str) -> JsonObject: class DeployWatchLostError(Exception): """The deploy API left a watch's reads unanswered for the whole retry window.""" + code = "deploy_watch_lost" + def __init__(self, deployment_id: str, cause: Exception | None = None, *, message: str | None = None) -> None: super().__init__( message or f"the deploy API stopped answering while following deployment {deployment_id} ({cause})" @@ -128,6 +131,23 @@ def __init__(self, deployment_id: str, cause: Exception | None = None, *, messag self.deployment_id = deployment_id +class DeployWatchAuthLostError(DeployWatchLostError): + """The deploy API refused a watch's read while a refreshable sign-in remains stored. + + The client has already attempted shared session recovery and at most one + retry. The deployment may continue remotely, so it is the watch that ended. + """ + + code = "watch_auth_lost" + + def __init__(self, deployment_id: str) -> None: + super().__init__( + deployment_id, + message=f"the deploy API refused the sign-in (401) while following deployment {deployment_id}, " + "after attempting session recovery", + ) + + def _unanswered(error: Exception) -> bool: """A read the service never answered: a 5xx, or no response at all. @@ -139,12 +159,26 @@ def _unanswered(error: Exception) -> bool: return error.code == "deploy_server_error" and (error.status is None or error.status >= 500) +def _sign_in_refused(client: DeployUpClient, error: Exception) -> bool: + """A 401 on a watch's read, from a client whose refreshable sign-in remains. + + A key, a handed token, or a sign-in without a refresh token keeps the + sign-in code, because changing that credential is the fix. + """ + if not isinstance(error, DeployAPIError) or error.status != 401: + return False + return credentials.sign_in_still_stored(getattr(client, "sign_in_base_url", None)) + + def exit_watch_lost(renderer: Renderer, error: DeployWatchLostError, hint: str | None = None) -> NoReturn: """Say it is the watch that gave out, not the deployment, and how to pick it up again.""" + hint = hint or f"run `{reattach_hint(error.deployment_id)}` to watch it again" + if isinstance(error, DeployWatchAuthLostError): + hint = f"{hint}; the stored sign-in was preserved" renderer.error( - code="deploy_watch_lost", + code=error.code, message=f"{error}; the deployment may still be coming up", - hint=hint or f"run `{reattach_hint(error.deployment_id)}` to watch it again", + hint=hint, details={"deployment_id": error.deployment_id}, exit_code=EXIT_WATCH_LOST, ) @@ -172,7 +206,9 @@ def poll_deployment( The progress a watcher shows rides the same read the loop already makes, so watching costs the service nothing it was not already answering. A read the service does not answer is retried, and the first of each run of them goes to - ``on_unanswered``; past the window this raises ``DeployWatchLostError``. + ``on_unanswered``; past the window this raises ``DeployWatchLostError``. A + 401 session recovery did not clear, with a refreshable sign-in still stored, + raises ``DeployWatchAuthLostError`` at once. """ failures, since, started = 0, 0.0, clock() while True: @@ -186,6 +222,8 @@ def poll_deployment( # The client maps a timeout, but a connection dropped before the status # line or partway through a 200's body escapes it unmapped. except (DeployAPIError, ConnectionError, http.client.HTTPException) as error: + if _sign_in_refused(client, error): + raise DeployWatchAuthLostError(deployment_id) from error if not _unanswered(error): raise if failures == 0: diff --git a/comfy_cli/credentials.py b/comfy_cli/credentials.py index 9cf75efd4..bcba23d50 100644 --- a/comfy_cli/credentials.py +++ b/comfy_cli/credentials.py @@ -92,6 +92,7 @@ def get_session( force: bool = False, allow_clear: bool = True, expect: Callable[[CloudSession], bool] | None = None, + rejected_access_token: str | None = None, ) -> CloudSession | None: """Read the stored Comfy Cloud OAuth session. @@ -104,6 +105,8 @@ def get_session( ``force=True`` (implies the refresh path) refreshes unconditionally, ignoring the local expiry check. Reserved for the *reactive* path: after a server 401, the token is known-rejected even if our clock disagrees. + ``rejected_access_token`` lets that path adopt a peer's newer access token + instead of rotating it again; the comparison happens under the refresh lock. ``allow_clear=False`` forwards to ``ensure_fresh_session`` so a fatal refresh failure does NOT clear the stored session. Background watchers pass @@ -117,9 +120,12 @@ def get_session( if refresh or force: from comfy_cli.cloud import oauth + options = {} if expect is not None: - return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear, expect=expect) - return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear) + options["expect"] = expect + if rejected_access_token is not None: + options["rejected_access_token"] = rejected_access_token + return oauth.ensure_fresh_session(force=force, allow_clear=allow_clear, **options) from comfy_cli.auth import store as auth_store return auth_store.get_cloud_session() @@ -140,14 +146,52 @@ def refreshed_access_token( :func:`get_session`. """ if expect is not None: - session = get_session(refresh=True, force=True, allow_clear=allow_clear, expect=expect) + session = get_session( + refresh=True, force=True, allow_clear=allow_clear, expect=expect, rejected_access_token=rejected + ) else: - session = get_session(refresh=True, force=True, allow_clear=allow_clear) + session = get_session(refresh=True, force=True, allow_clear=allow_clear, rejected_access_token=rejected) if session is None or not session.access_token or session.access_token == rejected: return None return session.access_token +def sign_in_still_stored(base_url: str | None) -> bool: + """Whether a refreshable sign-in for the refused client's environment remains. + + A 401 that outlives :func:`refreshed_access_token` and one retry does not show + that nobody is signed in: a peer process can hold the refresh lock past this + one's timeout, a refresh that fails on the network keeps the old token, and + the server can refuse a token it minted a moment ago. A sign-in without a + refresh token cannot recover this way and retains the sign-in guidance. + ``base_url`` is the environment the client's sign-in came from; ``None``, + for a key or a handed token, never matches. + """ + if base_url is None: + return False + from comfy_cli import locking + from comfy_cli.auth import store as auth_store + + try: + # Do not spend another lock budget after session recovery timed out. + # The store's nested read reuses this thread's lock reentrantly. + with locking.file_lock(auth_store.lock_path(), timeout=0): + session = get_session(refresh=False) + except TimeoutError: + # Contention cannot prove the held sign-in disappeared. End only the + # watch, rather than hanging or prescribing another login. + return True + except OSError as error: + _log.warning("could not re-read the stored sign-in after a 401: %s", error) + return False + return ( + session is not None + and bool(session.access_token) + and bool(session.refresh_token) + and session.base_url == base_url + ) + + class DataPlaneToken: """The bearer one long command sends to a deployment endpoint, kept fresh. diff --git a/comfy_cli/error_codes.py b/comfy_cli/error_codes.py index d402b1332..66ed508aa 100644 --- a/comfy_cli/error_codes.py +++ b/comfy_cli/error_codes.py @@ -1760,6 +1760,19 @@ class ErrorCode: "after a move, read `comfy deploy show --deployment ` instead: the move has landed once it shows no " "pendingUpdate and its releaseId is the release asked for", ), + ErrorCode( + "watch_auth_lost", + "A watch over work the server had already accepted (`comfy build release create --watch`, `comfy deploy " + "up`, `promote` and `rollback`, `comfy deploy status --watch`) was refused with HTTP 401 despite " + "serialized session recovery and at most one retry, while a refreshable sign-in remains stored. " + "Only the watch ended: the release may still be building and the deployment may still be coming up. " + "The exit code is 75, not 1. `details.releaseId` (build) or " + "`details.deployment_id` (deploy) names what is still running. A 401 before anything was accepted, on a " + "workspace API key, or with no refreshable sign-in remaining stays `build_not_signed_in` or " + "`deploy_not_signed_in`.", + "the stored sign-in was preserved: re-read the work with `comfy build " + "release show ` or `comfy deploy status --deployment --watch`", + ), ErrorCode( "deploy_update_failed", "`comfy deploy up`, `promote` or `rollback` moved a deployment onto another release, and the watch saw " diff --git a/comfy_cli/skills/comfy-build/SKILL.md b/comfy_cli/skills/comfy-build/SKILL.md index 83a2e85ce..57758174a 100644 --- a/comfy_cli/skills/comfy-build/SKILL.md +++ b/comfy_cli/skills/comfy-build/SKILL.md @@ -76,7 +76,10 @@ blob ls (hidden; workspace private blobs that reaches the builder need it; a plain scan and a plain `validate` do not. A CI job sets a workspace API key in `COMFY_CLOUD_API_KEY` instead, and that key wins over a stored sign-in, so when the hint says that key was refused, - signing in cannot help: replace the key. + signing in cannot help: replace the key. `watch_auth_lost` (exit 75) means + `release create --watch` was refused after the cut despite session recovery, + while a refreshable sign-in remains stored. The release may still be building. + Read it with `comfy build release show ` before considering a new sign-in. On `build_not_enabled` the platform is in limited beta and this account is not enabled — stop and say so. diff --git a/comfy_cli/skills/comfy-deploy-failures/SKILL.md b/comfy_cli/skills/comfy-deploy-failures/SKILL.md index cb44d581b..f0f68c1dc 100644 --- a/comfy_cli/skills/comfy-deploy-failures/SKILL.md +++ b/comfy_cli/skills/comfy-deploy-failures/SKILL.md @@ -55,6 +55,7 @@ argument you pass. | `deploy_update_replaced` | A watched `up`, `promote` or `rollback` was replaced by a newer update; `details.replacing_release_id` names its release | Follow it with `comfy deploy status --deployment `, or end it with `comfy deploy cancel --deployment ` | | `deploy_updates_unavailable` | `promote`, `rollback`, `history`, `cancel` or `events --release` on a deployment with no `revision`, or `cancel` against a deploy service too old for it | `promote` or `rollback`: `up --create` on the release instead. `history`: read `events`. `cancel`: wait for the update to land or fail. `events`: drop `--release`; those events do not say which release made them | | `deploy_watch_lost` | The watch's reads went unanswered for a minute, or a move by `up`, `promote` or `rollback` did not land within an hour (exit 75); the deployment may still be coming up | `comfy deploy status --deployment --watch`, or after a move `comfy deploy show --deployment ` until it shows no `pendingUpdate`; do not redeploy | +| `watch_auth_lost` | A watch's read was refused with a 401 despite session recovery, and a refreshable sign-in remains stored (exit 75); the deployment may still be coming up | Reattach as for `deploy_watch_lost` before considering a new sign-in; do not redeploy | | `deploy_delete_needs_confirm` | `delete` without `--yes` non-interactively | Confirm with the user, then pass `--yes` | ### Submitting a workflow diff --git a/comfy_cli/skills/comfy-deploy/SKILL.md b/comfy_cli/skills/comfy-deploy/SKILL.md index 46893eceb..df9b5dbab 100644 --- a/comfy_cli/skills/comfy-deploy/SKILL.md +++ b/comfy_cli/skills/comfy-deploy/SKILL.md @@ -176,6 +176,10 @@ stops answering (a 5xx or a dropped connection, as during its own rollout) is retried for about a minute; if it is still down, the wait ends with `deploy_watch_lost` and **exit 75**, not 1. That is not a failed deployment: attach again with the command in the hint rather than redeploying. +`watch_auth_lost`, also exit 75, is a read refused with a 401 despite session +recovery, while a refreshable sign-in remains stored. Parallel requests adopt a +peer's refreshed token and retry at most once. Attach again the same way before +considering a new sign-in; the deployment may still be coming up. `status` also reports **why** a deployment stopped, as `stopReason`: `user`, `credits`, or `policy`. `credits` is a billing problem and not something a retry diff --git a/docs/adr/AUTHENTICATION-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md b/docs/adr/AUTHENTICATION-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md new file mode 100644 index 000000000..af9ae4bab --- /dev/null +++ b/docs/adr/AUTHENTICATION-0007-recover-a-rejected-sign-in-once-and-keep-watches-alive.md @@ -0,0 +1,225 @@ +# ADR-AUTHENTICATION-0007: Recover a Rejected Sign-In Once, and Keep Watches Alive + +Date: 2026-11-04 + +## Status + +Proposed + +Relation: Builds on [ADR-AUTOMATION-0003](AUTOMATION-0003-one-versioned-machine-output-dialect.md). +One envelope per command, append-only error codes and human diagnostics off +stdout all stay as they are. This ADR adds one error code, `watch_auth_lost`, +and fixes what it means. + +## Context + +A Comfy Cloud sign-in is one stored OAuth session: an access token that lasts +about fifteen minutes and a refresh token that rotates on every use. The +server tracks each refresh token as part of a family. Sending a refresh token +that was already used is treated as theft: the server kills the whole family, +and the user has to sign in again. + +Several `comfy` processes can share that one stored session at once: a +`comfy build release create --watch` in one terminal, a `comfy deploy status +--watch` in another, an agent running `comfy deploy up` in a third. Each of +them can get a 401 when its access token expires mid-wait. Before this +decision, two failure modes followed from that: + +- Two processes rejected on the same old access token could both refresh. The + second one either replayed a consumed refresh token (and killed the family) + or rotated the token the first had just stored, so every peer refreshed in + turn. +- A watch over work the server had already accepted (a release that is + building, a deployment that is coming up) ended with `build_not_signed_in` + or `deploy_not_signed_in` and exit 1 when its recovery failed. That told the + user to sign in again and suggested the work had failed, while the release + or deployment was still running, and the stored sign-in was often fine. + +Who owns the token family, when a process may refresh it, and what a watch +reports when it loses its sign-in are contracts that later auth or watch code +must keep. They belong in an ADR, not only in code comments. + +## Decision + +### One owner of the token family at a time + +- Only a process holding the cross-process secrets lock + (`auth_store.lock_path()`) may refresh or write the stored session. The wait + for the lock that guards the refresh is bounded at 90 seconds + (`_REFRESH_LOCK_TIMEOUT_S`). The first read of the store, before a refresh + is decided (`auth_store.get_cloud_session()`), takes the same lock with no + timeout, as it did before this decision; see Known limits. +- Under the lock the process re-reads the store and decides from what is + stored now, never from what it read before locking. It never sends a + refresh token it observed before taking the lock if the store has since + moved on. +- If the lock wait times out, the process does not refresh and does not + re-read the store. It returns the session it held before locking and + leaves the stored session untouched. + +### What a process rejected with a 401 does + +A request refused with a 401 passes the access token that was refused +(`rejected_access_token`) into the shared refresh. Under the lock: + +1. **Adopt a usable peer session.** If the stored access token differs from + the rejected one and has a known, unexpired `expires_at` (outside the usual + leeway), a peer already refreshed. Use that session and do not refresh. +2. **Otherwise refresh the current family.** A stored token with no + `expires_at`, an expired one, or the rejected token itself is not usable. + The process refreshes with the refresh token stored now. A token whose expiry + is unknown is never adopted, because the one retry would just get another 401. +3. **Proactive callers** (a process refreshing before expiry, with no 401) keep + the earlier rule: any peer rotation since their pre-lock read means the + family already moved, and they use the stored session. + +### When the stored session is cleared + +- Only an error that means the family is dead (`invalid_grant`, + `invalid_token`, or a reuse-detection signal) clears the stored session, and + only if the caller did not pass `allow_clear=False`. Background read-mostly + callers, such as the detached `comfy run` watcher, pass that. +- Any other refresh failure (network, 5xx, timeout, `invalid_client`) never + clears it. The old session stays and the caller's request surfaces its 401. + +### One retry per rejected request + +`BuilderClient._send` and the deploy API client retry a 401 at most once, and +only for a client built from the stored sign-in. The order is: 401, shared +recovery, one retry with the new token, and if that is refused too, the +command reports the refusal. A client handed a token (`COMFY_BUILDER_TOKEN`, +`COMFY_CLOUD_AUTH_TOKEN`) or a workspace API key never refreshes and never +retries. The server refuses a 401 before acting, so the retry is safe for +every method; a retried create carries the same idempotency key. + +### What a watch reports when it loses its sign-in + +A watch that reads work the server already accepted ends with `watch_auth_lost` +and exit code **75** (`EX_TEMPFAIL`, the same exit as `deploy_watch_lost`), +not 1, when all of these hold: + +- the read got a 401 after shared recovery and the one retry; +- the client was built from the stored sign-in (it has a `sign_in_base_url`); +- a sign-in that can still recover remains stored: it has an access token, a + refresh token, and the same `base_url` as the client's. + +That re-read (`credentials.sign_in_still_stored`) takes the secrets lock with +`timeout=0`. If a peer holds the lock, the answer is "still stored": being +locked out does not prove the sign-in is gone, and the watch must not hang +for another lock budget after recovery already timed out. If the store +cannot be read at all (`OSError`), the answer is "not stored". + +The envelope says only the watch ended: the message says the release may still +be building or the deployment may still be coming up. `details.releaseId` +(build) or `details.deployment_id` (deploy) names the work. The hint names the +command that reads it again (`comfy build release show `, or `comfy deploy +status --deployment --watch`) and says the stored sign-in was preserved. +The watch never clears the stored sign-in on this path. + +Every other 401 keeps its sign-in code and its "sign in again" or "replace +the key" hint: a 401 before the server accepted anything, a 401 on an API key +or a handed token, or a 401 with no recoverable sign-in left. +`build_not_signed_in` and `deploy_not_signed_in` keep their meaning. + +## Alternatives considered + +- **Always refresh after a 401.** This is the simplest rule, and it is the bug: + N processes rejected on the same old token rotate the family N times, and a + late one replays a consumed token and kills the family. +- **Adopt any stored token that differs from the rejected one.** Rejected + because a peer token with no recorded expiry, or one that has since expired, + gets a second 401 on the only retry. This case was raised in review and is + handled by the expiry check in rule 1. +- **Retry until a request succeeds.** Rejected because a server that keeps + refusing a valid-looking token would loop forever, and each loop spends a + refresh. One retry covers the real case, a token that expired mid-wait. +- **Wait on the lock again to re-read the store before classifying a watch + 401.** Rejected because after a 90 second timeout a second wait can hang the + watch for as long again, or forever with no timeout. A non-blocking read + answers the only question the classification needs. +- **Reuse `deploy_watch_lost`, or add a separate code for build.** Rejected + because `deploy_watch_lost` means the service stopped answering, which needs a + different next step. One `watch_auth_lost` covers both build and deploy + watches, with the same exit code. +- **Keep `build_not_signed_in` / exit 1 for a lost watch.** Rejected because it + makes callers and agents resubmit work that is still running, and sign in + again when the stored sign-in is fine. + +## What we reject + +- Refreshing, or writing the stored session, outside the secrets lock. +- Sending a refresh token observed before the lock after the store has moved on. +- Treating a token with unknown expiry as usable for a reactive adoption. +- More than one retry per rejected request, or any retry for a key or a handed + token. +- Clearing the stored session on a refresh failure that does not mean the + family is dead, or from a watch that ends with `watch_auth_lost`. +- Waiting with no timeout for the lock that guards the refresh, on any path + that runs after a 401. The first store read is not covered; see Known limits. +- Exit 1 for a watch that ended while the server-side work may still be + running. +- Reusing `watch_auth_lost` for a failure before the server accepted the work. + +## Consequences + +### Positive + +- Parallel commands on one sign-in refresh once, not once each, and a late 401 + no longer kills the token family. +- A watch that loses its sign-in tells the truth: the work may still be + running, here is how to read it, and the sign-in was kept. Exit 75 lets + scripts tell "watch ended" from "work failed". +- Build and deploy watches report the same code and exit for the same + situation. + +### Negative + +- `watch_auth_lost` is a new public error code. Under ADR-AUTOMATION-0003 it + is append-only once released and cannot be renamed or repurposed. +- A watch can end with `watch_auth_lost` when a peer held the lock and the + sign-in was in fact about to be lost. The next command then reports the + sign-in code. That is the cost of not hanging. +- Callers that matched on `build_not_signed_in` / `deploy_not_signed_in` to + detect a lost watch now see a different code and exit 75. + +### Known limits + +- The lock is a file lock on one machine. Two machines sharing one stored + session (a synced home directory) are not coordinated. +- Only the refresh wait is bounded. The first store read in the shared refresh + (`auth_store.get_cloud_session()`) waits for the secrets lock with no + timeout, so a peer that already holds the lock when recovery starts can + keep a request or a watch waiting until it lets go. This is unchanged from + before; bounding that read is a separate change to the secret store. +- The tests cover real temporary credential storage and a real OS lock, with + mocked HTTP. They do not cover a live OAuth server's reuse detection under + concurrency. + +## Validation + +- `tests/comfy_cli/auth/test_oauth.py`: a rejected older token adopts a peer's + unexpired token; it does not adopt an expired peer token or one with no + expiry, and refreshes with the stored family instead. +- `tests/comfy_cli/test_session_refresh.py`: overlapping clients reuse the peer + refresh before retrying; one retry, then a second 401 surfaces; handed tokens + never swap; the retried create keeps its idempotency key; build and deploy + watches end as `watch_auth_lost` with the sign-in kept; a watch does not wait + again when a peer outlives the refresh lock; unrefreshable, signed-out and + pre-accept 401s keep the sign-in code. +- `tests/comfy_cli/command/test_deploy_up.py` and `test_deploy_status.py`: the + follow read's 401 with and without a stored sign-in, and a 401 on the create. +- The error-code registry tests keep `watch_auth_lost` registered and emitted + from the same places. + +## Revisit triggers + +- The server starts returning a distinct status or error for "token expired" + versus "token revoked". The adopt-or-refresh rule could then read it + directly. +- Access-token lifetime moves well past fifteen minutes, or the server stops + rotating refresh tokens. One retry and the family-ownership rule may then + be more than is needed. +- Field reports of `watch_auth_lost` followed by a work item that had in fact + failed, or of family-kill re-logins with this rule in place. +- Session storage moves off the local file store (for example to an OS + keychain with its own locking). diff --git a/docs/json-output.md b/docs/json-output.md index 523082a5f..ef463790b 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -572,6 +572,14 @@ watch ends with the error code `deploy_watch_lost` and exit code **75**, never 1 only the watch ended, and the deployment may still be coming up. Its `hint` carries the command that re-attaches and `details.deployment_id` the deployment. +A 401 on a watch's read triggers serialized session recovery: the client adopts +a peer's newer token or refreshes the rejected one, then retries at most once. +If recovery fails while a refreshable sign-in for the same environment remains +stored, the watch ends with `watch_auth_lost`, again with exit code **75** and +the same `details`. Its `hint` reattaches the watch without a re-login prompt. +The remote work may continue. A 401 with no refreshable sign-in remaining, or +on a workspace API key, stays `deploy_not_signed_in`. + ## Success envelope On `--wait` success, `data` carries: diff --git a/tests/comfy_cli/auth/test_oauth.py b/tests/comfy_cli/auth/test_oauth.py index c7064763b..bfdb821fb 100644 --- a/tests/comfy_cli/auth/test_oauth.py +++ b/tests/comfy_cli/auth/test_oauth.py @@ -703,7 +703,8 @@ def worker(): # (b) Double-check: if a peer already rotated the token between our pre-lock # read and the post-lock re-read, we adopt it and do NOT refresh again. - def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, monkeypatch): + @pytest.mark.parametrize("peer_expiry", [None, 1, 4102444800], ids=["unknown-expiry", "expired", "known-unexpired"]) + def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, monkeypatch, peer_expiry): self._persist_expired(refresh_token="RT0") # Simulate a peer that rotated RT0 → RT1 while we waited for the lock: @@ -716,7 +717,7 @@ def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, m access_token="PEER_NEW", refresh_token="RT1", token_type="Bearer", - expires_at=int(time.time()) + 3600, + expires_at=peer_expiry, saved_at="2026-01-01T00:00:02+00:00", ) reads = [self._stale(), fresh_peer] @@ -727,6 +728,65 @@ def test_double_check_skips_refresh_when_peer_already_rotated(self, persisted, m assert result.refresh_token == "RT1" assert result.access_token == "PEER_NEW" + def test_a_rejected_older_token_does_not_adopt_an_already_expired_peer_token(self, persisted, monkeypatch): + from comfy_cli.credentials import refreshed_access_token + + self._persist_expired() + monkeypatch.setattr(oauth, "refresh_tokens", lambda **_: self._token_set(access="NEW", refresh="RT1")) + + assert refreshed_access_token("BEFORE_OLD") == "NEW" + assert auth_store.get_cloud_session().refresh_token == "RT1" + + @pytest.mark.parametrize( + ("peer_access", "peer_expiry"), + [("PEER_UNUSABLE", None), ("PEER_UNUSABLE", 1), ("REJECTED_OLDER", 4102444800)], + ids=["unknown-expiry", "expired", "same-rejected-token"], + ) + @pytest.mark.parametrize("peer_timing", ["already-stored", "after-first-read"]) + def test_a_peer_token_that_cannot_be_retried_uses_its_available_refresh( + self, persisted, monkeypatch, peer_timing, peer_access, peer_expiry + ): + from comfy_cli.credentials import refreshed_access_token + + def save(access, refresh, expiry): + return auth_store.save_cloud_session( + base_url="https://c", + resource="https://c/api", + client_id="cid", + scope="s", + access_token=access, + refresh_token=refresh, + token_type="Bearer", + expires_at=expiry, + ) + + save("REJECTED_OLDER", "RT0", 4102444800) + if peer_timing == "already-stored": + save(peer_access, "RT1", peer_expiry) + else: + real_read = auth_store.get_cloud_session + + def read_then_peer_rotates(): + session = real_read() + if session.refresh_token == "RT0": + save(peer_access, "RT1", peer_expiry) + return session + + monkeypatch.setattr(auth_store, "get_cloud_session", read_then_peer_rotates) + refreshes = [] + + def refresh(**kwargs): + refreshes.append(kwargs["refresh_token"]) + return self._token_set(access="NEW", refresh="RT2") + + monkeypatch.setattr(oauth, "refresh_tokens", refresh) + + assert refreshed_access_token("REJECTED_OLDER") == "NEW" + stored = auth_store.get_cloud_session() + assert stored.access_token == "NEW" + assert stored.refresh_token == "RT2" + assert refreshes == ["RT1"] + def _stale(self) -> auth_store.CloudSession: return auth_store.CloudSession( base_url="https://c", diff --git a/tests/comfy_cli/command/test_assets_library.py b/tests/comfy_cli/command/test_assets_library.py index 6874443ab..68e9306e3 100644 --- a/tests/comfy_cli/command/test_assets_library.py +++ b/tests/comfy_cli/command/test_assets_library.py @@ -1162,7 +1162,7 @@ def signed_in(monkeypatch: pytest.MonkeyPatch): session=SimpleNamespace(access_token="old-token", base_url="https://cloud.example.com"), forced=0 ) - def get_session(*, refresh=True, force=False, allow_clear=True, expect=None): + def get_session(*, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: state.forced += 1 state.session = SimpleNamespace(access_token="new-token", base_url="https://cloud.example.com") @@ -1275,7 +1275,7 @@ def stored_login(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv(var, raising=False) state = SimpleNamespace(login=("a", 1), forced=0) - def get_session(*, refresh=True, force=False, allow_clear=True, expect=None): + def get_session(*, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: state.forced += 1 state.login = (state.login[0], state.login[1] + 1) diff --git a/tests/comfy_cli/command/test_deploy_run.py b/tests/comfy_cli/command/test_deploy_run.py index f2629f77f..c67768daf 100644 --- a/tests/comfy_cli/command/test_deploy_run.py +++ b/tests/comfy_cli/command/test_deploy_run.py @@ -613,7 +613,7 @@ class _RotatingSignIn: def __init__(self) -> None: self.token = "token-at-start" - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): return SimpleNamespace(access_token=self.token, base_url="https://cloud.comfy.org") @@ -673,7 +673,7 @@ def test_an_unreadable_sign_in_at_cancel_still_cancels_with_the_token_the_run_ho monkeypatch.setattr(deploy_run, "resolve_partner_credential", lambda: None) store = {"readable": True} - def get_session(*, refresh=True, force=False, allow_clear=True): + def get_session(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if not store["readable"]: raise OSError("auth store lock unavailable") return SimpleNamespace(access_token="cloud-secret", base_url="https://cloud.comfy.org") diff --git a/tests/comfy_cli/command/test_deploy_status.py b/tests/comfy_cli/command/test_deploy_status.py index 0076d559d..86b9d5a4d 100644 --- a/tests/comfy_cli/command/test_deploy_status.py +++ b/tests/comfy_cli/command/test_deploy_status.py @@ -4,6 +4,7 @@ import json from datetime import datetime, timedelta, timezone from pathlib import Path +from types import SimpleNamespace import jsonschema import pytest @@ -801,6 +802,34 @@ def get_deployment(self, deployment_id: str) -> JsonObject: assert len(client.get_calls) == 7 +def test_a_watch_refused_with_the_sign_in_still_stored_ends_as_watch_auth_lost(tmp_path, monkeypatch) -> None: + # Given a watch whose read is refused after the client's refresh and retry + class Refused(RecordingDeploy): + sign_in_base_url = "https://cloud.comfy.org" + + def get_deployment(self, deployment_id: str) -> JsonObject: + self.get_calls.append(deployment_id) + raise DeployAPIError("deploy_not_signed_in", "invalid auth token", status=401) + + client = Refused([_status_deployment(status="provisioning")]) + sleeps: list[float] = [] + _install_clients(monkeypatch, FakeBuilder(), client, sleeps) + stored = SimpleNamespace(access_token="token", refresh_token="rt", base_url="https://cloud.comfy.org") + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: stored) + + # When + result = _invoke_json(write_spec(tmp_path), "--watch") + + # Then + assert result.exit_code == 75, result.stderr + error = _json_envelope(result)["error"] + assert error["code"] == "watch_auth_lost" + assert "comfy deploy status --deployment dep-status --watch" in error["hint"] + assert "comfy cloud login" not in error["hint"] + assert client.get_calls == ["dep-status"] + assert sleeps == [] + + def test_a_watch_that_gives_up_takes_the_live_line_down_before_the_error_prints(tmp_path, monkeypatch) -> None: """An error printed while the live line is up lands under a spinner that is still turning.""" # Given a watch that draws its line, then never hears back. CliRunner's diff --git a/tests/comfy_cli/command/test_deploy_up.py b/tests/comfy_cli/command/test_deploy_up.py index d10e6927d..5df61ef8c 100644 --- a/tests/comfy_cli/command/test_deploy_up.py +++ b/tests/comfy_cli/command/test_deploy_up.py @@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace import jsonschema import pytest @@ -814,6 +814,93 @@ def test_a_4xx_on_the_follow_read_still_fails_at_once(tmp_path, monkeypatch) -> assert client.get_calls == 2 +_CLOUD = "https://cloud.comfy.org" + + +def _refused() -> DeployAPIError: + # What the client raises once a 401 outlives its forced refresh and its one retry. + return DeployAPIError( + "deploy_not_signed_in", + "invalid auth token", + status=401, + details={"operation": "get", "status": 401, "server_code": "UNAUTHORIZED"}, + ) + + +def _stored_sign_in(monkeypatch, base_url: str | None) -> None: + session = ( + SimpleNamespace(access_token="token", refresh_token="rt", base_url=base_url) if base_url is not None else None + ) + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: session) + + +def test_a_401_on_the_follow_read_with_the_sign_in_still_stored_ends_the_watch_not_the_deployment( + tmp_path, monkeypatch +) -> None: + """Parallel watches can each be refused after the shared refresh. The deployment + was accepted with that same sign-in and keeps coming up, so the envelope must + not send an agent to sign in, nor report a failure that did not happen.""" + # Given a deployment the service accepted, then a follow read it refuses + client = FlakyFollow([_refused()], get_statuses=["queued", "ready"]) + client.sign_in_base_url = _CLOUD # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, _CLOUD) + sleeps: list[float] = [] + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, sleeps) + + # Then + assert result.exit_code == 75, result.stderr + error = _json_envelope(result)["error"] + assert error["code"] == "watch_auth_lost" + assert "may still be coming up" in error["message"] + assert error["hint"].startswith("run `comfy deploy status --deployment dep-1 --watch` to watch it again") + assert "comfy cloud login" not in error["hint"] + assert error["details"] == {"deployment_id": "dep-1"} + assert sleeps == [] + assert len(client.create_keys) == 1 + + +@pytest.mark.parametrize( + ("client_sign_in", "stored_sign_in"), + [(_CLOUD, None), (_CLOUD, "https://staging.comfy.test"), (None, _CLOUD)], + ids=["refresh_signed_the_user_out", "signed_in_to_another_environment", "workspace_key"], +) +def test_a_401_on_the_follow_read_with_no_sign_in_to_keep_still_says_to_sign_in( + tmp_path, monkeypatch, client_sign_in, stored_sign_in +) -> None: + # Given a follow read refused, and no sign-in this watch's client could keep using + client = FlakyFollow([_refused()], get_statuses=["queued", "ready"]) + client.sign_in_base_url = client_sign_in # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, stored_sign_in) + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, []) + + # Then + assert result.exit_code == 1 + assert _json_envelope(result)["error"]["code"] == "deploy_not_signed_in" + + +class _RefusesTheCreate(FakeDeploy): + def create_deployment(self, release_id: str, compute_config: JsonObject, **kwargs) -> JsonObject: + raise _refused() + + +def test_a_401_on_the_create_keeps_the_sign_in_code(tmp_path, monkeypatch) -> None: + # Given a sign-in still stored, and a service that refuses the create itself + client = _RefusesTheCreate() + client.sign_in_base_url = _CLOUD # type: ignore[attr-defined] + _stored_sign_in(monkeypatch, _CLOUD) + + # When + result = _invoke_watched_up(tmp_path, monkeypatch, client, []) + + # Then nothing was accepted, so nothing runs on, and the refusal is the sign-in's + assert result.exit_code == 1 + assert _json_envelope(result)["error"]["code"] == "deploy_not_signed_in" + + def test_a_second_outage_in_one_watch_is_announced_and_backed_off_afresh(tmp_path, monkeypatch) -> None: """A read that is answered ends the outage, so the next one is new: it is announced again and gets the whole retry schedule, not what was left of the first.""" diff --git a/tests/comfy_cli/command/test_transfer_download.py b/tests/comfy_cli/command/test_transfer_download.py index 889515bf5..0dc887921 100644 --- a/tests/comfy_cli/command/test_transfer_download.py +++ b/tests/comfy_cli/command/test_transfer_download.py @@ -1428,7 +1428,7 @@ def __init__(self, base_url: str, first: str) -> None: self.token = first self.refreshing_reads: list[bool] = [] - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): from types import SimpleNamespace if refresh: diff --git a/tests/comfy_cli/test_data_plane_token.py b/tests/comfy_cli/test_data_plane_token.py index 9f138686c..82323c836 100644 --- a/tests/comfy_cli/test_data_plane_token.py +++ b/tests/comfy_cli/test_data_plane_token.py @@ -27,7 +27,7 @@ def __init__(self, proactive: str, forced: str | None) -> None: self.forced = forced self.forced_calls = 0 - def __call__(self, *, refresh=True, force=False, allow_clear=True, expect=None): + def __call__(self, *, refresh=True, force=False, allow_clear=True, expect=None, rejected_access_token=None): if force: self.forced_calls += 1 if not self.forced: @@ -181,7 +181,7 @@ def test_a_sign_in_to_another_environment_mid_run_is_never_sent_or_refreshed(mon refreshing: list[bool] = [] - def get_session(*, refresh=True, force=False, allow_clear=True): + def get_session(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if refresh or force: refreshing.append(allow_clear) return stored @@ -382,3 +382,29 @@ def get_session(**kwargs): # Then: the refresh is asked exactly as before, with no expectation attached assert calls and all("expect" not in kwargs for kwargs in calls) + + +def test_an_account_bound_holder_adopts_the_same_accounts_peer_refresh(login_race) -> None: + # A peer finishes its refresh while the held token's rejected request is in flight. + held = _save_login("a", expires_at=_LATER).access_token + token = DataPlaneToken(held, refreshes=True, base_url=_CLOUD, same_account=True) + peer = _jwt("a", 2) + from comfy_cli.auth import store + + stored = store.get_cloud_session() + store.save_cloud_session( + base_url=stored.base_url, + resource=stored.resource, + client_id=stored.client_id, + scope=stored.scope, + access_token=peer, + refresh_token="refresh-a-peer", + token_type="Bearer", + expires_at=_LATER, + ) + + assert token.after_rejection() is True + assert token.current() == peer + assert login_race.refreshed == [] + assert store.get_cloud_session().refresh_token == "refresh-a-peer" + assert token.account_replaced is False diff --git a/tests/comfy_cli/test_platform_api_key.py b/tests/comfy_cli/test_platform_api_key.py index 64a14f17d..3c6c2b1d8 100644 --- a/tests/comfy_cli/test_platform_api_key.py +++ b/tests/comfy_cli/test_platform_api_key.py @@ -41,7 +41,7 @@ def __call__(self, url, target, **kwargs): def _install(monkeypatch, server: _Server, key: Credential | None, sign_in: str | None = "signed-in-token") -> list: refreshes: list = [] - def sessions(*, refresh=True, force=False, allow_clear=True): + def sessions(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if force: refreshes.append(force) return SimpleNamespace(access_token=sign_in, base_url="https://cloud.comfy.org") if sign_in else None @@ -187,7 +187,7 @@ def test_a_refreshed_sign_in_the_builder_still_refuses_names_the_login(monkeypat server = _Server(status=401) refreshed = iter(["refreshed-token"]) - def sessions(*, refresh=True, force=False, allow_clear=True): + def sessions(*, refresh=True, force=False, allow_clear=True, rejected_access_token=None): return SimpleNamespace( access_token=next(refreshed) if force else "signed-in-token", base_url="https://cloud.comfy.org" ) diff --git a/tests/comfy_cli/test_session_refresh.py b/tests/comfy_cli/test_session_refresh.py index a0db03ba4..027d285bd 100644 --- a/tests/comfy_cli/test_session_refresh.py +++ b/tests/comfy_cli/test_session_refresh.py @@ -10,12 +10,16 @@ import http.client import io +import os +import threading +import time import urllib.error from types import SimpleNamespace import pytest from comfy_cli.builder_api import BuilderClient +from comfy_cli.credentials import Credential from comfy_cli.deploy_api import DeployAPIError, DeployClient _DEPLOY_URL = "https://deploy.test/deploy" @@ -38,18 +42,27 @@ def __call__(self, url, target, **kwargs): class _Sessions: - """Stands in for the stored sign-in: hands out ``first`` until forced, then ``after``.""" + """Stands in for the stored sign-in: hands out ``first`` until forced, then ``after``. + + A forced refresh that yields no token clears the store, as a fatal refresh does. + """ def __init__(self, first: str, after: str | None) -> None: self.first = first self.after = after self.forced = 0 + self.cleared = False - def __call__(self, *, refresh=True, force=False, allow_clear=True): + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): if force: self.forced += 1 - return SimpleNamespace(access_token=self.after, base_url="https://cloud.comfy.org") if self.after else None - return SimpleNamespace(access_token=self.first, base_url="https://cloud.comfy.org") + if self.after is None: + self.cleared = allow_clear + return None + return SimpleNamespace(access_token=self.after, refresh_token="rt", base_url="https://cloud.comfy.org") + if self.cleared: + return None + return SimpleNamespace(access_token=self.first, refresh_token="rt", base_url="https://cloud.comfy.org") def _install(monkeypatch, server: _Server, sessions: _Sessions) -> None: @@ -75,6 +88,63 @@ def _install(monkeypatch, server: _Server, sessions: _Sessions) -> None: } +@pytest.mark.parametrize("lagging", ["builder", "deploy"]) +def test_overlapping_clients_reuse_the_peer_refresh_before_retrying(monkeypatch, tmp_path, lagging) -> None: + """A late 401 for OLD must not rotate the NEW token a peer just obtained.""" + from comfy_cli.auth import store + from comfy_cli.cloud import oauth + + monkeypatch.setattr(store, "secrets_path", lambda: tmp_path / "secrets.json") + store.save_cloud_session( + base_url="https://cloud.comfy.org", + resource="https://cloud.comfy.org/api", + client_id="cli", + scope="s", + access_token="OLD", + refresh_token="RT0", + token_type="Bearer", + expires_at=int(time.time()) + 3600, + ) + refreshes = [] + + def refresh(**kwargs): + refreshes.append(kwargs["refresh_token"]) + generation = len(refreshes) + return oauth.TokenSet( + access_token=f"NEW{generation}", + refresh_token=f"RT{generation}", + token_type="Bearer", + expires_in=3600, + expires_at=int(time.time()) + 3600, + scope="s", + ) + + monkeypatch.setattr(oauth, "refresh_tokens", refresh) + clients = {"builder": BuilderClient.from_session(_BUILDER_URL), "deploy": DeployClient.from_session(_DEPLOY_URL)} + peer = "deploy" if lagging == "builder" else "builder" + calls = { + "builder": lambda: clients["builder"].get_release("r1"), + "deploy": lambda: clients["deploy"].get_deployment("dep-1"), + } + requests = [] + + def send(url, target, **kwargs): + name = "builder" if url.startswith(_BUILDER_URL) else "deploy" + requests.append((name, target.auth_token)) + if target.auth_token == "OLD": + if name == lagging: + calls[peer]() # Peer finishes its refresh while this stale request is in flight. + raise urllib.error.HTTPError(url, 401, "expired", http.client.HTTPMessage(), io.BytesIO(b"{}")) + return 200, dict(_ANSWER) + + monkeypatch.setattr("comfy_cli.builder_api.request_json", send) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", send) + + assert calls[lagging]()["status"] == "ready" + assert refreshes == ["RT0"] + assert requests == [(lagging, "OLD"), (peer, "OLD"), (peer, "NEW1"), (lagging, "NEW1")] + + @pytest.mark.parametrize("call", _DEPLOY_CALLS.values(), ids=_DEPLOY_CALLS.keys()) def test_deploy_client_retries_once_with_a_refreshed_token_after_a_401(monkeypatch, call) -> None: # Given @@ -260,3 +330,348 @@ def test_a_refused_injected_token_says_to_replace_it_not_to_sign_in(monkeypatch) assert "COMFY_BUILDER_TOKEN" in envelope["error"]["hint"] assert "comfy cloud login" not in envelope["error"]["hint"] assert server.tokens == ["injected"] + + +# --- A 401 that outlives the refresh during a watch ----------------------------- +# +# Several commands polling at once can each be refused after the shared refresh: +# a peer can hold the refresh lock past this process's timeout, and then this one +# retries nothing. The build or deployment was accepted with the same sign-in a +# moment earlier and runs on, so while that sign-in is still stored, signing in +# again is not the fix. A 401 before anything was submitted keeps its sign-in code. + + +class _AnswersTheCut(_Server): + """Answers the cut whatever token it carries, then refuses every read the watch makes.""" + + def __call__(self, url, target, **kwargs): + if kwargs.get("method") == "POST": + self.tokens.append(target.auth_token) + return 200, dict(_ANSWER) + return super().__call__(url, target, **kwargs) + + +def _watched_cut(monkeypatch, server: _Server, sessions: _Sessions): + import json + + from typer.testing import CliRunner + + from comfy_cli.cmdline import app + + _install(monkeypatch, server, sessions) + monkeypatch.delenv("COMFY_BUILDER_TOKEN", raising=False) + monkeypatch.delenv("COMFY_CLOUD_API_KEY", raising=False) + result = CliRunner().invoke( + app, + ["--json", "build", "release", "create", "--id", "b1", "--target", "linux/nvidia", "--watch"], + env={"COMFY_BUILDER_URL": _BUILDER_URL}, + ) + envelope = json.loads([line for line in result.stdout.splitlines() if line.strip()][-1]) + return result, envelope["error"] + + +def test_a_release_watch_refused_after_the_refresh_says_the_release_keeps_building(monkeypatch) -> None: + # Given a cut the builder accepts, then a watch it refuses even after the refresh + server = _AnswersTheCut(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then only the watch ended: its own code and exit status, and no sign-in prompt + assert result.exit_code == 75 + assert error["code"] == "watch_auth_lost" + assert "may still be building" in error["message"] + assert "comfy build release show r1" in error["hint"] + assert "comfy cloud login" not in error["hint"] + assert error["details"] == {"releaseId": "r1"} + assert server.tokens == ["signed-in", "signed-in", "refreshed"] + assert sessions.forced == 1 + + +@pytest.mark.parametrize("misses", [1, 4]) +@pytest.mark.parametrize("refreshed", ["refreshed", None], ids=["sign-in-retained", "signed-out"]) +@pytest.mark.parametrize("later_poll", [False, True], ids=["initial-poll", "later-poll"]) +def test_a_release_watch_transport_retry_preserves_the_following_auth_refusal( + monkeypatch, misses, refreshed, later_poll +) -> None: + from comfy_cli.command import build + + class TimeoutThenRefuse(_AnswersTheCut): + remaining = misses + pending_building = later_poll + + def __call__(self, url, target, **kwargs): + if kwargs.get("method") != "POST" and self.pending_building: + self.pending_building = False + return 200, {"id": "r1", "status": "building"} + if kwargs.get("method") != "POST" and self.remaining: + self.remaining -= 1 + raise TimeoutError("timed out") + return super().__call__(url, target, **kwargs) + + server = TimeoutThenRefuse(valid=set()) + sessions = _Sessions(first="signed-in", after=refreshed) + sleeps = [] + monkeypatch.setattr(build.time, "sleep", sleeps.append) + + result, error = _watched_cut(monkeypatch, server, sessions) + + assert result.exit_code == (75 if refreshed else 1), result.output + assert error["code"] == ("watch_auth_lost" if refreshed else "build_not_signed_in") + assert sleeps == ([build._RELEASE_POLL_SECONDS] if later_poll else []) + [4.0, 8.0, 16.0, 30.0][:misses] + assert server.tokens == (["signed-in", "signed-in", "refreshed"] if refreshed else ["signed-in", "signed-in"]) + assert sessions.forced == 1 + assert sessions.cleared is (refreshed is None) + assert server.remaining == 0 + if refreshed: + assert error["details"] == {"releaseId": "r1"} + assert "may still be building" in error["message"] + assert "comfy build release show r1" in error["hint"] + assert "comfy cloud login" not in error["hint"] + + +def test_a_release_watch_whose_refresh_signed_the_user_out_still_says_to_sign_in(monkeypatch) -> None: + # Given a watch refused, and a refresh that cleared the sign-in + server = _AnswersTheCut(valid=set()) + sessions = _Sessions(first="signed-in", after=None) + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then nobody is signed in any more, so signing in is the move + assert result.exit_code == 1 + assert error["code"] == "build_not_signed_in" + assert "comfy cloud login" in error["hint"] + + +def test_a_cut_refused_before_anything_was_made_keeps_the_sign_in_code(monkeypatch) -> None: + # Given a builder that refuses the cut itself, with the sign-in still stored + server = _Server(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + + # When + result, error = _watched_cut(monkeypatch, server, sessions) + + # Then nothing is running, so the refusal is the sign-in's + assert result.exit_code == 1 + assert error["code"] == "build_not_signed_in" + assert server.tokens == ["signed-in", "refreshed"] + + +def test_a_deploy_watch_refused_after_the_refresh_ends_the_watch_not_the_deployment(monkeypatch) -> None: + # Given + from comfy_cli.command.deploy_runtime import DeployWatchAuthLostError, poll_deployment + + server = _Server(valid=set()) + sessions = _Sessions(first="signed-in", after="refreshed") + _install(monkeypatch, server, sessions) + sleeps: list[float] = [] + + # When + with pytest.raises(DeployWatchAuthLostError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", sleeps.append) + + # Then the read was refused after one refresh and one retry, and never retried again + assert exc_info.value.deployment_id == "dep-1" + assert exc_info.value.code == "watch_auth_lost" + assert server.tokens == ["signed-in", "refreshed"] + assert sleeps == [] + + +def test_a_watch_with_an_unrefreshable_stored_token_requires_sign_in(monkeypatch) -> None: + from comfy_cli.command.deploy_runtime import poll_deployment + + # A token without a refresh token can remain on disk after its lifetime ends. + session = SimpleNamespace(access_token="expired", refresh_token=None, base_url="https://cloud.comfy.org") + monkeypatch.setattr("comfy_cli.credentials.get_session", lambda **_: session) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", _Server(valid=set())) + + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", lambda _: None) + + assert exc_info.value.code == "deploy_not_signed_in" + + +class _SignedInElsewhere(_Sessions): + """A sign-in to another environment, stored while the watch ran.""" + + def __call__(self, *, refresh=True, force=False, allow_clear=True, rejected_access_token=None): + session = super().__call__(refresh=refresh, force=force, allow_clear=allow_clear) + if self.forced and not force: + return SimpleNamespace(access_token=self.after, refresh_token="rt", base_url="https://staging.comfy.test") + return session + + +@pytest.mark.parametrize( + "sessions", + [_Sessions(first="signed-in", after=None), _SignedInElsewhere(first="signed-in", after="staging")], + ids=["refresh_signed_the_user_out", "signed_in_to_another_environment"], +) +def test_a_deploy_watch_with_no_sign_in_left_to_keep_says_to_sign_in(monkeypatch, sessions) -> None: + # Given + from comfy_cli.command.deploy_runtime import poll_deployment + + _install(monkeypatch, _Server(valid=set()), sessions) + + # When + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(DeployClient.from_session(_DEPLOY_URL), "dep-1", lambda _: None) + + # Then + assert exc_info.value.code == "deploy_not_signed_in" + + +@pytest.mark.parametrize( + "build", + [ + lambda: DeployClient(_DEPLOY_URL, "injected"), + lambda: DeployClient( + _DEPLOY_URL, api_key=Credential(kind="api_key", value="k", source="env:COMFY_CLOUD_API_KEY") + ), + ], + ids=["handed_token", "workspace_key"], +) +def test_a_deploy_watch_on_a_credential_it_cannot_refresh_keeps_the_sign_in_code(monkeypatch, build) -> None: + # Given a sign-in stored beside a client that was never built from it + from comfy_cli.command.deploy_runtime import poll_deployment + + server = _Server(valid=set()) + _install(monkeypatch, server, _Sessions(first="signed-in", after="refreshed")) + + # When + with pytest.raises(DeployAPIError) as exc_info: + poll_deployment(build(), "dep-1", lambda _: None) + + # Then the credential the client sent is the one to replace + assert exc_info.value.code == "deploy_not_signed_in" + + +@pytest.fixture +def peer_takes_refresh_lock_after_first_read(monkeypatch, tmp_path): + """Own a real secrets lock; compress only its contended acquisition attempts.""" + from comfy_cli import locking + from comfy_cli.auth import store + from comfy_cli.cloud import oauth + + monkeypatch.setenv("COMFY_SECRETS_PATH", str(tmp_path / "secrets.json")) + store.save_cloud_session( + base_url="https://cloud.comfy.org", + resource="https://cloud.comfy.org/api", + client_id="cli", + scope="s", + access_token="signed-in", + refresh_token="RT0", + token_type="Bearer", + expires_at=4102444800, + ) + initial = store.secrets_path().read_bytes() + state = SimpleNamespace(armed=False, started=False, contended=False, timeouts=[], errors=[]) + held, release = threading.Event(), threading.Event() + real_acquire, real_read = locking._acquire, store.get_cloud_session + lock_stat = store.lock_path().stat() + + def peer(): + try: + with locking.file_lock(store.lock_path(), timeout=0): + state.contended = True + held.set() + if not release.wait(5): + raise AssertionError("test did not release its owned secrets lock") + state.contended = False + except BaseException as error: + state.errors.append(error) + held.set() + + owner = threading.Thread(target=peer) + + def acquire(fd, *, timeout): + info = os.fstat(fd) + owns_path = (info.st_dev, info.st_ino) == (lock_stat.st_dev, lock_stat.st_ino) + if owns_path and state.contended: + state.timeouts.append(timeout) + if timeout is None: + pytest.fail("unbounded secrets read after the refresh lock timed out") + # Exercise the actual contended OS lock without waiting out the + # production refresh budget. Classification already requests zero. + return real_acquire(fd, timeout=0) + return real_acquire(fd, timeout=timeout) + + def read(): + session = real_read() + if state.armed and not state.started: + state.started = True + owner.start() + assert held.wait(5), "test peer never acquired its secrets lock" + assert not state.errors + return session + + monkeypatch.setattr(locking, "_acquire", acquire) + monkeypatch.setattr(store, "get_cloud_session", read) + monkeypatch.setattr(oauth, "refresh_tokens", lambda **_: pytest.fail("must not refresh outside the held lock")) + + def release_peer(): + release.set() + if owner.ident is not None: + # Acquisition was nonblocking; the only possible wait is our own + # event, released above even when a regression assertion fails. + owner.join() + assert not state.errors + + state.release_peer = release_peer + try: + yield state + finally: + release_peer() + assert store.secrets_path().read_bytes() == initial + + +@pytest.mark.parametrize("kind", ["builder", "deploy"]) +def test_a_watch_does_not_wait_again_when_a_peer_outlives_the_refresh_lock( + monkeypatch, capsys, peer_takes_refresh_lock_after_first_read, kind +): + import json + + import typer + + from comfy_cli import credentials + from comfy_cli.caller import Caller + from comfy_cli.command.build import _watched_release + from comfy_cli.command.deploy_runtime import DeployWatchAuthLostError, exit_watch_lost, poll_deployment + from comfy_cli.output.renderer import Renderer + + state = peer_takes_refresh_lock_after_first_read + client = BuilderClient.from_session(_BUILDER_URL) if kind == "builder" else DeployClient.from_session(_DEPLOY_URL) + requests = [] + + def refused(url, target, **kwargs): + requests.append(target.auth_token) + state.armed = True + raise urllib.error.HTTPError(url, 401, "refused", http.client.HTTPMessage(), io.BytesIO(b"{}")) + + monkeypatch.setattr("comfy_cli.builder_api.request_json", refused) + monkeypatch.setattr("comfy_cli.deploy_api.request_json", refused) + renderer = Renderer.resolve( + is_stdout_tty=False, env={}, caller=Caller(kind="user", agentic=False, source_env=None), json_flag=True + ) + with pytest.raises(typer.Exit) as stopped: + if kind == "builder": + _watched_release(renderer, client, "r1") + else: + try: + poll_deployment(client, "dep-1", lambda _: pytest.fail("must not sleep after the refused read")) + except DeployWatchAuthLostError as error: + exit_watch_lost(renderer, error) + + envelope = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + assert stopped.value.exit_code == 75 + assert envelope["error"]["code"] == "watch_auth_lost" + assert "comfy cloud login" not in envelope["error"]["hint"] + assert requests == ["signed-in"] + assert state.timeouts == [90.0, 0] + assert state.started is True + + # With the same real store readable again, the nested lock is reentrant. + state.release_peer() + assert credentials.sign_in_still_stored("https://cloud.comfy.org") is True + assert credentials.sign_in_still_stored(None) is False