diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 0c61165924917..ae013bb36c89d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -4023,6 +4023,111 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); }); + it.effect.each([ + { + failure: "authentication_failed", + apiErrorStatus: undefined, + opensFreshProcess: true, + backgroundShell: false, + }, + { + failure: "authentication_failed", + apiErrorStatus: undefined, + opensFreshProcess: true, + backgroundShell: true, + }, + { + failure: "server_error", + apiErrorStatus: undefined, + opensFreshProcess: false, + backgroundShell: false, + }, + { + failure: "authentication_failed", + apiErrorStatus: 403, + opensFreshProcess: false, + backgroundShell: false, + }, + ] as const)( + "after a turn fails with $failure (API status: $apiErrorStatus), the next turn opens a fresh process: $opensFreshProcess (background shell: $backgroundShell)", + ({ failure, apiErrorStatus, opensFreshProcess, backgroundShell }) => + Effect.gen(function* () { + let closeCount = 0; + const harness = yield* makeWakeHarnessWithOptions({ + freshQueueOnReopen: true, + close: (messages) => + Effect.sync(() => { + closeCount += 1; + }).pipe(Effect.andThen(Queue.shutdown(messages))), + }); + const now = yield* DateTime.now; + const startTurn = (attempt: string, text: string) => + harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make(attempt), + text, + attachments: [], + }), + ); + + yield* startTurn("attempt-claude-relogin-1", "Continue."); + if (backgroundShell) yield* harness.offerAndWait(wakeTaskStarted); + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ uuid: "00000000-0000-4000-8000-000000000611", error: failure }), + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000612", + result: "API Error", + ...(apiErrorStatus === undefined ? {} : { apiErrorStatus }), + terminalReason: "api_error", + isError: true, + }), + ]); + const failed = yield* Queue.take(harness.terminalReceipts); + assert.equal(failed.status, "failed"); + + if (backgroundShell) { + const refused = yield* startTurn("attempt-claude-relogin-blocked", "Try again.").pipe( + Effect.result, + ); + assert.equal(refused._tag, "Failure"); + if (refused._tag === "Failure") { + assert.instanceOf( + refused.failure.cause, + ClaudeAdapterV2.ClaudeBackgroundWorkBlocksQueryReplacementError, + ); + } + assert.lengthOf(harness.processQueues, 1); + assert.equal(closeCount, 0); + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId: failed.providerTurnId, + requestRuntimeRestart: true, + }); + } + yield* startTurn("attempt-claude-relogin-2", "Try again."); + assert.equal(harness.processQueues.length, opensFreshProcess ? 2 : 1); + assert.equal(closeCount, opensFreshProcess ? 1 : 0); + if (opensFreshProcess) { + // The fresh process resumes the same Claude conversation. + assert.equal(harness.getOpenedOptions()?.resume, WAKE_NATIVE_SESSION); + assert.isUndefined(harness.getOpenedOptions()?.sessionId); + } + yield* Queue.offer( + harness.processQueues.at(-1)!, + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000613", result: "Done." }), + ); + const completed = yield* Queue.take(harness.terminalReceipts); + assert.equal(completed.status, "completed"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + it.effect( "carries earlier user-only skill instructions on sends and steering with attachments", () => @@ -4860,6 +4965,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(terminal.status, "failed"); if (terminal.status !== "failed") return; assert.include(terminal.failure.message, "run `claude auth login`"); + assert.include(terminal.failure.message, "then send the message again"); assert.include(terminal.failure.message, configDir); assert.include(terminal.failure.message, JSON.stringify(cwd)); assert.notInclude(terminal.failure.message, "repeated API errors"); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts index 95b55fa413c95..717a7aafdfafe 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts @@ -525,6 +525,7 @@ function makeReplayQueryRunner( ): AsyncGenerator { while (true) { throwIfFailed(); + if (signal.aborted) return; const entry = transcript.entries[cursor]; if (entry === undefined) { @@ -803,6 +804,7 @@ function makeReplayQueryRunner( return { open: (input) => { + abortReplayIterators(); assertNextOutboundFrame(makeClaudeQueryOpenFrame(input)); return { messages: Stream.fromAsyncIterable(replayMessagesWithGateCleanup(input.options), (cause) => diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index f8cbbdabaf2ef..5f8573201ef23 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -539,7 +539,7 @@ export class ClaudeBackgroundWorkBlocksQueryReplacementError extends Schema.Tagg {}, ) { override get message(): string { - return "Claude is still running background agents or commands, and this model or setting change would end them. Wait for them to finish, or press Stop, then send the message again."; + return "Claude is still running background agents or commands, and starting a new Claude process would end them. Wait for them to finish, or press Stop, then send the message again."; } } @@ -3759,6 +3759,8 @@ interface ClaudeLiveQueryContext { receivedInit: boolean; permissionMode: PermissionMode; stopping: boolean; + // A fresh process picks up a new login after authentication fails. + signedOut: boolean; readonly subagentsFromEarlierProcesses: ReadonlySet; } @@ -7148,8 +7150,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.gen(function* () { const nativeThreadId = input.context.input.providerThread.nativeThreadRef?.nativeId ?? null; + const liveQuery = yield* Ref.get(queryContext); if (nativeThreadId !== null) { - if (input.status !== "completed") { + if (input.status !== "completed" && liveQuery?.signedOut !== true) { yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId); yield* clearNativeThreadTaskIdSet( wakeEligibleBackgroundTasksByNativeThread, @@ -7182,7 +7185,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( nativeThreadId === null ? undefined : goalsByNativeThread.get(nativeThreadId); const goalChecked = goalCheckedTurns.delete(input.context.providerTurnId); const terminalReason = input.result?.terminal_reason; - const liveQuery = yield* Ref.get(queryContext); if ( nativeThreadId !== null && goal?.status === "active" && @@ -8960,13 +8962,21 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( resultFailure?.class === "usage_limit" ? { ...resultFailure, resetAt: resetAt ?? resultFailure.resetAt ?? null } : resultFailure; + const status = interrupted + ? "interrupted" + : limitedCompaction !== null + ? "failed" + : terminalStatusFromResult(message, failureHint); + if ( + status === "failed" && + context.authenticationFailureMessage !== undefined && + !permissionDenied + ) { + liveQuery.signedOut = true; + } yield* finalizeActiveTurn({ context, - status: interrupted - ? "interrupted" - : limitedCompaction !== null - ? "failed" - : terminalStatusFromResult(message, failureHint), + status, completedAt, result: message, ...(terminalFailure === null ? {} : { failure: terminalFailure }), @@ -9555,6 +9565,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if ( existing !== null && existing.nativeThreadId === nativeThreadId && + !existing.signedOut && existing.queryPolicyKey === queryPolicyKey && !existing.stopping && existing.selection.queryIdentity !== compiledSelection.queryIdentity && @@ -9578,6 +9589,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if ( existing !== null && existing.nativeThreadId === nativeThreadId && + !existing.signedOut && existing.queryPolicyKey === queryPolicyKey && existing.selection.queryIdentity === compiledSelection.queryIdentity ) { @@ -9714,6 +9726,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( receivedInit: false, permissionMode: queryOptions.permissionMode, stopping: false, + signedOut: false, subagentsFromEarlierProcesses: new Set( [...(yield* Ref.get(sessionSubagentsByTaskId)).values()].filter( (subagent) => subagent.task.status === "running", diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/claude_transcript.ndjson index 2597109378ee2..8c4553f98d196 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/claude_transcript.ndjson @@ -4,6 +4,7 @@ {"type":"emit_inbound","label":"system:init:1","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.183","cwd":"/tmp/claude-replay-claude_result_is_error","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"aa401aa4-0000-4000-8000-000000000001","session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40"}} {"type":"emit_inbound","label":"assistant:synthetic_auth_error","frame":{"type":"assistant","message":{"id":"7f28a402-6d4c-491e-abbb-2b3c231a06f0","container":null,"model":"","role":"assistant","stop_details":null,"stop_reason":"stop_sequence","stop_sequence":"","type":"message","usage":{"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":null,"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":null,"iterations":null,"speed":null},"content":[{"type":"text","text":"Failed to authenticate. API Error: 401 Invalid authentication credentials"}],"context_management":null},"parent_tool_use_id":null,"session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40","uuid":"aa401aa4-0000-4000-8000-000000000002","error":"authentication_failed"}} {"type":"emit_inbound","label":"result:1_is_error","frame":{"type":"result","subtype":"success","is_error":true,"api_error_status":401,"duration_ms":2116,"duration_api_ms":0,"num_turns":1,"result":"Failed to authenticate. API Error: 401 Invalid authentication credentials","stop_reason":"stop_sequence","session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40","total_cost_usd":0,"usage":{"input_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"modelUsage":{},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"aa401aa4-0000-4000-8000-000000000003"}} +{"type":"expect_outbound","label":"query.open:resume","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"settings":{"showThinkingSummaries":true},"resume":"aa401aa4-e597-4e40-8e40-1e401e401e40"}}} {"type":"expect_outbound","label":"prompt.offer:2","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Try again now that auth is back."},"parent_tool_use_id":null}}} {"type":"emit_inbound","label":"assistant:recovered","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_01iserrorfixture1","type":"message","role":"assistant","content":[{"type":"text","text":"claude result is_error fixture recovered"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40","uuid":"aa401aa4-0000-4000-8000-000000000004"}} {"type":"emit_inbound","label":"result:2_success","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":4000,"duration_api_ms":3500,"num_turns":1,"result":"claude result is_error fixture recovered","stop_reason":"end_turn","session_id":"aa401aa4-e597-4e40-8e40-1e401e401e40","total_cost_usd":0.001,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":1,"output_tokens":1,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":2,"outputTokens":2,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"aa401aa4-0000-4000-8000-000000000005"}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts index fe3167b22cbcd..9bd33d54bdd47 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts @@ -13,7 +13,7 @@ import { const AUTH_ERROR_TEXT = "Failed to authenticate. API Error: 401 Invalid authentication credentials"; const AUTH_FAILURE_TEXT = - "Claude could not authenticate. For subscription login, run `claude auth login` on this environment's machine, then start a new thread. For API-key authentication, check this instance's configured credentials."; + "Claude could not authenticate. For subscription login, run `claude auth login` on this environment's machine, then send the message again. For API-key authentication, check this instance's configured credentials."; export function assertClaudeResultIsErrorOutput( result: OrchestratorV2ScenarioResult, diff --git a/apps/server/src/provider/Drivers/ClaudeHome.ts b/apps/server/src/provider/Drivers/ClaudeHome.ts index bd81aa10bce7a..4c9ccc31e3240 100644 --- a/apps/server/src/provider/Drivers/ClaudeHome.ts +++ b/apps/server/src/provider/Drivers/ClaudeHome.ts @@ -85,5 +85,5 @@ export const claudeSignedOutMessage = (input: { input.configDir !== undefined ? ` from ${quotePath(input.cwd)}, with CLAUDE_CONFIG_DIR set to ${quotePath(input.configDir)}` : ""; - return `Claude could not authenticate. For subscription login, run \`claude auth login\` on this environment's machine${configuration}, then start a new thread. For API-key authentication, check this instance's configured credentials.`; + return `Claude could not authenticate. For subscription login, run \`claude auth login\` on this environment's machine${configuration}, then send the message again. For API-key authentication, check this instance's configured credentials.`; };