From 3e69b685b9cd7b708160f32f4e3f4a1c56e388bb Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 03:20:50 +0600 Subject: [PATCH 1/5] fix(server): a usage-limited thread no longer auto-settles A run that stops on a usage limit ends the activity that kept the thread active, so the settle sweep could settle it in the same second (for example when its pull request had merged). Settled threads refuse limit recovery, so Resume at reset and Snooze until reset then always failed. Settlement candidates now skip a thread whose latest run failed with a usage_limit root error, and isAutoSettlementCandidate applies the same rule to shell threads. --- .../ProjectionSettlement.test.ts | 57 ++++++++++++++++++- .../src/orchestration-v2/ProjectionStore.ts | 12 ++++ .../ThreadSettlementService.test.ts | 11 ++++ .../ThreadSettlementService.ts | 2 + 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index 1352e17585bfd..442bc5e0a67b4 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -136,6 +136,44 @@ const createItem = Effect.fn(function* ( }); }); +const createFailure = Effect.fn(function* ( + threadId: ThreadId, + runId: RunId, + failureClass: "usage_limit" | "provider_error", +) { + const store = yield* ProjectionStore.ProjectionStoreV2; + yield* store.apply({ + id: EventId.make(`event:failure:${runId}`), + type: "turn-item.updated", + threadId, + occurredAt: old, + payload: { + id: TurnItemId.make(`failure:${runId}`), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + type: "error", + status: "failed", + title: failureClass, + startedAt: old, + completedAt: old, + updatedAt: old, + failure: { + class: failureClass, + message: failureClass, + code: null, + retryable: true, + resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })), + }, + }, + }); +}); + it.effect( "sql: discovers settlement work with the same activity and background semantics as the shell", () => @@ -210,6 +248,14 @@ it.effect( snoozedAt: DateTime.subtract(old, { days: 1 }), }); yield* createRun(woke, "failed"); + const limited = yield* createThread("usage-limited"); + yield* createFailure(limited, yield* createRun(limited, "failed"), "usage_limit"); + const providerFailed = yield* createThread("provider-failed"); + yield* createFailure( + providerFailed, + yield* createRun(providerFailed, "failed"), + "provider_error", + ); const background = yield* createThread("idle-background"); yield* createItem(background, yield* createRun(background), "idle"); const persistent = yield* createThread("persistent-monitor"); @@ -250,7 +296,16 @@ it.effect( ); assert.deepEqual( new Set(eligible.map((thread) => thread.id)), - new Set([idle, completed, queued, woke, background, persistent, rolledBack]), + new Set([ + idle, + completed, + queued, + woke, + providerFailed, + background, + persistent, + rolledBack, + ]), ); assert.deepEqual( new Set(eligible.map((thread) => thread.id)), diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2e88fdc119081..1347844d037e5 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -212,6 +212,7 @@ export type ProjectionSettlementCandidate = Pick< | "latestRunCompletedAt" | "latestUserMessageAt" | "status" + | "lastErrorClass" | "activityRunStartedAt" | "activityRunStatus" | "pendingRuntimeRequest" @@ -6292,6 +6293,15 @@ export const layer: Layer.Layer = WHERE link.thread_id = t.thread_id AND link.type = 'link_request' AND link.status = 'waiting' ) + AND NOT ( + r.status IS 'failed' AND EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items failure + WHERE failure.thread_id = t.thread_id AND failure.run_id = r.run_id + AND failure.type = 'error' AND failure.status = 'failed' + AND failure.node_id IS json_extract(r.payload_json, '$.rootNodeId') + AND json_extract(failure.payload_json, '$.failure.class') = 'usage_limit' + ) + ) ORDER BY t.updated_at ASC, t.thread_id ASC `; if (rows.length === 0) return []; @@ -6339,6 +6349,8 @@ export const layer: Layer.Layer = activityRunStatus: null, activityRunStartedAt: null, pendingRuntimeRequest: null, + // The query above drops usage-limited threads. + lastErrorClass: null, pendingGuestMessageCount: pendingGuestMessageCount(thread.guestShare), pendingBackgroundTasks: derivePendingBackgroundWork({ latestRun: diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts index 0a0996c0ede0c..607c2986197aa 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts @@ -158,6 +158,17 @@ describe("isAutoSettlementCandidate", () => { expect(ThreadSettlementService.isSnoozed(snoozed, NOW_MS)).toBe(false); }); + it("never settles a thread stopped by a usage limit", () => { + const limited = shell({ status: "failed", lastErrorClass: "usage_limit" }); + expect(ThreadSettlementService.isAutoSettlementCandidate(limited, NOW_MS)).toBe(false); + expect( + ThreadSettlementService.isAutoSettlementCandidate( + { ...limited, lastErrorClass: "provider_error" }, + NOW_MS, + ), + ).toBe(true); + }); + it("keeps snoozed threads parked until a fresh failure or timer wake", () => { const snoozed = shell({ snoozedUntil: at(60 * 60 * 1_000), diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.ts index 6a3d4fffb0399..32879f94d4ea0 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.ts @@ -152,6 +152,8 @@ export function isAutoSettlementCandidate( // A guest is waiting on the owner; settling would hide the Request. if ((thread.pendingGuestMessageCount ?? 0) > 0) return false; if (thread.activityRunStatus != null) return false; + // A usage limit interrupts the work; settling would hide Resume at reset. + if (thread.status === "failed" && thread.lastErrorClass === "usage_limit") return false; if (backgroundWorkHoldsCompletion(thread.pendingBackgroundTasks ?? [])) return false; if (threadHasQueuedTurnStart(thread, nowMs)) return false; // Interrupted runs must keep the same snooze protection the sidebar shows. From 54eada734f34c3cd8342ba5a0e7998eb583a51f8 Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 03:52:34 +0600 Subject: [PATCH 2/5] fix(server): match the shell's blocked run when skipping usage-limited threads A message queued behind a usage limit becomes the newest run, so checking only the newest run let the sweep settle a thread the shell still shows as limited. Use the latest executed run, as the shell and the recovery worker do, and its latest root error. --- .../ProjectionSettlement.test.ts | 8 +++++ .../src/orchestration-v2/ProjectionStore.ts | 34 +++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index 442bc5e0a67b4..e02a657e69e75 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -250,6 +250,14 @@ it.effect( yield* createRun(woke, "failed"); const limited = yield* createThread("usage-limited"); yield* createFailure(limited, yield* createRun(limited, "failed"), "usage_limit"); + // A message queued behind the limit is the newest run, but the limit still blocks. + const queuedBehindLimit = yield* createThread("queued-behind-limit"); + yield* createFailure( + queuedBehindLimit, + yield* createRun(queuedBehindLimit, "failed"), + "usage_limit", + ); + yield* createRun(queuedBehindLimit, "queued", 2); const providerFailed = yield* createThread("provider-failed"); yield* createFailure( providerFailed, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 1347844d037e5..2a4117e72c691 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -6293,14 +6293,42 @@ export const layer: Layer.Layer = WHERE link.thread_id = t.thread_id AND link.type = 'link_request' AND link.status = 'waiting' ) + -- A usage limit interrupts work; settling would hide its recovery controls. + -- Matches the shell's blocked run, so a message queued behind the limit + -- does not hide it. The cheap EXISTS keeps the run sort off other threads. AND NOT ( - r.status IS 'failed' AND EXISTS ( + EXISTS ( SELECT 1 FROM orchestration_v2_projection_turn_items failure - WHERE failure.thread_id = t.thread_id AND failure.run_id = r.run_id + WHERE failure.thread_id = t.thread_id AND failure.type = 'error' AND failure.status = 'failed' - AND failure.node_id IS json_extract(r.payload_json, '$.rootNodeId') AND json_extract(failure.payload_json, '$.failure.class') = 'usage_limit' ) + AND EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runs blocked + WHERE blocked.run_id = ( + SELECT latest.run_id FROM orchestration_v2_projection_runs latest + WHERE latest.thread_id = t.thread_id + AND latest.status <> 'queued' + AND NOT ( + latest.status = 'cancelled' + AND json_extract(latest.payload_json, '$.startedAt') IS NULL + ) + -- latestExecutedRun: the run that ended last (runRanAfter). + ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC, + latest.ordinal DESC, latest.run_id DESC + LIMIT 1 + ) + AND blocked.status = 'failed' + AND ( + SELECT json_extract(error.payload_json, '$.failure.class') + FROM orchestration_v2_projection_turn_items error + WHERE error.thread_id = t.thread_id AND error.run_id = blocked.run_id + AND error.type = 'error' AND error.status = 'failed' + AND error.node_id IS json_extract(blocked.payload_json, '$.rootNodeId') + ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC + LIMIT 1 + ) = 'usage_limit' + ) ) ORDER BY t.updated_at ASC, t.thread_id ASC `; From b2c9981a7418db9e0c4476262745c99a116b3259 Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 03:56:13 +0600 Subject: [PATCH 3/5] test(server): cover a usage limit that ended after a newer run --- .../orchestration-v2/ProjectionSettlement.test.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index e02a657e69e75..c822b42534323 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -72,6 +72,7 @@ const createRun = Effect.fn(function* ( threadId: ThreadId, status: OrchestrationV2Run["status"] = "completed", ordinal = 1, + completedAt = old, ) { const store = yield* ProjectionStore.ProjectionStoreV2; const runId = RunId.make(`run:${threadId}:${ordinal}`); @@ -93,7 +94,7 @@ const createRun = Effect.fn(function* ( status, requestedAt: old, startedAt: status === "queued" ? null : old, - completedAt: ["completed", "rolled_back", "failed"].includes(status) ? old : null, + completedAt: ["completed", "rolled_back", "failed"].includes(status) ? completedAt : null, checkpointId: null, contextHandoffId: null, }, @@ -258,6 +259,18 @@ it.effect( "usage_limit", ); yield* createRun(queuedBehindLimit, "queued", 2); + // Held queues run out of order; the run that ended last decides, not the ordinal. + const limitEndedLast = yield* createThread("limit-ended-last"); + yield* createFailure( + limitEndedLast, + yield* createRun(limitEndedLast, "failed", 1, DateTime.add(old, { hours: 1 })), + "usage_limit", + ); + yield* createFailure( + limitEndedLast, + yield* createRun(limitEndedLast, "failed", 2), + "provider_error", + ); const providerFailed = yield* createThread("provider-failed"); yield* createFailure( providerFailed, From 1b7badf4fdfdd759fc639de2e63826f5161f5931 Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 04:11:03 +0600 Subject: [PATCH 4/5] fix(server): a differing session error lets a usage-limited thread settle The shell drops the usage-limit class when the provider session reports a different, newer error, so it shows no recovery controls. The settlement query now applies the same rule instead of holding such a thread. --- .../ProjectionSettlement.test.ts | 30 +++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 15 ++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index c822b42534323..c0f3a703823da 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -8,6 +8,7 @@ import { ProjectId, ProviderDriverKind, ProviderInstanceId, + ProviderSessionId, ProviderThreadId, RunId, RuntimeRequestId, @@ -19,6 +20,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import { isAutoSettlementCandidate, resolveAutoSettlementAt } from "./ThreadSettlementService.ts"; @@ -271,6 +273,33 @@ it.effect( yield* createRun(limitEndedLast, "failed", 2), "provider_error", ); + // A newer, different session error replaces the limit, as in threadErrorSummary. + const sessionErrorAfterLimit = yield* createThread("session-error-after-limit"); + yield* createFailure( + sessionErrorAfterLimit, + yield* createRun(sessionErrorAfterLimit, "failed"), + "usage_limit", + ); + yield* store.apply({ + id: EventId.make("event:settlement:session-error"), + type: "provider-session.attached", + threadId: sessionErrorAfterLimit, + driver: ProviderDriverKind.make("codex"), + providerInstanceId, + occurredAt: old, + payload: { + id: ProviderSessionId.make("provider-session:settlement:session-error"), + driver: ProviderDriverKind.make("codex"), + providerInstanceId, + status: "error", + cwd: "/workspace", + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: old, + updatedAt: old, + lastError: "Session crashed.", + }, + }); const providerFailed = yield* createThread("provider-failed"); yield* createFailure( providerFailed, @@ -323,6 +352,7 @@ it.effect( queued, woke, providerFailed, + sessionErrorAfterLimit, background, persistent, rolledBack, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2a4117e72c691..75a88ff1883a3 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -6319,15 +6319,26 @@ export const layer: Layer.Layer = LIMIT 1 ) AND blocked.status = 'failed' + -- threadErrorSummary: a differing session error replaces the failure. AND ( - SELECT json_extract(error.payload_json, '$.failure.class') + SELECT json_extract(error.payload_json, '$.failure.class') = 'usage_limit' + AND COALESCE(( + SELECT json_extract(session.payload_json, '$.lastError') + FROM orchestration_v2_projection_provider_session_bindings binding + CROSS JOIN orchestration_v2_projection_provider_sessions session + ON session.provider_session_id = binding.provider_session_id + WHERE binding.thread_id = t.thread_id + AND session.provider_instance_id = t.provider_instance_id + ORDER BY session.updated_at DESC, session.provider_session_id DESC + LIMIT 1 + ) = json_extract(error.payload_json, '$.failure.message'), 1) FROM orchestration_v2_projection_turn_items error WHERE error.thread_id = t.thread_id AND error.run_id = blocked.run_id AND error.type = 'error' AND error.status = 'failed' AND error.node_id IS json_extract(blocked.payload_json, '$.rootNodeId') ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC LIMIT 1 - ) = 'usage_limit' + ) = 1 ) ) ORDER BY t.updated_at ASC, t.thread_id ASC From 26144a20e01c7bf55e16659a4051ac8225fe55d7 Mon Sep 17 00:00:00 2001 From: Ehsanul Haque Siam Date: Sun, 1 Nov 2026 04:23:18 +0600 Subject: [PATCH 5/5] test(server): cover a session error that matches the usage limit --- .../ProjectionSettlement.test.ts | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index c0f3a703823da..55258bf010666 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -177,6 +177,31 @@ const createFailure = Effect.fn(function* ( }); }); +const attachSessionError = Effect.fn(function* (threadId: ThreadId, lastError: string) { + const store = yield* ProjectionStore.ProjectionStoreV2; + const driver = ProviderDriverKind.make("codex"); + yield* store.apply({ + id: EventId.make(`event:session:${threadId}`), + type: "provider-session.attached", + threadId, + driver, + providerInstanceId, + occurredAt: old, + payload: { + id: ProviderSessionId.make(`provider-session:${threadId}`), + driver, + providerInstanceId, + status: "error", + cwd: "/workspace", + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: old, + updatedAt: old, + lastError, + }, + }); +}); + it.effect( "sql: discovers settlement work with the same activity and background semantics as the shell", () => @@ -273,33 +298,21 @@ it.effect( yield* createRun(limitEndedLast, "failed", 2), "provider_error", ); - // A newer, different session error replaces the limit, as in threadErrorSummary. + // threadErrorSummary: a matching session error keeps the limit, a differing one replaces it. + const sessionErrorMatchingLimit = yield* createThread("session-error-matching-limit"); + yield* createFailure( + sessionErrorMatchingLimit, + yield* createRun(sessionErrorMatchingLimit, "failed"), + "usage_limit", + ); + yield* attachSessionError(sessionErrorMatchingLimit, "usage_limit"); const sessionErrorAfterLimit = yield* createThread("session-error-after-limit"); yield* createFailure( sessionErrorAfterLimit, yield* createRun(sessionErrorAfterLimit, "failed"), "usage_limit", ); - yield* store.apply({ - id: EventId.make("event:settlement:session-error"), - type: "provider-session.attached", - threadId: sessionErrorAfterLimit, - driver: ProviderDriverKind.make("codex"), - providerInstanceId, - occurredAt: old, - payload: { - id: ProviderSessionId.make("provider-session:settlement:session-error"), - driver: ProviderDriverKind.make("codex"), - providerInstanceId, - status: "error", - cwd: "/workspace", - model: modelSelection.model, - capabilities: CodexProviderCapabilitiesV2, - createdAt: old, - updatedAt: old, - lastError: "Session crashed.", - }, - }); + yield* attachSessionError(sessionErrorAfterLimit, "Session crashed."); const providerFailed = yield* createThread("provider-failed"); yield* createFailure( providerFailed,