From 3d01ec08a2d1398ff041551665138c3b7087080d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 3 Nov 2026 12:18:18 +0000 Subject: [PATCH 1/7] test(server): reproduce #30144 armed subagent resumes after parent settles These tests reproduce #30144: a subagent child armed for usage-limit recovery while its parent was open still yields a limit-resume: command after the parent is settled. The final assertion of the #27951 test encoded the bug; it now expects no resume while the parent is settled, and a resume once the parent is unsettled. Co-Authored-By: Claude Opus 5.5 --- .../orchestration-v2/ProjectionStore.test.ts | 75 ++++++++++++------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 5b24a65a99567..5f72055481452 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5547,7 +5547,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }), ); - it.effect("does not freshly arm limit recovery for a subagent of a settled parent", () => + it.effect("does not arm or resume limit recovery for a subagent of a settled parent", () => Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; const now = yield* DateTime.now; @@ -5555,7 +5555,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const threadPayload = ( threadId: ThreadId, parentThreadId: ThreadId | null, - settledOverride: "settled" | null, + settledOverride: "settled" | "active" | null, ) => ({ createdBy: "user", @@ -5580,7 +5580,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { updatedAt: now, archivedAt: null, settledOverride, - settledAt: settledOverride === null ? null : now, + settledAt: settledOverride === "settled" ? now : null, lastVisitedAt: null, deletedAt: null, }) as const; @@ -5687,39 +5687,56 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { `only the child of the unsettled parent is armed (${JSON.stringify(options)})`, ); } - // A recovery armed before the parent settled still resumes at its reset. + const activeParentId = ThreadId.make("thread:settled-parent-limit:active:parent"); + const activeRunId = RunId.make("run:settled-parent-limit:active"); yield* store.apply({ - id: EventId.make("event:settled-parent-limit:settled:armed"), + id: EventId.make("event:settled-parent-limit:active:armed"), type: "thread.metadata-updated", - threadId: settledParentChild, + threadId: activeParentChild, occurredAt: now, payload: { - ...threadPayload( - settledParentChild, - ThreadId.make("thread:settled-parent-limit:settled:parent"), - null, - ), - limitRecovery: { - runId: RunId.make("run:settled-parent-limit:settled"), - resetAt, - autoResume: true, - }, + ...threadPayload(activeParentChild, activeParentId, null), + limitRecovery: { runId: activeRunId, resetAt, autoResume: true }, }, }); const afterReset = DateTime.add(now, { hours: 2 }); - const resumed = (yield* store.getLimitRecoveryCandidates({ - now: afterReset, - autoResume: true, - snooze: false, - })).find((row) => row.id === settledParentChild); - assert.isDefined(resumed); - assert.isTrue( - limitRecoveryCommand( - resumed, - true, - DateTime.toEpochMillis(afterReset), - false, - )?.commandId.startsWith("limit-resume:"), + const resumeCommandIds = Effect.fnUntraced(function* () { + const candidates = yield* store.getLimitRecoveryCandidates({ + now: afterReset, + autoResume: true, + snooze: false, + }); + return candidates + .filter((row) => row.id === activeParentChild) + .map( + (row) => + limitRecoveryCommand(row, true, DateTime.toEpochMillis(afterReset), false)?.commandId, + ) + .filter((commandId) => commandId?.startsWith("limit-resume:")); + }); + yield* store.apply({ + id: EventId.make("event:settled-parent-limit:active:parent-settled"), + type: "thread.settled", + threadId: activeParentId, + occurredAt: now, + payload: threadPayload(activeParentId, null, "settled"), + }); + assert.deepEqual( + yield* resumeCommandIds(), + [], + "a subagent armed while its parent was open does not resume after the parent settles", + ); + yield* store.apply({ + id: EventId.make("event:settled-parent-limit:active:parent-unsettled"), + type: "thread.unsettled", + threadId: activeParentId, + occurredAt: now, + payload: threadPayload(activeParentId, null, "active"), + }); + assert.lengthOf( + yield* resumeCommandIds(), + 1, + "the armed subagent resumes once its parent is unsettled", ); }), ); From 1434dbfb669aca94429557bfbc1a9281d82f497e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 3 Nov 2026 12:21:59 +0000 Subject: [PATCH 2/7] fix(server): armed subagent limit resume waits while its parent is settled An auto-resume armed for a subagent while its parent was open still fired after the parent was settled, and the child's completion reopened the parent. The settled-parent guard in getLimitRecoveryCandidates only covered fresh arming; it now gates the armed-resume branch too. Unsettling the parent makes the armed resume eligible again. Fixes pingdotgg/t3code#30144 Co-Authored-By: Claude Opus 5.5 --- .../server/src/orchestration-v2/ProjectionStore.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 3cffce15b70d6..e62de48678084 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -3954,6 +3954,13 @@ export const layer: Layer.Layer = ) AND json_extract(t.payload_json, '$.archivedAt') IS NULL AND json_extract(t.payload_json, '$.settledOverride') IS NOT 'settled' + -- A subagent of a settled parent is abandoned work; arming or resuming it would wake the parent. + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_threads parent + WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent' + AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId') + AND json_extract(parent.payload_json, '$.settledOverride') = 'settled' + ) AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit' AND json_extract(item.payload_json, '$.failure.resetAt') IS NOT NULL AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) @@ -3977,13 +3984,6 @@ export const layer: Layer.Layer = ${booleanInt(options.autoResume)} OR (${booleanInt(options.snooze)} AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)})) ) - -- A subagent of a settled parent is abandoned work; resuming it would wake the parent. - AND NOT EXISTS ( - SELECT 1 FROM orchestration_v2_projection_threads parent - WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent' - AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId') - AND json_extract(parent.payload_json, '$.settledOverride') = 'settled' - ) ) ) AND NOT EXISTS ( From 7fed97a6017e54b5012f24b60337cab9f3e7283c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 3 Nov 2026 12:42:54 +0000 Subject: [PATCH 3/7] fix(server): disarm an armed subagent limit resume once its parent settles Gating the resume branch on the parent only paused it: unsettling the parent later resumed the abandoned subagent at any age. The sweep now disarms such a recovery (autoResume false for the same run and reset, as Cancel auto-resume does) through a plain dispatch, so it stays disarmed after the parent is unsettled. Fresh arming stays blocked in SQL as on main. Co-Authored-By: Claude Opus 5.5 --- .../orchestration-v2/ProjectionStore.test.ts | 50 +++++++++++++------ .../src/orchestration-v2/ProjectionStore.ts | 18 ++++--- .../UsageLimitRecoveryWorker.test.ts | 46 +++++++++++++++++ .../UsageLimitRecoveryWorker.ts | 15 +++++- 4 files changed, 106 insertions(+), 23 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 5f72055481452..70f8e238a8691 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5253,6 +5253,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { pendingRuntimeRequest: null, limitRecovery: sqlShell.limitRecovery, snoozedUntil: sqlShell.snoozedUntil, + parentSettled: false, }); } else assert.isUndefined(candidate); assert.isUndefined(candidates.find((row) => row.id === otherThreadId)); @@ -5700,7 +5701,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }, }); const afterReset = DateTime.add(now, { hours: 2 }); - const resumeCommandIds = Effect.fnUntraced(function* () { + const sweepCommands = Effect.fnUntraced(function* () { const candidates = yield* store.getLimitRecoveryCandidates({ now: afterReset, autoResume: true, @@ -5708,11 +5709,10 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }); return candidates .filter((row) => row.id === activeParentChild) - .map( + .flatMap( (row) => - limitRecoveryCommand(row, true, DateTime.toEpochMillis(afterReset), false)?.commandId, - ) - .filter((commandId) => commandId?.startsWith("limit-resume:")); + limitRecoveryCommand(row, true, DateTime.toEpochMillis(afterReset), false) ?? [], + ); }); yield* store.apply({ id: EventId.make("event:settled-parent-limit:active:parent-settled"), @@ -5721,22 +5721,44 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { occurredAt: now, payload: threadPayload(activeParentId, null, "settled"), }); - assert.deepEqual( - yield* resumeCommandIds(), - [], - "a subagent armed while its parent was open does not resume after the parent settles", + const [disarm, ...extraCommands] = yield* sweepCommands(); + assert.deepEqual(extraCommands, []); + assert( + disarm?.type === "thread.metadata.update" && disarm.commandId.startsWith("limit-disarm:"), + "a subagent armed while its parent was open is disarmed after the parent settles", ); + assert.deepEqual(disarm.limitRecovery, { + runId: activeRunId, + resetAt, + autoResume: false, + }); + yield* store.apply({ + id: EventId.make("event:settled-parent-limit:active:disarmed"), + type: "thread.metadata-updated", + threadId: activeParentChild, + occurredAt: afterReset, + payload: { + ...threadPayload(activeParentChild, activeParentId, null), + limitRecovery: { + runId: activeRunId, + resetAt, + autoResume: false, + snooze: false, + requestId: disarm.commandId, + }, + }, + }); yield* store.apply({ id: EventId.make("event:settled-parent-limit:active:parent-unsettled"), type: "thread.unsettled", threadId: activeParentId, - occurredAt: now, + occurredAt: afterReset, payload: threadPayload(activeParentId, null, "active"), }); - assert.lengthOf( - yield* resumeCommandIds(), - 1, - "the armed subagent resumes once its parent is unsettled", + assert.deepEqual( + yield* sweepCommands(), + [], + "unsettling the parent neither resumes nor re-arms the disarmed subagent", ); }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index e62de48678084..d5f85fc202b65 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -184,7 +184,7 @@ export type ProjectionLimitRecoveryCandidate = Pick< | "updatedAt" | "limitRecovery" | "snoozedUntil" ->; +> & { readonly parentSettled?: boolean }; export type ProjectionThreadPullRequests = Pick< OrchestrationV2AppThread, @@ -3906,10 +3906,12 @@ export const layer: Layer.Layer = readonly run_id: string; readonly completed_at: string | null; readonly failure_payload_json: string; + readonly parent_settled: number; readonly last_error: string | null; }>` SELECT t.payload_json, r.run_id, r.completed_at, item.payload_json AS failure_payload_json, + settled_parent.thread_id IS NOT NULL AS parent_settled, ( SELECT json_extract(session.payload_json, '$.lastError') -- Keep bindings outermost: one provider instance can hold every session. @@ -3943,6 +3945,10 @@ export const layer: Layer.Layer = ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC LIMIT 1 ) + LEFT JOIN orchestration_v2_projection_threads settled_parent + ON json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent' + AND settled_parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId') + AND json_extract(settled_parent.payload_json, '$.settledOverride') = 'settled' WHERE t.deleted_at IS NULL -- The sweep runs every scheduler tick. Without this list the latest-run -- lookup above sorts every run of every thread, tens of milliseconds @@ -3954,13 +3960,6 @@ export const layer: Layer.Layer = ) AND json_extract(t.payload_json, '$.archivedAt') IS NULL AND json_extract(t.payload_json, '$.settledOverride') IS NOT 'settled' - -- A subagent of a settled parent is abandoned work; arming or resuming it would wake the parent. - AND NOT EXISTS ( - SELECT 1 FROM orchestration_v2_projection_threads parent - WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent' - AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId') - AND json_extract(parent.payload_json, '$.settledOverride') = 'settled' - ) AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit' AND json_extract(item.payload_json, '$.failure.resetAt') IS NOT NULL AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) @@ -3984,6 +3983,8 @@ export const layer: Layer.Layer = ${booleanInt(options.autoResume)} OR (${booleanInt(options.snooze)} AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)})) ) + -- A subagent of a settled parent is abandoned work; arming it would wake the parent. + AND settled_parent.thread_id IS NULL ) ) AND NOT EXISTS ( @@ -4015,6 +4016,7 @@ export const layer: Layer.Layer = pendingRuntimeRequest: null, limitRecovery: thread.limitRecovery ?? null, snoozedUntil: thread.snoozedUntil ?? null, + parentSettled: row.parent_settled === 1, }); } return candidates; diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts index 430018adcb06a..0a2d6bf8c4510 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts @@ -116,3 +116,49 @@ describe("limitRecoveryCommand arming", () => { assert.isNull(limitRecoveryCommand(thread, true, nowMs)); }); }); + +it.effect("disarms an armed subagent of a settled parent through a plain dispatch", () => + Effect.gen(function* () { + yield* TestClock.setTime(nowMs); + const failed = failedThread(2 * HOUR_MS, HOUR_MS); + const thread: ProjectionStore.ProjectionLimitRecoveryCandidate = { + ...failed, + limitRecovery: { + runId: failed.latestRunId!, + resetAt: failed.usageLimitResetAt!, + autoResume: true, + snooze: false, + }, + parentSettled: true, + }; + const sent = yield* Queue.unbounded(); + const layer = UsageLimitRecoveryWorker.layer.pipe( + Layer.provide( + Layer.mergeAll( + Scheduler.layer, + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getLimitRecoveryCandidates: () => Effect.succeed([thread]), + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + dispatch: (command) => + Queue.offer(sent, `dispatch ${command.commandId}`).pipe(Effect.as({} as never)), + }), + // Arming skips a recovery the thread already holds, so a disarm sent here is lost. + Layer.mock(Orchestrator.OrchestratorV2)({ + streamDomainEvents: Stream.never, + autoArmLimitRecovery: (input) => Queue.offer(sent, `arm ${input.commandId}`), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + getSettings: Effect.succeed({ + autoResumeLimitedThreads: true, + snoozeLimitedThreads: false, + } as never), + streamChanges: Stream.never, + }), + ), + ), + ); + const first = yield* Queue.take(sent).pipe(Effect.provide(layer)); + assert.isTrue(first.startsWith(`dispatch limit-disarm:${thread.id}:`), first); + }), +); diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts index e46ac34c8be61..cf22064efb5ab 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts @@ -61,6 +61,14 @@ export function limitRecoveryCommand( ) return null; const deliveryIdentity = `${identity}:${recovery.requestId ?? "legacy"}`; + // A subagent of a settled parent is abandoned work: disarm it, as Cancel auto-resume would. + if (thread.parentSettled) + return { + type: "thread.metadata.update", + commandId: CommandId.make(`limit-disarm:${deliveryIdentity}`), + threadId: thread.id, + limitRecovery: { runId: recovery.runId, resetAt: recovery.resetAt, autoResume: false }, + }; return { type: "message.dispatch", commandId: CommandId.make(`limit-resume:${deliveryIdentity}`), @@ -121,7 +129,12 @@ const makeSweep = Effect.gen(function* () { if (command === null) continue; acted = true; yield* Effect.gen(function* () { - if (command.type === "thread.metadata.update" && command.limitRecovery != null) { + // Arming re-checks Stop races; a disarm changes the recovery the thread already holds. + if ( + command.type === "thread.metadata.update" && + command.limitRecovery != null && + command.commandId.startsWith("limit-arm:") + ) { yield* orchestrator.autoArmLimitRecovery({ commandId: command.commandId, threadId: command.threadId, From 32d7a77ccaa55f412a4a343f8380cf4600ebd946 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 3 Nov 2026 13:02:00 +0000 Subject: [PATCH 4/7] fix(server): saved subagent limit resume no longer starts after its parent settles The usage-limit sweep can read a subagent's candidates while its parent is open, and the parent can be settled before the saved limit-resume dispatch is decided. The continuation guard checked only the child's own settled state, so the child started and its completion wake reopened the settled parent. The guard now also refuses when the thread is a subagent whose direct parent is settled, taking the existing no-op path; the next sweep then sees parentSettled and disarms the recovery. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/Orchestrator.ts | 15 ++ .../src/orchestration-v2/runtimeLayer.test.ts | 140 ++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 600764704e57f..805a1a24f5e71 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -5418,6 +5418,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const failure = latestRootProviderFailure(run, projection.turnItems); const recovery = projection.thread.limitRecovery; const now = yield* DateTime.now; + const parentId = + projection.thread.lineage.relationshipToParent === "subagent" + ? projection.thread.lineage.parentThreadId + : null; + // The sweep may have read an open parent; resuming now would reopen a settled one. + const parentSettled = + parentId !== null && + (yield* projectionStore + .getThreadShell(parentId) + .pipe( + Effect.mapError( + (cause) => new OrchestratorProjectionError({ threadId: parentId, cause }), + ), + ))?.settledOverride === "settled"; if ( run?.id !== command.usageLimitContinuationOfRunId || failure?.class !== "usage_limit" || @@ -5430,6 +5444,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || projection.thread.settledOverride === "settled" || + parentSettled || projection.thread.providerInstanceId !== run.providerInstanceId || projection.runtimeRequests.some((request) => request.status === "pending") || (projection.thread.snoozedUntil != null && diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index b38fb737c91db..21b3c70f07c29 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -8524,6 +8524,146 @@ it.layer(layerTest)("usage-limit recovery", (it) => { } }), ); + + it.effect( + "does not resume a usage-limited subagent whose parent settled after the sweep read it", + () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const events = yield* EventSink.EventSinkV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const parentId = ThreadId.make("recovery:settled-parent:parent"); + const childId = ThreadId.make("recovery:settled-parent:child"); + const projectId = ProjectId.make("recovery:settled-parent:project"); + yield* seedProject({ + projectId, + title: "Recovery project", + workspaceRoot: process.cwd(), + defaultModelSelection: modelSelection, + createdAt: DateTime.formatIso(yield* DateTime.now), + }); + for (const threadId of [parentId, childId]) + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId, + title: "Limited thread", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("recovery:settled-parent:message"), + threadId: childId, + messageId: MessageId.make("recovery:settled-parent:message"), + text: "Work on this.", + attachments: [], + dispatchMode: { type: "defer_start" }, + createdBy: "user", + creationSource: "web", + }); + const child = yield* orchestrator.getThreadProjection(childId); + const run = child.runs[0]!; + const now = yield* DateTime.now; + const resetAt = DateTime.formatIso(DateTime.add(now, { minutes: 1 })); + yield* events.write({ + events: [ + { + id: EventId.make("recovery:settled-parent:lineage"), + type: "thread.metadata-updated", + threadId: childId, + occurredAt: now, + payload: { + ...child.thread, + lineage: { + parentThreadId: parentId, + rootThreadId: parentId, + relationshipToParent: "subagent", + }, + }, + }, + { + id: EventId.make("recovery:settled-parent:run"), + type: "run.updated", + threadId: childId, + occurredAt: now, + payload: { ...run, status: "failed", completedAt: now }, + }, + { + id: EventId.make("recovery:settled-parent:error"), + type: "turn-item.updated", + threadId: childId, + occurredAt: now, + payload: { + id: TurnItemId.make("recovery:settled-parent:error"), + type: "error", + threadId: childId, + runId: run.id, + nodeId: run.rootNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "failed", + title: "Usage limit reached", + startedAt: now, + completedAt: now, + updatedAt: now, + failure: { + class: "usage_limit", + message: "Plan limit reached.", + code: "usageLimitExceeded", + retryable: null, + resetAt, + }, + }, + }, + ], + }); + const sweep = Effect.fnUntraced(function* (nowMs: number) { + const candidate = (yield* projections.getLimitRecoveryCandidates({ + now: DateTime.makeUnsafe(nowMs), + autoResume: true, + snooze: false, + })).find((row) => row.id === childId); + return candidate === undefined ? null : limitRecoveryCommand(candidate, true, nowMs); + }); + const arm = yield* sweep(DateTime.toEpochMillis(now)); + assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null); + yield* orchestrator.autoArmLimitRecovery({ + commandId: arm.commandId, + threadId: childId, + limitRecovery: arm.limitRecovery, + }); + const resume = yield* sweep(Date.parse(resetAt)); + assert.equal(resume?.type, "message.dispatch"); + yield* orchestrator.dispatch({ + type: "thread.settle", + commandId: CommandId.make("recovery:settled-parent:settle"), + threadId: parentId, + }); + yield* TestClock.adjust("1 minute"); + yield* orchestrator.dispatch(resume!); + assert.lengthOf((yield* orchestrator.getThreadProjection(childId)).runs, 1); + assert.equal( + (yield* orchestrator.getThreadProjection(parentId)).thread.settledOverride, + "settled", + ); + const disarm = yield* sweep(DateTime.toEpochMillis(yield* DateTime.now)); + assert(disarm?.type === "thread.metadata.update"); + assert.isTrue(disarm.commandId.startsWith("limit-disarm:")); + yield* orchestrator.dispatch(disarm); + assert.isNull(yield* sweep(DateTime.toEpochMillis(yield* DateTime.now))); + assert.lengthOf((yield* orchestrator.getThreadProjection(childId)).runs, 1); + }), + ); }); const queueResumeDisabledInstanceProjectId = ProjectId.make( From 9d7834ca4e3e9e64bf422c2da224caf2e069417c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 3 Nov 2026 13:17:16 +0000 Subject: [PATCH 5/7] fix(server): settled parent retires its subagent's armed limit resume The previous shape split the cancellation across the recovery sweep, the worker and the orchestrator guard, which left two races: a skipped resume kept the same requestId and replayed its receipt forever, and a disarm prepared while the parent was settled could overwrite a newer explicit re-arm. Revert the store and worker to main and do the cancellation in the serialized resume dispatch: when the parent is settled and the recovery still matches the command, the no-op metadata update sets autoResume to false with a fresh requestId. Stale commands keep the existing no-op. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/Orchestrator.ts | 20 +- .../orchestration-v2/ProjectionStore.test.ts | 93 ++---- .../src/orchestration-v2/ProjectionStore.ts | 18 +- .../UsageLimitRecoveryWorker.test.ts | 46 --- .../UsageLimitRecoveryWorker.ts | 15 +- .../src/orchestration-v2/runtimeLayer.test.ts | 278 ++++++++++-------- docs/user/thread-sidebar.md | 3 +- 7 files changed, 200 insertions(+), 273 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 805a1a24f5e71..47a9d4e89dc39 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -5450,6 +5450,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (projection.thread.snoozedUntil != null && DateTime.toEpochMillis(projection.thread.snoozedUntil) > DateTime.toEpochMillis(now)) ) { + const recoveryMatches = + recovery?.autoResume && + recovery.requestId === command.usageLimitRecoveryRequestId && + recovery.runId === command.usageLimitContinuationOfRunId; yield* emit( events, command, @@ -5459,13 +5463,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio occurredAt: now, payload: { ...projection.thread, - ...(recovery?.autoResume && - recovery.requestId === command.usageLimitRecoveryRequestId && - recovery.runId === command.usageLimitContinuationOfRunId && - projection.thread.snoozedUntil != null && - DateTime.toEpochMillis(projection.thread.snoozedUntil) > DateTime.toEpochMillis(now) - ? { limitRecovery: { ...recovery, requestId: command.commandId } } - : {}), + // A settled parent retires the resume; leaving it armed would replay this receipt. + ...(recoveryMatches && parentSettled + ? { limitRecovery: { ...recovery, autoResume: false, requestId: command.commandId } } + : recoveryMatches && + projection.thread.snoozedUntil != null && + DateTime.toEpochMillis(projection.thread.snoozedUntil) > + DateTime.toEpochMillis(now) + ? { limitRecovery: { ...recovery, requestId: command.commandId } } + : {}), }, }); return; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 70f8e238a8691..5b24a65a99567 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5253,7 +5253,6 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { pendingRuntimeRequest: null, limitRecovery: sqlShell.limitRecovery, snoozedUntil: sqlShell.snoozedUntil, - parentSettled: false, }); } else assert.isUndefined(candidate); assert.isUndefined(candidates.find((row) => row.id === otherThreadId)); @@ -5548,7 +5547,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }), ); - it.effect("does not arm or resume limit recovery for a subagent of a settled parent", () => + it.effect("does not freshly arm limit recovery for a subagent of a settled parent", () => Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; const now = yield* DateTime.now; @@ -5556,7 +5555,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const threadPayload = ( threadId: ThreadId, parentThreadId: ThreadId | null, - settledOverride: "settled" | "active" | null, + settledOverride: "settled" | null, ) => ({ createdBy: "user", @@ -5581,7 +5580,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { updatedAt: now, archivedAt: null, settledOverride, - settledAt: settledOverride === "settled" ? now : null, + settledAt: settledOverride === null ? null : now, lastVisitedAt: null, deletedAt: null, }) as const; @@ -5688,77 +5687,39 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { `only the child of the unsettled parent is armed (${JSON.stringify(options)})`, ); } - const activeParentId = ThreadId.make("thread:settled-parent-limit:active:parent"); - const activeRunId = RunId.make("run:settled-parent-limit:active"); + // A recovery armed before the parent settled still resumes at its reset. yield* store.apply({ - id: EventId.make("event:settled-parent-limit:active:armed"), + id: EventId.make("event:settled-parent-limit:settled:armed"), type: "thread.metadata-updated", - threadId: activeParentChild, - occurredAt: now, - payload: { - ...threadPayload(activeParentChild, activeParentId, null), - limitRecovery: { runId: activeRunId, resetAt, autoResume: true }, - }, - }); - const afterReset = DateTime.add(now, { hours: 2 }); - const sweepCommands = Effect.fnUntraced(function* () { - const candidates = yield* store.getLimitRecoveryCandidates({ - now: afterReset, - autoResume: true, - snooze: false, - }); - return candidates - .filter((row) => row.id === activeParentChild) - .flatMap( - (row) => - limitRecoveryCommand(row, true, DateTime.toEpochMillis(afterReset), false) ?? [], - ); - }); - yield* store.apply({ - id: EventId.make("event:settled-parent-limit:active:parent-settled"), - type: "thread.settled", - threadId: activeParentId, + threadId: settledParentChild, occurredAt: now, - payload: threadPayload(activeParentId, null, "settled"), - }); - const [disarm, ...extraCommands] = yield* sweepCommands(); - assert.deepEqual(extraCommands, []); - assert( - disarm?.type === "thread.metadata.update" && disarm.commandId.startsWith("limit-disarm:"), - "a subagent armed while its parent was open is disarmed after the parent settles", - ); - assert.deepEqual(disarm.limitRecovery, { - runId: activeRunId, - resetAt, - autoResume: false, - }); - yield* store.apply({ - id: EventId.make("event:settled-parent-limit:active:disarmed"), - type: "thread.metadata-updated", - threadId: activeParentChild, - occurredAt: afterReset, payload: { - ...threadPayload(activeParentChild, activeParentId, null), + ...threadPayload( + settledParentChild, + ThreadId.make("thread:settled-parent-limit:settled:parent"), + null, + ), limitRecovery: { - runId: activeRunId, + runId: RunId.make("run:settled-parent-limit:settled"), resetAt, - autoResume: false, - snooze: false, - requestId: disarm.commandId, + autoResume: true, }, }, }); - yield* store.apply({ - id: EventId.make("event:settled-parent-limit:active:parent-unsettled"), - type: "thread.unsettled", - threadId: activeParentId, - occurredAt: afterReset, - payload: threadPayload(activeParentId, null, "active"), - }); - assert.deepEqual( - yield* sweepCommands(), - [], - "unsettling the parent neither resumes nor re-arms the disarmed subagent", + const afterReset = DateTime.add(now, { hours: 2 }); + const resumed = (yield* store.getLimitRecoveryCandidates({ + now: afterReset, + autoResume: true, + snooze: false, + })).find((row) => row.id === settledParentChild); + assert.isDefined(resumed); + assert.isTrue( + limitRecoveryCommand( + resumed, + true, + DateTime.toEpochMillis(afterReset), + false, + )?.commandId.startsWith("limit-resume:"), ); }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index d5f85fc202b65..3cffce15b70d6 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -184,7 +184,7 @@ export type ProjectionLimitRecoveryCandidate = Pick< | "updatedAt" | "limitRecovery" | "snoozedUntil" -> & { readonly parentSettled?: boolean }; +>; export type ProjectionThreadPullRequests = Pick< OrchestrationV2AppThread, @@ -3906,12 +3906,10 @@ export const layer: Layer.Layer = readonly run_id: string; readonly completed_at: string | null; readonly failure_payload_json: string; - readonly parent_settled: number; readonly last_error: string | null; }>` SELECT t.payload_json, r.run_id, r.completed_at, item.payload_json AS failure_payload_json, - settled_parent.thread_id IS NOT NULL AS parent_settled, ( SELECT json_extract(session.payload_json, '$.lastError') -- Keep bindings outermost: one provider instance can hold every session. @@ -3945,10 +3943,6 @@ export const layer: Layer.Layer = ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC LIMIT 1 ) - LEFT JOIN orchestration_v2_projection_threads settled_parent - ON json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent' - AND settled_parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId') - AND json_extract(settled_parent.payload_json, '$.settledOverride') = 'settled' WHERE t.deleted_at IS NULL -- The sweep runs every scheduler tick. Without this list the latest-run -- lookup above sorts every run of every thread, tens of milliseconds @@ -3983,8 +3977,13 @@ export const layer: Layer.Layer = ${booleanInt(options.autoResume)} OR (${booleanInt(options.snooze)} AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)})) ) - -- A subagent of a settled parent is abandoned work; arming it would wake the parent. - AND settled_parent.thread_id IS NULL + -- A subagent of a settled parent is abandoned work; resuming it would wake the parent. + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_threads parent + WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent' + AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId') + AND json_extract(parent.payload_json, '$.settledOverride') = 'settled' + ) ) ) AND NOT EXISTS ( @@ -4016,7 +4015,6 @@ export const layer: Layer.Layer = pendingRuntimeRequest: null, limitRecovery: thread.limitRecovery ?? null, snoozedUntil: thread.snoozedUntil ?? null, - parentSettled: row.parent_settled === 1, }); } return candidates; diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts index 0a2d6bf8c4510..430018adcb06a 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts @@ -116,49 +116,3 @@ describe("limitRecoveryCommand arming", () => { assert.isNull(limitRecoveryCommand(thread, true, nowMs)); }); }); - -it.effect("disarms an armed subagent of a settled parent through a plain dispatch", () => - Effect.gen(function* () { - yield* TestClock.setTime(nowMs); - const failed = failedThread(2 * HOUR_MS, HOUR_MS); - const thread: ProjectionStore.ProjectionLimitRecoveryCandidate = { - ...failed, - limitRecovery: { - runId: failed.latestRunId!, - resetAt: failed.usageLimitResetAt!, - autoResume: true, - snooze: false, - }, - parentSettled: true, - }; - const sent = yield* Queue.unbounded(); - const layer = UsageLimitRecoveryWorker.layer.pipe( - Layer.provide( - Layer.mergeAll( - Scheduler.layer, - Layer.mock(ProjectionStore.ProjectionStoreV2)({ - getLimitRecoveryCandidates: () => Effect.succeed([thread]), - }), - Layer.mock(ThreadManagement.ThreadManagementService)({ - dispatch: (command) => - Queue.offer(sent, `dispatch ${command.commandId}`).pipe(Effect.as({} as never)), - }), - // Arming skips a recovery the thread already holds, so a disarm sent here is lost. - Layer.mock(Orchestrator.OrchestratorV2)({ - streamDomainEvents: Stream.never, - autoArmLimitRecovery: (input) => Queue.offer(sent, `arm ${input.commandId}`), - }), - Layer.mock(ServerSettings.ServerSettingsService)({ - getSettings: Effect.succeed({ - autoResumeLimitedThreads: true, - snoozeLimitedThreads: false, - } as never), - streamChanges: Stream.never, - }), - ), - ), - ); - const first = yield* Queue.take(sent).pipe(Effect.provide(layer)); - assert.isTrue(first.startsWith(`dispatch limit-disarm:${thread.id}:`), first); - }), -); diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts index cf22064efb5ab..e46ac34c8be61 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts @@ -61,14 +61,6 @@ export function limitRecoveryCommand( ) return null; const deliveryIdentity = `${identity}:${recovery.requestId ?? "legacy"}`; - // A subagent of a settled parent is abandoned work: disarm it, as Cancel auto-resume would. - if (thread.parentSettled) - return { - type: "thread.metadata.update", - commandId: CommandId.make(`limit-disarm:${deliveryIdentity}`), - threadId: thread.id, - limitRecovery: { runId: recovery.runId, resetAt: recovery.resetAt, autoResume: false }, - }; return { type: "message.dispatch", commandId: CommandId.make(`limit-resume:${deliveryIdentity}`), @@ -129,12 +121,7 @@ const makeSweep = Effect.gen(function* () { if (command === null) continue; acted = true; yield* Effect.gen(function* () { - // Arming re-checks Stop races; a disarm changes the recovery the thread already holds. - if ( - command.type === "thread.metadata.update" && - command.limitRecovery != null && - command.commandId.startsWith("limit-arm:") - ) { + if (command.type === "thread.metadata.update" && command.limitRecovery != null) { yield* orchestrator.autoArmLimitRecovery({ commandId: command.commandId, threadId: command.threadId, diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 21b3c70f07c29..2dfec54539479 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -8525,144 +8525,164 @@ it.layer(layerTest)("usage-limit recovery", (it) => { }), ); - it.effect( - "does not resume a usage-limited subagent whose parent settled after the sweep read it", - () => - Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const events = yield* EventSink.EventSinkV2; - const projections = yield* ProjectionStore.ProjectionStoreV2; - const parentId = ThreadId.make("recovery:settled-parent:parent"); - const childId = ThreadId.make("recovery:settled-parent:child"); - const projectId = ProjectId.make("recovery:settled-parent:project"); - yield* seedProject({ - projectId, - title: "Recovery project", - workspaceRoot: process.cwd(), - defaultModelSelection: modelSelection, - createdAt: DateTime.formatIso(yield* DateTime.now), - }); - for (const threadId of [parentId, childId]) - yield* orchestrator.dispatch({ - type: "thread.create", - commandId: CommandId.make(`${threadId}:create`), - threadId, - projectId, - title: "Limited thread", - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: null, - createdBy: "user", - creationSource: "web", - }); + it.effect("retires a usage-limited subagent resume once its parent settles", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const events = yield* EventSink.EventSinkV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const parentId = ThreadId.make("recovery:settled-parent:parent"); + const childId = ThreadId.make("recovery:settled-parent:child"); + const projectId = ProjectId.make("recovery:settled-parent:project"); + yield* seedProject({ + projectId, + title: "Recovery project", + workspaceRoot: process.cwd(), + defaultModelSelection: modelSelection, + createdAt: DateTime.formatIso(yield* DateTime.now), + }); + for (const threadId of [parentId, childId]) yield* orchestrator.dispatch({ - type: "message.dispatch", - commandId: CommandId.make("recovery:settled-parent:message"), - threadId: childId, - messageId: MessageId.make("recovery:settled-parent:message"), - text: "Work on this.", - attachments: [], - dispatchMode: { type: "defer_start" }, + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId, + title: "Limited thread", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, createdBy: "user", creationSource: "web", }); - const child = yield* orchestrator.getThreadProjection(childId); - const run = child.runs[0]!; - const now = yield* DateTime.now; - const resetAt = DateTime.formatIso(DateTime.add(now, { minutes: 1 })); - yield* events.write({ - events: [ - { - id: EventId.make("recovery:settled-parent:lineage"), - type: "thread.metadata-updated", - threadId: childId, - occurredAt: now, - payload: { - ...child.thread, - lineage: { - parentThreadId: parentId, - rootThreadId: parentId, - relationshipToParent: "subagent", - }, + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("recovery:settled-parent:message"), + threadId: childId, + messageId: MessageId.make("recovery:settled-parent:message"), + text: "Work on this.", + attachments: [], + dispatchMode: { type: "defer_start" }, + createdBy: "user", + creationSource: "web", + }); + const child = yield* orchestrator.getThreadProjection(childId); + const run = child.runs[0]!; + const now = yield* DateTime.now; + const resetAt = DateTime.formatIso(DateTime.add(now, { minutes: 1 })); + yield* events.write({ + events: [ + { + id: EventId.make("recovery:settled-parent:lineage"), + type: "thread.metadata-updated", + threadId: childId, + occurredAt: now, + payload: { + ...child.thread, + lineage: { + parentThreadId: parentId, + rootThreadId: parentId, + relationshipToParent: "subagent", }, }, - { - id: EventId.make("recovery:settled-parent:run"), - type: "run.updated", - threadId: childId, - occurredAt: now, - payload: { ...run, status: "failed", completedAt: now }, - }, - { - id: EventId.make("recovery:settled-parent:error"), - type: "turn-item.updated", + }, + { + id: EventId.make("recovery:settled-parent:run"), + type: "run.updated", + threadId: childId, + occurredAt: now, + payload: { ...run, status: "failed", completedAt: now }, + }, + { + id: EventId.make("recovery:settled-parent:error"), + type: "turn-item.updated", + threadId: childId, + occurredAt: now, + payload: { + id: TurnItemId.make("recovery:settled-parent:error"), + type: "error", threadId: childId, - occurredAt: now, - payload: { - id: TurnItemId.make("recovery:settled-parent:error"), - type: "error", - threadId: childId, - runId: run.id, - nodeId: run.rootNodeId, - providerThreadId: null, - providerTurnId: null, - nativeItemRef: null, - parentItemId: null, - ordinal: 2, - status: "failed", - title: "Usage limit reached", - startedAt: now, - completedAt: now, - updatedAt: now, - failure: { - class: "usage_limit", - message: "Plan limit reached.", - code: "usageLimitExceeded", - retryable: null, - resetAt, - }, + runId: run.id, + nodeId: run.rootNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "failed", + title: "Usage limit reached", + startedAt: now, + completedAt: now, + updatedAt: now, + failure: { + class: "usage_limit", + message: "Plan limit reached.", + code: "usageLimitExceeded", + retryable: null, + resetAt, }, }, - ], - }); - const sweep = Effect.fnUntraced(function* (nowMs: number) { - const candidate = (yield* projections.getLimitRecoveryCandidates({ - now: DateTime.makeUnsafe(nowMs), - autoResume: true, - snooze: false, - })).find((row) => row.id === childId); - return candidate === undefined ? null : limitRecoveryCommand(candidate, true, nowMs); - }); - const arm = yield* sweep(DateTime.toEpochMillis(now)); - assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null); - yield* orchestrator.autoArmLimitRecovery({ - commandId: arm.commandId, - threadId: childId, - limitRecovery: arm.limitRecovery, - }); - const resume = yield* sweep(Date.parse(resetAt)); - assert.equal(resume?.type, "message.dispatch"); - yield* orchestrator.dispatch({ - type: "thread.settle", - commandId: CommandId.make("recovery:settled-parent:settle"), - threadId: parentId, - }); - yield* TestClock.adjust("1 minute"); - yield* orchestrator.dispatch(resume!); - assert.lengthOf((yield* orchestrator.getThreadProjection(childId)).runs, 1); - assert.equal( - (yield* orchestrator.getThreadProjection(parentId)).thread.settledOverride, - "settled", - ); - const disarm = yield* sweep(DateTime.toEpochMillis(yield* DateTime.now)); - assert(disarm?.type === "thread.metadata.update"); - assert.isTrue(disarm.commandId.startsWith("limit-disarm:")); - yield* orchestrator.dispatch(disarm); - assert.isNull(yield* sweep(DateTime.toEpochMillis(yield* DateTime.now))); - assert.lengthOf((yield* orchestrator.getThreadProjection(childId)).runs, 1); - }), + }, + ], + }); + const sweep = Effect.fnUntraced(function* (nowMs: number) { + const candidate = (yield* projections.getLimitRecoveryCandidates({ + now: DateTime.makeUnsafe(nowMs), + autoResume: true, + snooze: false, + })).find((row) => row.id === childId); + return candidate === undefined ? null : limitRecoveryCommand(candidate, true, nowMs); + }); + const arm = yield* sweep(DateTime.toEpochMillis(now)); + assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null); + yield* orchestrator.autoArmLimitRecovery({ + commandId: arm.commandId, + threadId: childId, + limitRecovery: arm.limitRecovery, + }); + const resume = yield* sweep(Date.parse(resetAt)); + assert.equal(resume?.type, "message.dispatch"); + yield* orchestrator.dispatch({ + type: "thread.settle", + commandId: CommandId.make("recovery:settled-parent:settle"), + threadId: parentId, + }); + yield* TestClock.adjust("1 minute"); + yield* orchestrator.dispatch(resume!); + let current = yield* orchestrator.getThreadProjection(childId); + assert.lengthOf(current.runs, 1); + assert.isFalse(current.thread.limitRecovery!.autoResume); + assert.equal( + (yield* orchestrator.getThreadProjection(parentId)).thread.settledOverride, + "settled", + ); + yield* orchestrator.dispatch({ + type: "thread.unsettle", + commandId: CommandId.make("recovery:settled-parent:unsettle"), + threadId: parentId, + reason: "user", + }); + assert.isNull(yield* sweep(DateTime.toEpochMillis(yield* DateTime.now))); + yield* orchestrator.dispatch(resume!); + current = yield* orchestrator.getThreadProjection(childId); + assert.lengthOf(current.runs, 1); + assert.isFalse(current.thread.limitRecovery!.autoResume); + yield* orchestrator.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("recovery:settled-parent:rearm"), + threadId: childId, + limitRecovery: { runId: run.id, resetAt, autoResume: true }, + }); + const rearmed = (yield* orchestrator.getThreadProjection(childId)).thread.limitRecovery; + assert.isTrue(rearmed!.autoResume); + yield* orchestrator.dispatch({ + ...resume!, + commandId: CommandId.make("recovery:settled-parent:stale-resume"), + }); + current = yield* orchestrator.getThreadProjection(childId); + assert.lengthOf(current.runs, 1); + assert.deepEqual(current.thread.limitRecovery, rearmed); + }), ); }); diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index bed4ed73a40c5..00fae66c05056 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -249,7 +249,8 @@ continuation. You can cancel it from the thread. Enable **Resume after usage limits** in **Settings → Threads** to schedule limit stops by default. The environment must be running when the reset arrives; it resumes overdue continuations after a restart. Stopping the thread, sending a new message, -archiving, or settling it prevents a pending continuation from starting. +archiving, or settling it prevents a pending continuation from starting; +settling also cancels the pending continuations of tasks it delegated. Choose **Snooze until reset** to hide the thread until its allowance returns. Snooze and auto-resume are independent: snooze alone wakes the thread without From 1effe8b9d5e762fbac91b1f67ea46d66a638d49b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 3 Nov 2026 13:18:18 +0000 Subject: [PATCH 6/7] fix(server): describe the settled-parent resume guard as the primary gate Co-Authored-By: Claude Opus 5.5 --- apps/server/src/orchestration-v2/Orchestrator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 47a9d4e89dc39..8eb3362af0c4e 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -5422,7 +5422,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection.thread.lineage.relationshipToParent === "subagent" ? projection.thread.lineage.parentThreadId : null; - // The sweep may have read an open parent; resuming now would reopen a settled one. + // A subagent of a settled parent is abandoned work; resuming it would reopen the parent. const parentSettled = parentId !== null && (yield* projectionStore From 0b048fa480d46da8c4c23114f9b01bbcc37f6a9d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 3 Nov 2026 13:30:41 +0000 Subject: [PATCH 7/7] docs: a delegated task's continuation is cancelled when its parent is settled at the reset Co-Authored-By: Claude Opus 5.5 --- docs/user/thread-sidebar.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index 00fae66c05056..3fc960625552c 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -249,8 +249,9 @@ continuation. You can cancel it from the thread. Enable **Resume after usage limits** in **Settings → Threads** to schedule limit stops by default. The environment must be running when the reset arrives; it resumes overdue continuations after a restart. Stopping the thread, sending a new message, -archiving, or settling it prevents a pending continuation from starting; -settling also cancels the pending continuations of tasks it delegated. +archiving, or settling it prevents a pending continuation from starting. A +task the thread delegated does not continue at its reset while the thread is +settled; its continuation is cancelled instead. Choose **Snooze until reset** to hide the thread until its allowance returns. Snooze and auto-resume are independent: snooze alone wakes the thread without