From c0dd17ba85031f615770120f3f259c59960d180d Mon Sep 17 00:00:00 2001 From: fixfon <12512757+fixfon@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:56:45 +0300 Subject: [PATCH 1/2] fix(server): do not freshly arm days-old usage-limit recoveries With auto-resume on, the sweep armed any failed usage-limit thread that had no matching arm as long as its reset was after the failure, even when that reset had passed days earlier. The next sweep then dispatched "Continue where you left off." right away, so a server update resumed long-abandoned tasks. Arming now requires the reset to be at most 24 hours in the past, the smallest whole-day window that covers the reported multi-day-old failures while still letting a reset that just passed (e.g. after a short outage) arm. Closes #21716 --- .../UsageLimitRecoveryWorker.test.ts | 41 ++++++++++++++++++- .../UsageLimitRecoveryWorker.ts | 3 ++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts index 2e077314b8212..0848156157175 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts @@ -1,5 +1,5 @@ -import { assert, it } from "@effect/vitest"; -import type { OrchestrationV2DomainEvent } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import { RunId, ThreadId, type OrchestrationV2DomainEvent } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -14,6 +14,7 @@ import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts"; +import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts"; it.effect("idle sweeps skip SQL until the next wake time or a relevant event", () => Effect.gen(function* () { @@ -68,3 +69,39 @@ it.effect("idle sweeps skip SQL until the next wake time or a relevant event", ( }).pipe(Effect.provide(layer)); }), ); + +const HOUR_MS = 60 * 60 * 1000; +const nowMs = Date.parse("2026-10-19T12:00:00.000Z"); + +const failedThread = (failedAgoMs: number, resetAgoMs: number) => { + const failedAt = DateTime.makeUnsafe(nowMs - failedAgoMs); + const candidate: ProjectionStore.ProjectionLimitRecoveryCandidate = { + id: ThreadId.make("thread:limited"), + status: "failed", + lastErrorClass: "usage_limit", + usageLimitResetAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - resetAgoMs)), + latestRunId: RunId.make("run:limited"), + latestRunCompletedAt: failedAt, + updatedAt: failedAt, + archivedAt: null, + settledOverride: null, + pendingRuntimeRequest: null, + limitRecovery: null, + snoozedUntil: null, + }; + return candidate; +}; + +describe("limitRecoveryCommand arming", () => { + it("does not freshly arm a failure whose reset passed days ago", () => { + const thread = failedThread(5 * 24 * HOUR_MS, 4 * 24 * HOUR_MS); + assert.isNull(limitRecoveryCommand(thread, true, nowMs)); + assert.isNull(limitRecoveryCommand(thread, true, nowMs, true)); + }); + + it("still arms a failure whose reset only just passed", () => { + const thread = failedThread(6 * HOUR_MS, HOUR_MS); + const arm = limitRecoveryCommand(thread, true, nowMs); + assert.equal(arm?.type, "thread.metadata.update"); + }); +}); diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts index 224eecbbe5f69..e46ac34c8be61 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts @@ -12,6 +12,8 @@ import * as ProjectionStore from "./ProjectionStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; +const MAX_OVERDUE_ARM_MS = 24 * 60 * 60 * 1000; + export function limitRecoveryCommand( thread: ProjectionStore.ProjectionLimitRecoveryCandidate, autoResume: boolean, @@ -38,6 +40,7 @@ export function limitRecoveryCommand( const identity = `${thread.id}:${thread.latestRunId}:${resetMs}`; const recovery = thread.limitRecovery; if (recovery?.runId !== thread.latestRunId || recovery.resetAt !== thread.usageLimitResetAt) { + if (nowMs - resetMs > MAX_OVERDUE_ARM_MS) return null; if (!autoResume && (!snooze || resetMs <= nowMs)) return null; return { type: "thread.metadata.update", From c69180bf871e55a1e561b65cd2ea582187c84745 Mon Sep 17 00:00:00 2001 From: fixfon <12512757+fixfon@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:28:24 +0300 Subject: [PATCH 2/2] test(server): cover usage-limit arm overdue boundary Pin the 24-hour cutoff on both sides: a reset exactly 24 hours overdue still arms, and one 24 hours and 1ms overdue does not. --- .../orchestration-v2/UsageLimitRecoveryWorker.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts index 0848156157175..430018adcb06a 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts @@ -104,4 +104,15 @@ describe("limitRecoveryCommand arming", () => { const arm = limitRecoveryCommand(thread, true, nowMs); assert.equal(arm?.type, "thread.metadata.update"); }); + + it("arms a reset that is exactly 24 hours overdue", () => { + const thread = failedThread(30 * HOUR_MS, 24 * HOUR_MS); + const arm = limitRecoveryCommand(thread, true, nowMs); + assert.equal(arm?.type, "thread.metadata.update"); + }); + + it("does not arm a reset that is 24 hours and 1ms overdue", () => { + const thread = failedThread(30 * HOUR_MS, 24 * HOUR_MS + 1); + assert.isNull(limitRecoveryCommand(thread, true, nowMs)); + }); });