diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts index 2e077314b8212..430018adcb06a 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts @@ -1,5 +1,5 @@ -import { assert, it } from "@effect/vitest"; -import type { OrchestrationV2DomainEvent } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import { RunId, ThreadId, type OrchestrationV2DomainEvent } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -14,6 +14,7 @@ import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts"; +import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts"; it.effect("idle sweeps skip SQL until the next wake time or a relevant event", () => Effect.gen(function* () { @@ -68,3 +69,50 @@ it.effect("idle sweeps skip SQL until the next wake time or a relevant event", ( }).pipe(Effect.provide(layer)); }), ); + +const HOUR_MS = 60 * 60 * 1000; +const nowMs = Date.parse("2026-10-19T12:00:00.000Z"); + +const failedThread = (failedAgoMs: number, resetAgoMs: number) => { + const failedAt = DateTime.makeUnsafe(nowMs - failedAgoMs); + const candidate: ProjectionStore.ProjectionLimitRecoveryCandidate = { + id: ThreadId.make("thread:limited"), + status: "failed", + lastErrorClass: "usage_limit", + usageLimitResetAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - resetAgoMs)), + latestRunId: RunId.make("run:limited"), + latestRunCompletedAt: failedAt, + updatedAt: failedAt, + archivedAt: null, + settledOverride: null, + pendingRuntimeRequest: null, + limitRecovery: null, + snoozedUntil: null, + }; + return candidate; +}; + +describe("limitRecoveryCommand arming", () => { + it("does not freshly arm a failure whose reset passed days ago", () => { + const thread = failedThread(5 * 24 * HOUR_MS, 4 * 24 * HOUR_MS); + assert.isNull(limitRecoveryCommand(thread, true, nowMs)); + assert.isNull(limitRecoveryCommand(thread, true, nowMs, true)); + }); + + it("still arms a failure whose reset only just passed", () => { + const thread = failedThread(6 * HOUR_MS, HOUR_MS); + const arm = limitRecoveryCommand(thread, true, nowMs); + assert.equal(arm?.type, "thread.metadata.update"); + }); + + it("arms a reset that is exactly 24 hours overdue", () => { + const thread = failedThread(30 * HOUR_MS, 24 * HOUR_MS); + const arm = limitRecoveryCommand(thread, true, nowMs); + assert.equal(arm?.type, "thread.metadata.update"); + }); + + it("does not arm a reset that is 24 hours and 1ms overdue", () => { + const thread = failedThread(30 * HOUR_MS, 24 * HOUR_MS + 1); + assert.isNull(limitRecoveryCommand(thread, true, nowMs)); + }); +}); diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts index 224eecbbe5f69..e46ac34c8be61 100644 --- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts +++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts @@ -12,6 +12,8 @@ import * as ProjectionStore from "./ProjectionStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; +const MAX_OVERDUE_ARM_MS = 24 * 60 * 60 * 1000; + export function limitRecoveryCommand( thread: ProjectionStore.ProjectionLimitRecoveryCandidate, autoResume: boolean, @@ -38,6 +40,7 @@ export function limitRecoveryCommand( const identity = `${thread.id}:${thread.latestRunId}:${resetMs}`; const recovery = thread.limitRecovery; if (recovery?.runId !== thread.latestRunId || recovery.resetAt !== thread.usageLimitResetAt) { + if (nowMs - resetMs > MAX_OVERDUE_ARM_MS) return null; if (!autoResume && (!snooze || resetMs <= nowMs)) return null; return { type: "thread.metadata.update",