diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index 5f945abd1858c..6d87a03e55a76 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -112,14 +112,16 @@ describe("V2 storage cleanup eligibility", () => { const candidate = () => shell({ branch: "feature", worktreePath: "/worktrees/feature" }); it("allows an idle worktree and rejects the project checkout", () => { - expect(storageCleanupThreadIdle(candidate(), NOW_MS)).toBe(true); - expect(storageCleanupThreadIdle(shell(), NOW_MS)).toBe(false); + expect(storageCleanupThreadIdle(candidate(), NOW_MS, DEFAULT_SERVER_SETTINGS)).toBe(true); + expect(storageCleanupThreadIdle(shell(), NOW_MS, DEFAULT_SERVER_SETTINGS)).toBe(false); }); it.each(["running", "starting", "preparing", "waiting", "queued"] as const)( "retains a worktree while its thread is %s", (status) => { - expect(storageCleanupThreadIdle(candidateWithStatus(status), NOW_MS)).toBe(false); + expect( + storageCleanupThreadIdle(candidateWithStatus(status), NOW_MS, DEFAULT_SERVER_SETTINGS), + ).toBe(false); }, ); @@ -133,6 +135,7 @@ describe("V2 storage cleanup eligibility", () => { pendingBackgroundTasks: [{ taskId: "task-1", kind: "command" }], }, NOW_MS, + DEFAULT_SERVER_SETTINGS, ), ).toBe(false); }, @@ -152,6 +155,7 @@ describe("V2 storage cleanup eligibility", () => { }, }, NOW_MS, + DEFAULT_SERVER_SETTINGS, ), ).toBe(false); }, @@ -159,16 +163,83 @@ describe("V2 storage cleanup eligibility", () => { it("retains an active run even if the shell status is idle", () => { expect( - storageCleanupThreadIdle({ ...candidate(), activeRunId: RunId.make("run") }, NOW_MS), + storageCleanupThreadIdle( + { ...candidate(), activeRunId: RunId.make("run") }, + NOW_MS, + DEFAULT_SERVER_SETTINGS, + ), ).toBe(false); }); it("retains a queued prompt before the new run has been projected", () => { expect( - storageCleanupThreadIdle({ ...candidate(), latestUserMessageAt: at(-1_000) }, NOW_MS), + storageCleanupThreadIdle( + { ...candidate(), latestUserMessageAt: at(-1_000) }, + NOW_MS, + DEFAULT_SERVER_SETTINGS, + ), ).toBe(false); }); + it.each([ + { name: "snoozed until a future reset", snooze: true, resetIn: DAY_MS, idle: false }, + { name: "set to auto-resume", autoResume: true, resetIn: DAY_MS, idle: false }, + { name: "with recovery turned off", resetIn: DAY_MS, idle: true }, + { name: "auto-resuming after the reset", autoResume: true, resetIn: -1_000, idle: false }, + { name: "snoozed past its reset", snooze: true, resetIn: -1_000, idle: true }, + { + name: "settled with an auto-resume pending", + autoResume: true, + settled: true, + resetIn: -1_000, + idle: true, + }, + { name: "not yet armed by the recovery worker", armed: false, resetIn: DAY_MS, idle: false }, + { + name: "not yet armed, auto-resume after reset", + armed: false, + autoResumeSetting: true, + resetIn: -1_000, + idle: false, + }, + { + name: "not armed with recovery settings off", + armed: false, + settingsOff: true, + resetIn: DAY_MS, + idle: true, + }, + ])( + "treats a usage-limited thread $name as idle: $idle", + ({ + autoResume = false, + snooze = false, + armed = true, + autoResumeSetting = false, + settled, + settingsOff, + resetIn, + idle, + }) => { + const runId = RunId.make("run-1"); + const resetAt = DateTime.formatIso(at(resetIn)); + expect( + storageCleanupThreadIdle( + { + ...candidateWithStatus("failed"), + latestRunId: runId, + lastErrorClass: "usage_limit", + usageLimitResetAt: resetAt, + settledOverride: settled ? "settled" : null, + limitRecovery: armed ? { runId, resetAt, autoResume, snooze } : null, + }, + NOW_MS, + { autoResumeLimitedThreads: autoResumeSetting, snoozeLimitedThreads: !settingsOff }, + ), + ).toBe(idle); + }, + ); + it("uses V2 run activity instead of metadata refreshes for retention", () => { const thread = candidate(); const runTime = at(-3 * DAY_MS); @@ -2211,6 +2282,33 @@ describe("storage cleanup reports and local file policies", () => { }); describe("settled worktree rechecks", () => { + it.live.each(["autoResumeLimitedThreads", "snoozeLimitedThreads"] as const)( + "keeps a usage-limited checkout when %s is enabled during measurement", + (preference) => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* pausedMeasurementFixture(); + fixture.setSettings({ autoResumeLimitedThreads: false, snoozeLimitedThreads: false }); + fixture.setThreads([ + shell({ + branch: "feature", + worktreePath: fixture.worktree, + status: "failed", + latestRunId: RunId.make("limited-run"), + lastErrorClass: "usage_limit", + usageLimitResetAt: DateTime.formatIso(at(DAY_MS)), + }), + ]); + const run = yield* fixture.service.runNow.pipe(Effect.forkChild); + yield* Deferred.await(fixture.measuring); + fixture.setSettings({ [preference]: true }); + yield* Deferred.succeed(fixture.resume, undefined); + const report = yield* Fiber.join(run); + expect(report.entries[0]?.outcome).toBe("kept"); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); + }), + ), + ); it.live.each([{ settledAt: null }, { settledAt: at(DAY_MS) }])( "keeps a checkout changed during measurement (%j)", (change) => diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index 48154835e7bb4..deffe41cec571 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -115,7 +115,16 @@ function sameProjectWorktreePolicies(left: ServerSettings, right: ServerSettings ); } -export function storageCleanupThreadIdle(thread: OrchestrationV2ThreadShell, now: number): boolean { +type LimitRecoverySettings = Pick< + ServerSettings, + "autoResumeLimitedThreads" | "snoozeLimitedThreads" +>; + +export function storageCleanupThreadIdle( + thread: OrchestrationV2ThreadShell, + now: number, + settings: LimitRecoverySettings, +): boolean { return ( thread.worktreePath !== null && thread.activeRunId === null && @@ -127,10 +136,41 @@ export function storageCleanupThreadIdle(thread: OrchestrationV2ThreadShell, now thread.status === "rolled_back") && (thread.pendingBackgroundTasks?.length ?? 0) === 0 && thread.pendingRuntimeRequest === null && - !threadHasQueuedTurnStart(thread, now) + !threadHasQueuedTurnStart(thread, now) && + !heldForLimitRecovery(thread, now, settings) ); } +// A run held for its usage-limit reset has its work still ahead of it, and a +// fresh worktree would otherwise look unchanged and be removed before it runs. +function heldForLimitRecovery( + thread: OrchestrationV2ThreadShell, + now: number, + settings: LimitRecoverySettings, +): boolean { + if ( + thread.status !== "failed" || + thread.lastErrorClass !== "usage_limit" || + !thread.usageLimitResetAt || + thread.archivedAt !== null || + thread.settledOverride === "settled" + ) + return false; + const resetMs = Date.parse(thread.usageLimitResetAt); + const recovery = thread.limitRecovery; + if (recovery?.runId === thread.latestRunId && recovery.resetAt === thread.usageLimitResetAt) + // An auto-resume holds until its continuation replaces the failed run. + return recovery.autoResume || (recovery.snooze === true && resetMs > now); + // The recovery worker arms this failure on a later tick; hold it until then. + // Like the worker, skip a reset that had already passed when the run ended. + if ( + !Number.isFinite(resetMs) || + resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt) + ) + return false; + return settings.autoResumeLimitedThreads || (settings.snoozeLimitedThreads && resetMs > now); +} + /** PR metadata refreshes must not reset the inactivity clock. */ export function storageCleanupActivityAt(thread: OrchestrationV2ThreadShell): number { return Math.max( @@ -799,7 +839,7 @@ const make = Effect.gen(function* () { if (project === undefined) return keep("project is unavailable"); if ( thread.branch === null || - !group.every((member) => storageCleanupThreadIdle(member, now)) + !group.every((member) => storageCleanupThreadIdle(member, now, serverSettings)) ) return keep(waitForStop("thread is running or has pending work")); if (group.some((member) => member.branch !== thread.branch)) @@ -968,7 +1008,7 @@ const make = Effect.gen(function* () { shell.branch !== thread.branch || !Equal.equals(shell.settledAt, previous.settledAt) || (shell.settledAt !== null && !Equal.equals(shell.updatedAt, previous.updatedAt)) || - !storageCleanupThreadIdle(shell, now) || + !storageCleanupThreadIdle(shell, now, serverSettings) || path.resolve(shell.worktreePath!) !== worktreePath || storageCleanupActivityAt(shell) !== storageCleanupActivityAt(previous) ); @@ -1008,11 +1048,11 @@ const make = Effect.gen(function* () { return keep("commit changed since check"); const finalChanges = yield* localChangesReason(worktreePath, settings); if (finalChanges !== null) return keep(finalChanges); - const current = resolveWorktreeCleanup( - yield* settingsService.getSettings, - thread.projectId, - ); + const currentSettings = yield* settingsService.getSettings; + const current = resolveWorktreeCleanup(currentSettings, thread.projectId); if ( + currentSettings.autoResumeLimitedThreads !== serverSettings.autoResumeLimitedThreads || + currentSettings.snoozeLimitedThreads !== serverSettings.snoozeLimitedThreads || Object.keys(settings).some( (key) => current[key as keyof typeof settings] !== settings[key as keyof typeof settings],