From e58a4bcef569ef46467506d285b2b4a3f9b0a6a9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:53:05 +1100 Subject: [PATCH 1/5] fix(server): limited turns through CLIProxyAPI resume when the hub refills CLIProxyAPI reports a spent pool as a 429 with a reset time, but Claude Code and Codex drop that time, so T3 marked the thread Limited with no reset and Resume at reset, auto-resume, and snooze could not start. When a usage-limit terminal has no reset time, the ingestor now asks the connected CLIProxyAPI hubs. If every account a hub pools for that provider is spent, the reset is when the first one refills. Co-Authored-By: Claude Opus 5.5 --- .../ProviderEventIngestor.test.ts | 32 +++++++- .../orchestration-v2/ProviderEventIngestor.ts | 10 ++- apps/server/src/server.ts | 1 + .../src/usage/UsageLimitSources.test.ts | 73 ++++++++++++++++++- apps/server/src/usage/UsageLimitSources.ts | 50 +++++++++++++ docs/user/usage.md | 5 ++ 6 files changed, 164 insertions(+), 7 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 7f1006f1c5ec5..a6bf3b4d77903 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -47,6 +47,7 @@ import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; +import * as UsageLimitSources from "../usage/UsageLimitSources.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; @@ -233,6 +234,16 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit Effect.scoped( Effect.gen(function* () { const refreshed = yield* Deferred.make(); + const hubResetAt = "2026-10-30T05:00:00.000Z"; + const nativeResetAt = "2026-10-30T03:00:00.000Z"; + const fallbackDrivers: ProviderDriverKind[] = []; + const layerResetFallback = Layer.succeed(UsageLimitSources.UsageLimitResetFallback, { + resetAt: (driver) => + Effect.sync(() => { + fallbackDrivers.push(driver); + return hubResetAt; + }), + }); const adapter = yield* ProviderAdapter.ProviderAdapterV2.pipe( Effect.provide( Layer.mock(ProviderAdapter.ProviderAdapterV2)({ @@ -290,7 +301,11 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit providerInstanceId: instance.instanceId, threadId: threadEvent.threadId, }; - for (const failureClass of ["validation_error", "usage_limit"] as const) { + for (const [failureClass, resetAt, expectedResetAt] of [ + ["validation_error", undefined, undefined], + ["usage_limit", undefined, hubResetAt], + ["usage_limit", nativeResetAt, nativeResetAt], + ] as const) { const stored = yield* ingestor.ingestNormalized({ ...input, event: { @@ -302,7 +317,7 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit }), providerTurnId: idAllocator.derive.providerTurn({ driver: instance.driverKind, - nativeTurnId: failureClass, + nativeTurnId: `${failureClass}:${resetAt ?? "unknown"}`, }), runOrdinal: 1, failureItemOrdinal: 1, @@ -310,16 +325,25 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit failure: makeProviderFailure({ class: failureClass, message: "Provider rejected turn.", + ...(resetAt === undefined ? {} : { resetAt }), }), threadDisposition: "reusable", }, }); assert.equal(stored.length, 1); + const event = stored[0]?.event; + const item = event?.type === "turn-item.updated" ? event.payload : undefined; + assert.equal(item?.type === "error" ? item.failure.resetAt : null, expectedResetAt); if (failureClass === "validation_error") assert.deepEqual(lookups, []); } - assert.deepEqual(lookups, [instance.instanceId]); + assert.deepEqual(lookups, [instance.instanceId, instance.instanceId]); + assert.deepEqual(fallbackDrivers, [instance.driverKind]); yield* Deferred.await(refreshed); - }).pipe(Effect.provide(makeLayerTest(layerTestEventSink, instances))); + }).pipe( + Effect.provide( + makeLayerTest(layerTestEventSink, instances).pipe(Layer.provide(layerResetFallback)), + ), + ); }), ), ); diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 06f444776cb3c..41b4c019e1856 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -35,6 +35,7 @@ import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3too import * as AnalyticsService from "../telemetry/AnalyticsService.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; +import { UsageLimitResetFallback } from "../usage/UsageLimitSources.ts"; import * as EventSink from "./EventSink.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; @@ -339,6 +340,7 @@ export const layer: Layer.Layer< const analytics = yield* ProviderTurnAnalytics; const agentScope = yield* AgentScope; const instances = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; + const usageLimitResets = yield* UsageLimitResetFallback; const scope = yield* Effect.scope; const completedTurnAnalytics = new Set(); @@ -709,9 +711,15 @@ export const layer: Layer.Layer< return dismissed; } const occurredAt = yield* DateTime.now; - const failure = (yield* agentScope.oomKilled(input.threadId)) + const reported = (yield* agentScope.oomKilled(input.threadId)) ? { ...input.event.failure, message: "Killed: out of memory.", code: "oom_kill" } : input.event.failure; + const fallbackResetAt = + reported.class === "usage_limit" && reported.resetAt == null + ? yield* usageLimitResets.resetAt(input.event.driver) + : null; + const failure = + fallbackResetAt === null ? reported : { ...reported, resetAt: fallbackResetAt }; return [ ...dismissed, yield* makeDomainEvent(input, { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 741fc73103b7d..cb82125af316c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -520,6 +520,7 @@ const layerScheduledTaskWebhookOrigin = Layer.effect( const layerOrchestrationV2Runtime = RuntimeLayer.layerProduction.pipe( Layer.provide(layerScheduledTaskWebhookOrigin), Layer.provide(ProviderEventIngestor.layerAnalytics), + Layer.provide(UsageLimitSources.layerResetFallback), Layer.provide(layerCheckpointStore), Layer.provide(layerGitWorkflow), Layer.provide( diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index b8976fca81e7d..050ed74bc6af1 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -1,6 +1,13 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; -import { expect, it } from "@effect/vitest"; -import { ServerConfigStreamEvent, UsageLimitSourceId } from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import { + ProviderDriverKind, + ServerConfigStreamEvent, + UsageLimitSourceId, + type ServerProviderUsageWindow, + type UsageLimitSourceSnapshot, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; @@ -83,3 +90,65 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => Effect.scoped, ), ); + +describe("hubPoolResetAt", () => { + const now = Date.parse("2026-10-30T00:00:00.000Z"); + const hours = (count: number) => DateTime.formatIso(DateTime.makeUnsafe(now + count * 3_600_000)); + const window = (usedPercent: number, resetsAt?: string): ServerProviderUsageWindow => ({ + id: resetsAt ?? `window-${usedPercent}`, + kind: "session", + label: "5h", + usedPercent, + ...(resetsAt === undefined ? {} : { resetsAt }), + }); + const hub = ( + ...accounts: ReadonlyArray]> + ): UsageLimitSourceSnapshot => ({ + id: UsageLimitSourceId.make("hub"), + kind: "cliproxy", + label: "hub", + checkedAt: hours(0), + accounts: accounts.map(([driver, windows], index) => ({ + id: `account-${index}`, + driver: ProviderDriverKind.make(driver), + usageLimits: { checkedAt: hours(0), windows }, + })), + }); + const codex = ProviderDriverKind.make("codex"); + + it.each([ + { + name: "the first spent account to recover, after all its spent windows reset", + sources: [ + hub( + ["codex", [window(100, hours(2))]], + ["codex", [window(100, hours(1)), window(100, hours(3))]], + ), + hub(["claudeAgent", [window(100, hours(0.5))]]), + ], + expected: hours(2), + }, + { + name: "nothing while an account still has quota", + sources: [hub(["codex", [window(100, hours(2))]], ["codex", [window(40, hours(1))]])], + expected: null, + }, + { + name: "nothing once a spent window's reset has passed", + sources: [hub(["codex", [window(100, hours(2))]], ["codex", [window(100, hours(-1))]])], + expected: null, + }, + { + name: "a recovery time while skipping accounts without readings or reset times", + sources: [hub(["codex", []], ["codex", [window(100)]], ["codex", [window(100, hours(4))]])], + expected: hours(4), + }, + { + name: "nothing when the hub pools none of the driver's accounts", + sources: [hub(["claudeAgent", [window(100, hours(1))]])], + expected: null, + }, + ])("returns $name", ({ sources, expected }) => { + expect(UsageLimitSources.hubPoolResetAt(sources, codex, now)).toBe(expected); + }); +}); diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index 7a068cde85f6c..9356982b556c8 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -3,6 +3,7 @@ import { UsageLimitSourceError, type UsageLimitSourceConsumeResetCreditInput, type ProviderConsumeResetCreditResult, + type ProviderDriverKind, type ServerSettings, type UsageLimitSourceConfig, type UsageLimitSourceId, @@ -168,3 +169,52 @@ export const make = Effect.gen(function* () { }); export const layer = Layer.effect(UsageLimitSources, make); + +/** + * A hub answers for a 429 only when every account it pools for the driver is spent; + * the pool recovers when its first account does. + */ +export function hubPoolResetAt( + sources: ReadonlyArray, + driver: ProviderDriverKind, + nowMs: number, +): string | null { + let earliest: number | null = null; + for (const account of sources.flatMap((source) => source.accounts)) { + if (account.driver !== driver) continue; + const { windows } = account.usageLimits; + const exhausted = windows.filter((window) => window.usedPercent >= 100); + if (exhausted.length === 0) { + if (windows.length > 0) return null; + continue; + } + const resets = exhausted.map((window) => Date.parse(window.resetsAt ?? "")); + if (resets.some((reset) => !Number.isFinite(reset))) continue; + const recoversAt = Math.max(...resets); + if (recoversAt <= nowMs) return null; + earliest = earliest === null ? recoversAt : Math.min(earliest, recoversAt); + } + return earliest === null ? null : DateTime.formatIso(DateTime.makeUnsafe(earliest)); +} + +/** Supplies a reset time for usage limits a provider reports without one. */ +export class UsageLimitResetFallback extends Context.Reference<{ + readonly resetAt: (driver: ProviderDriverKind) => Effect.Effect; +}>("t3/usage/UsageLimitResetFallback", { + defaultValue: () => ({ resetAt: () => Effect.succeed(null) }), +}) {} + +export const layerResetFallback = Layer.effect( + UsageLimitResetFallback, + Effect.gen(function* () { + const sources = yield* UsageLimitSources; + return { + resetAt: Effect.fn("UsageLimitResetFallback.resetAt")(function* (driver: ProviderDriverKind) { + // The terminal waits on this read, so a slow hub falls back to its last snapshot. + yield* sources.refresh.pipe(Effect.timeoutOption("5 seconds")); + const now = yield* DateTime.now; + return hubPoolResetAt(yield* sources.current, driver, DateTime.toEpochMillis(now)); + }), + }; + }), +); diff --git a/docs/user/usage.md b/docs/user/usage.md index e41e60ca73d76..56e244c121038 100644 --- a/docs/user/usage.md +++ b/docs/user/usage.md @@ -146,6 +146,11 @@ This connection supplies usage information; configure the provider separately to send agent requests through the hub. Remove the hub from the same settings section when you no longer need it. +CLIProxyAPI does not pass its reset time through Claude Code or Codex. When a Claude or Codex +turn stops on a usage limit without one, and every account the hub pools for that provider is +spent, the thread resets when the first of those accounts refills. **Resume at reset**, automatic +resume, and snooze then work as they do for direct subscriptions. + ## Subscription usage widget Add **Subscription usage** from your iOS or Android widget gallery to see remaining Codex and From baacd26f321b0f4f32566d71f2a9be1afd8da621 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:03:00 +1100 Subject: [PATCH 2/5] fix(server): take a hub's refill only for instances that route through it Macroscope noted that matching hubs by driver alone let a direct instance, or one behind another hub, inherit an unrelated pool's refill time. CLIProxyAPI serves agents and management on one address, so a hub now counts only when the instance's environment or launch arguments name its host. Co-Authored-By: Claude Opus 5.5 --- .../ProviderEventIngestor.test.ts | 11 +++-- .../orchestration-v2/ProviderEventIngestor.ts | 5 +- .../src/usage/UsageLimitSources.test.ts | 46 +++++++++++++++++++ apps/server/src/usage/UsageLimitSources.ts | 39 ++++++++++++++-- docs/user/usage.md | 6 ++- 5 files changed, 97 insertions(+), 10 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index a6bf3b4d77903..5116590c07322 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -236,11 +236,12 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit const refreshed = yield* Deferred.make(); const hubResetAt = "2026-10-30T05:00:00.000Z"; const nativeResetAt = "2026-10-30T03:00:00.000Z"; - const fallbackDrivers: ProviderDriverKind[] = []; + const fallbackLookups: Array<{ instanceId: ProviderInstanceId; driver: ProviderDriverKind }> = + []; const layerResetFallback = Layer.succeed(UsageLimitSources.UsageLimitResetFallback, { - resetAt: (driver) => + resetAt: (lookup) => Effect.sync(() => { - fallbackDrivers.push(driver); + fallbackLookups.push(lookup); return hubResetAt; }), }); @@ -337,7 +338,9 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit if (failureClass === "validation_error") assert.deepEqual(lookups, []); } assert.deepEqual(lookups, [instance.instanceId, instance.instanceId]); - assert.deepEqual(fallbackDrivers, [instance.driverKind]); + assert.deepEqual(fallbackLookups, [ + { instanceId: instance.instanceId, driver: instance.driverKind }, + ]); yield* Deferred.await(refreshed); }).pipe( Effect.provide( diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 41b4c019e1856..31486a9f01fb2 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -716,7 +716,10 @@ export const layer: Layer.Layer< : input.event.failure; const fallbackResetAt = reported.class === "usage_limit" && reported.resetAt == null - ? yield* usageLimitResets.resetAt(input.event.driver) + ? yield* usageLimitResets.resetAt({ + instanceId: input.providerInstanceId, + driver: input.event.driver, + }) : null; const failure = fallbackResetAt === null ? reported : { ...reported, resetAt: fallbackResetAt }; diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index 050ed74bc6af1..d0c8b941a89b3 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -2,6 +2,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { describe, expect, it } from "@effect/vitest"; import { ProviderDriverKind, + ProviderInstanceId, ServerConfigStreamEvent, UsageLimitSourceId, type ServerProviderUsageWindow, @@ -152,3 +153,48 @@ describe("hubPoolResetAt", () => { expect(UsageLimitSources.hubPoolResetAt(sources, codex, now)).toBe(expected); }); }); + +describe("instanceHubIds", () => { + const source = (url: string, enabled = true) => ({ + kind: "cliproxy" as const, + url, + managementKey: "key", + enabled, + }); + const settings = { + usageLimitSources: { + [UsageLimitSourceId.make("fleet")]: source("https://hub.test:8443/"), + [UsageLimitSourceId.make("local")]: source("http://127.0.0.1:8317"), + [UsageLimitSourceId.make("off")]: source("https://off.test", false), + }, + providerInstances: { + [ProviderInstanceId.make("claudeProxy")]: { + driver: ProviderDriverKind.make("claudeAgent"), + environment: [ + { name: "ANTHROPIC_BASE_URL", value: "https://HUB.test:8443", sensitive: false }, + ], + }, + [ProviderInstanceId.make("codexProxy")]: { + driver: ProviderDriverKind.make("codex"), + config: { + launchArgs: `-c 'model_providers.p={base_url="http://127.0.0.1:8317/v1"}' https://off.test`, + }, + }, + [ProviderInstanceId.make("otherPort")]: { + driver: ProviderDriverKind.make("codex"), + config: { launchArgs: `-c 'model_providers.p={base_url="http://127.0.0.1:83170/v1"}'` }, + }, + }, + }; + + it.each([ + ["claudeProxy", ["fleet"]], + ["codexProxy", ["local"]], + ["otherPort", []], + ["missing", []], + ])("matches %s to the enabled hubs its settings name", (instanceId, expected) => { + expect([ + ...UsageLimitSources.instanceHubIds(settings, ProviderInstanceId.make(instanceId)), + ]).toEqual(expected); + }); +}); diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index 9356982b556c8..a16acb9508b05 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -4,6 +4,7 @@ import { type UsageLimitSourceConsumeResetCreditInput, type ProviderConsumeResetCreditResult, type ProviderDriverKind, + type ProviderInstanceId, type ServerSettings, type UsageLimitSourceConfig, type UsageLimitSourceId, @@ -197,9 +198,29 @@ export function hubPoolResetAt( return earliest === null ? null : DateTime.formatIso(DateTime.makeUnsafe(earliest)); } +/** CLIProxyAPI serves agents and management on one address, so an instance uses the hubs it names. */ +export function instanceHubIds( + settings: Pick, + instanceId: ProviderInstanceId, +): ReadonlySet { + const instance = settings.providerInstances[instanceId]; + if (!instance) return new Set(); + const routing = JSON.stringify([instance.environment ?? [], instance.config ?? null]); + return new Set( + Object.entries(settings.usageLimitSources).flatMap(([id, config]) => { + if (!config.enabled || !URL.canParse(config.url)) return []; + const host = new URL(config.url).host.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`//${host}(?![\\w.:-])`, "i").test(routing) ? [id] : []; + }), + ); +} + /** Supplies a reset time for usage limits a provider reports without one. */ export class UsageLimitResetFallback extends Context.Reference<{ - readonly resetAt: (driver: ProviderDriverKind) => Effect.Effect; + readonly resetAt: (input: { + readonly instanceId: ProviderInstanceId; + readonly driver: ProviderDriverKind; + }) => Effect.Effect; }>("t3/usage/UsageLimitResetFallback", { defaultValue: () => ({ resetAt: () => Effect.succeed(null) }), }) {} @@ -208,12 +229,24 @@ export const layerResetFallback = Layer.effect( UsageLimitResetFallback, Effect.gen(function* () { const sources = yield* UsageLimitSources; + const settingsService = yield* Settings.ServerSettingsService; return { - resetAt: Effect.fn("UsageLimitResetFallback.resetAt")(function* (driver: ProviderDriverKind) { + resetAt: Effect.fn("UsageLimitResetFallback.resetAt")(function* (input: { + readonly instanceId: ProviderInstanceId; + readonly driver: ProviderDriverKind; + }) { + const settings = yield* settingsService.getSettings.pipe(Effect.option); + if (settings._tag === "None") return null; + const hubIds = instanceHubIds(settings.value, input.instanceId); + if (hubIds.size === 0) return null; // The terminal waits on this read, so a slow hub falls back to its last snapshot. yield* sources.refresh.pipe(Effect.timeoutOption("5 seconds")); const now = yield* DateTime.now; - return hubPoolResetAt(yield* sources.current, driver, DateTime.toEpochMillis(now)); + return hubPoolResetAt( + (yield* sources.current).filter((source) => hubIds.has(source.id)), + input.driver, + DateTime.toEpochMillis(now), + ); }), }; }), diff --git a/docs/user/usage.md b/docs/user/usage.md index 56e244c121038..55d4acb4d5466 100644 --- a/docs/user/usage.md +++ b/docs/user/usage.md @@ -147,8 +147,10 @@ the provider separately to send agent requests through the hub. Remove the hub f settings section when you no longer need it. CLIProxyAPI does not pass its reset time through Claude Code or Codex. When a Claude or Codex -turn stops on a usage limit without one, and every account the hub pools for that provider is -spent, the thread resets when the first of those accounts refills. **Resume at reset**, automatic +account sends requests through the hub, its environment variables or launch arguments name the +hub's address. If such an account stops on a usage limit without a reset time, and every account +the hub pools for that provider is spent, the thread resets when the first of those accounts +refills. **Resume at reset**, automatic resume, and snooze then work as they do for direct subscriptions. ## Subscription usage widget From 58ce7e82d5935315fd5910711b953ce08478d55a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:05:20 +1100 Subject: [PATCH 3/5] fix(server): a model-scoped Claude allowance no longer delays hub recovery CodeRabbit noted that a spent Fable-style weekly window made an account wait for that window even when the turn used another model. Those windows leave the account usable, so the hub pool's recovery now ignores them. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/claudeUsageLimits.ts | 9 ++++++- .../src/usage/UsageLimitSources.test.ts | 26 +++++++++++++++---- apps/server/src/usage/UsageLimitSources.ts | 5 +++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/apps/server/src/provider/claudeUsageLimits.ts b/apps/server/src/provider/claudeUsageLimits.ts index d0acf43bb4968..0196cbe9ad7ca 100644 --- a/apps/server/src/provider/claudeUsageLimits.ts +++ b/apps/server/src/provider/claudeUsageLimits.ts @@ -47,8 +47,15 @@ export const makeClaudeScopedLimitNames = Ref.make({ overageIncluded: undefined, }); +const SCOPED_WINDOW_PREFIX = "seven_day_"; + function scopedWindowId(displayName: string): string { - return `seven_day_${displayName.toLowerCase().replace(/[^a-z0-9]+/g, "_")}`; + return `${SCOPED_WINDOW_PREFIX}${displayName.toLowerCase().replace(/[^a-z0-9]+/g, "_")}`; +} + +/** Model-scoped allowances leave the account usable for other models. */ +export function isClaudeModelScopedWindow(window: Pick): boolean { + return window.id.startsWith(SCOPED_WINDOW_PREFIX); } function scopedWindow( diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index d0c8b941a89b3..6bf57fe5738f2 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -95,8 +95,12 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => describe("hubPoolResetAt", () => { const now = Date.parse("2026-10-30T00:00:00.000Z"); const hours = (count: number) => DateTime.formatIso(DateTime.makeUnsafe(now + count * 3_600_000)); - const window = (usedPercent: number, resetsAt?: string): ServerProviderUsageWindow => ({ - id: resetsAt ?? `window-${usedPercent}`, + const window = ( + usedPercent: number, + resetsAt?: string, + id = resetsAt ?? `window-${usedPercent}`, + ): ServerProviderUsageWindow => ({ + id, kind: "session", label: "5h", usedPercent, @@ -115,7 +119,6 @@ describe("hubPoolResetAt", () => { usageLimits: { checkedAt: hours(0), windows }, })), }); - const codex = ProviderDriverKind.make("codex"); it.each([ { @@ -144,13 +147,26 @@ describe("hubPoolResetAt", () => { sources: [hub(["codex", []], ["codex", [window(100)]], ["codex", [window(100, hours(4))]])], expected: hours(4), }, + { + name: "a recovery time that ignores spent model-scoped allowances", + driver: "claudeAgent", + sources: [ + hub([ + "claudeAgent", + [window(100, hours(2), "five_hour"), window(100, hours(90), "seven_day_fable")], + ]), + ], + expected: hours(2), + }, { name: "nothing when the hub pools none of the driver's accounts", sources: [hub(["claudeAgent", [window(100, hours(1))]])], expected: null, }, - ])("returns $name", ({ sources, expected }) => { - expect(UsageLimitSources.hubPoolResetAt(sources, codex, now)).toBe(expected); + ])("returns $name", ({ sources, expected, driver = "codex" }) => { + expect(UsageLimitSources.hubPoolResetAt(sources, ProviderDriverKind.make(driver), now)).toBe( + expected, + ); }); }); diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index a16acb9508b05..a6d6eaadb1b40 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -23,6 +23,7 @@ import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; +import { isClaudeModelScopedWindow } from "../provider/claudeUsageLimits.ts"; import * as Settings from "../serverSettings.ts"; import { makeCliproxyApi } from "./cliproxyApi.ts"; @@ -183,7 +184,9 @@ export function hubPoolResetAt( let earliest: number | null = null; for (const account of sources.flatMap((source) => source.accounts)) { if (account.driver !== driver) continue; - const { windows } = account.usageLimits; + const windows = account.usageLimits.windows.filter( + (window) => !isClaudeModelScopedWindow(window), + ); const exhausted = windows.filter((window) => window.usedPercent >= 100); if (exhausted.length === 0) { if (windows.length > 0) return null; From 594f8763dfcdd59d3eb817c7b5a3105f0d4cf98b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:06:27 +1100 Subject: [PATCH 4/5] fix(server): match hubs to instances by URL origin Comparing hosts as text missed an endpoint that spells out its default port and matched one on the same host over another scheme. Compare normalized origins instead. Co-Authored-By: Claude Opus 5.5 --- .../server/src/usage/UsageLimitSources.test.ts | 9 +++++++++ apps/server/src/usage/UsageLimitSources.ts | 18 ++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index 6bf57fe5738f2..4ff3dbd481a8f 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -182,6 +182,7 @@ describe("instanceHubIds", () => { [UsageLimitSourceId.make("fleet")]: source("https://hub.test:8443/"), [UsageLimitSourceId.make("local")]: source("http://127.0.0.1:8317"), [UsageLimitSourceId.make("off")]: source("https://off.test", false), + [UsageLimitSourceId.make("plain")]: source("https://plain.test"), }, providerInstances: { [ProviderInstanceId.make("claudeProxy")]: { @@ -196,6 +197,13 @@ describe("instanceHubIds", () => { launchArgs: `-c 'model_providers.p={base_url="http://127.0.0.1:8317/v1"}' https://off.test`, }, }, + [ProviderInstanceId.make("defaultPort")]: { + driver: ProviderDriverKind.make("claudeAgent"), + environment: [ + { name: "ANTHROPIC_BASE_URL", value: "https://plain.test:443/v1", sensitive: false }, + { name: "OTHER_URL", value: "http://hub.test:8443", sensitive: false }, + ], + }, [ProviderInstanceId.make("otherPort")]: { driver: ProviderDriverKind.make("codex"), config: { launchArgs: `-c 'model_providers.p={base_url="http://127.0.0.1:83170/v1"}'` }, @@ -206,6 +214,7 @@ describe("instanceHubIds", () => { it.each([ ["claudeProxy", ["fleet"]], ["codexProxy", ["local"]], + ["defaultPort", ["plain"]], ["otherPort", []], ["missing", []], ])("matches %s to the enabled hubs its settings name", (instanceId, expected) => { diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index a6d6eaadb1b40..47ee53bb8eafd 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -208,13 +208,19 @@ export function instanceHubIds( ): ReadonlySet { const instance = settings.providerInstances[instanceId]; if (!instance) return new Set(); - const routing = JSON.stringify([instance.environment ?? [], instance.config ?? null]); + const origins = new Set( + ( + JSON.stringify([instance.environment ?? [], instance.config ?? null]).match( + /https?:\/\/[^\s"'\\]+/gi, + ) ?? [] + ).flatMap((url) => (URL.canParse(url) ? [new URL(url).origin] : [])), + ); return new Set( - Object.entries(settings.usageLimitSources).flatMap(([id, config]) => { - if (!config.enabled || !URL.canParse(config.url)) return []; - const host = new URL(config.url).host.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - return new RegExp(`//${host}(?![\\w.:-])`, "i").test(routing) ? [id] : []; - }), + Object.entries(settings.usageLimitSources).flatMap(([id, config]) => + config.enabled && URL.canParse(config.url) && origins.has(new URL(config.url).origin) + ? [id] + : [], + ), ); } From 15e93bd18b35991a74806b39d46f34c110f65502 Mon Sep 17 00:00:00 2001 From: SAPHID Date: Sat, 31 Oct 2026 06:12:58 +0000 Subject: [PATCH 5/5] fix(server): read fallback hubs independently of global refresh --- .../src/usage/UsageLimitSources.test.ts | 75 ++++++++++++++++++- apps/server/src/usage/UsageLimitSources.ts | 22 +++++- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index 4ff3dbd481a8f..0783a2ff063c8 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -10,9 +10,12 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; -import { HttpClient } from "effect/http"; +import { HttpClient, HttpClientResponse } from "effect/http"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ServerSettings from "../serverSettings.ts"; @@ -92,6 +95,76 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => ), ); +it.effect("reads the selected hub while an unrelated refresh is still waiting", () => + Effect.gen(function* () { + const stalled = yield* Deferred.make(); + const release = yield* Deferred.make(); + const http = HttpClient.make((request) => + Effect.gen(function* () { + if (new URL(request.url).hostname === "slow.test") { + yield* Deferred.succeed(stalled, undefined); + yield* Deferred.await(release); + } + const body = request.url.endsWith("/auth-files") + ? { files: [{ id: "account", auth_index: "one", provider: "claude" }] } + : { + status_code: 200, + body: JSON.stringify({ + five_hour: { utilization: 100, resets_at: "2099-01-01T00:00:00.000Z" }, + }), + }; + return HttpClientResponse.fromWeb(request, Response.json(body)); + }), + ); + const sources = yield* UsageLimitSources.make.pipe( + Effect.provideService(HttpClient.HttpClient, http), + ); + yield* Deferred.await(stalled); + const fallback = yield* UsageLimitSources.UsageLimitResetFallback.pipe( + Effect.provide(UsageLimitSources.layerResetFallback), + Effect.provideService(UsageLimitSources.UsageLimitSources, sources), + ); + const result = yield* fallback + .resetAt({ + instanceId: ProviderInstanceId.make("proxy"), + driver: ProviderDriverKind.make("claudeAgent"), + }) + .pipe(Effect.forkChild); + yield* TestClock.adjust("5 seconds"); + expect(yield* Fiber.join(result)).toBe("2099-01-01T00:00:00.000Z"); + expect(yield* Deferred.isDone(release)).toBe(false); + }).pipe( + Effect.provide([ + ServerSettings.layerTest({ + providerInstances: { + [ProviderInstanceId.make("proxy")]: { + driver: ProviderDriverKind.make("claudeAgent"), + environment: [ + { name: "ANTHROPIC_BASE_URL", value: "https://fast.test", sensitive: false }, + ], + }, + }, + usageLimitSources: Object.fromEntries( + ["fast", "slow"].map((id) => [ + id, + { + kind: "cliproxy" as const, + url: `https://${id}.test`, + managementKey: "key", + enabled: true, + }, + ]), + ), + }), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + shouldRunScopeWork: () => Effect.succeed(false), + }), + NodeCrypto.layer, + ]), + Effect.scoped, + ), +); + describe("hubPoolResetAt", () => { const now = Date.parse("2026-10-30T00:00:00.000Z"); const hours = (count: number) => DateTime.formatIso(DateTime.makeUnsafe(now + count * 3_600_000)); diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index 47ee53bb8eafd..5e5d8a3dd6805 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -33,6 +33,9 @@ export class UsageLimitSources extends Context.Service< readonly current: Effect.Effect>; readonly streamChanges: Stream.Stream>; readonly refresh: Effect.Effect; + readonly read: ( + sourceIds: ReadonlySet, + ) => Effect.Effect>; readonly consumeResetCredit: ( input: UsageLimitSourceConsumeResetCreditInput, ) => Effect.Effect; @@ -103,6 +106,19 @@ export const make = Effect.gen(function* () { yield* publish(snapshots); }).pipe(refreshLock.withPermits(1), Effect.ignoreCause({ log: true })); + // Targeted reads do not publish, so unrelated refreshes and redemptions keep cache ownership. + const read = Effect.fn("UsageLimitSources.read")(function* (sourceIds: ReadonlySet) { + const settings = yield* settingsService.getSettings.pipe(Effect.option); + const entries = Object.entries( + settings._tag === "Some" ? settings.value.usageLimitSources : {}, + ).filter(([id, config]) => sourceIds.has(id) && config.enabled); + return yield* Effect.forEach( + entries, + ([id, config]) => readSource(id as UsageLimitSourceId, config), + { concurrency: 4 }, + ); + }); + // Shares the refresh lock so a stale in-flight read cannot overwrite a redemption. const consumeResetCredit = Effect.fn("UsageLimitSources.consumeResetCredit")(function* ( input: UsageLimitSourceConsumeResetCreditInput, @@ -154,6 +170,7 @@ export const make = Effect.gen(function* () { return { current: Ref.get(stateRef), + read, consumeResetCredit, refresh, get streamChanges() { @@ -249,10 +266,11 @@ export const layerResetFallback = Layer.effect( const hubIds = instanceHubIds(settings.value, input.instanceId); if (hubIds.size === 0) return null; // The terminal waits on this read, so a slow hub falls back to its last snapshot. - yield* sources.refresh.pipe(Effect.timeoutOption("5 seconds")); + const refreshed = yield* sources.read(hubIds).pipe(Effect.timeoutOption("5 seconds")); + const snapshots = refreshed._tag === "Some" ? refreshed.value : yield* sources.current; const now = yield* DateTime.now; return hubPoolResetAt( - (yield* sources.current).filter((source) => hubIds.has(source.id)), + snapshots.filter((source) => hubIds.has(source.id)), input.driver, DateTime.toEpochMillis(now), );