Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionStore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4601,6 +4601,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
if (lastErrorClass === "usage_limit") {
assert.deepEqual(candidate, {
id: sqlShell.id,
projectId: sqlShell.projectId,
modelSelection: sqlShell.modelSelection,
status: sqlShell.status,
lastErrorClass: sqlShell.lastErrorClass,
usageLimitResetAt: sqlShell.usageLimitResetAt,
Expand Down Expand Up @@ -4769,6 +4771,18 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
(row) => row.id === threadId,
),
);
for (const [since, found] of [
[DateTime.subtract(now, { minutes: 1 }), true],
[DateTime.add(now, { minutes: 1 }), false],
] as const) {
assert.strictEqual(
(yield* store.getLimitRecoveryCandidates({
...recoveryOptions,
switchAccountsSince: since,
})).some((row) => row.id === threadId),
found,
);
}
const reset = DateTime.makeUnsafe(limitItem.failure.resetAt);
const recovery = {
runId: original.id,
Expand Down
11 changes: 11 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ export type ProjectionRecoveryKind =
export type ProjectionLimitRecoveryCandidate = Pick<
OrchestrationV2ThreadShell,
| "id"
| "projectId"
| "modelSelection"
| "status"
| "lastErrorClass"
| "latestRunId"
Expand Down Expand Up @@ -507,6 +509,8 @@ export interface ProjectionStoreV2Shape {
readonly now: DateTime.Utc;
readonly autoResume: boolean;
readonly snooze: boolean;
/** Also returns limits hit since then whose window has not reset, to move to another account. */
readonly switchAccountsSince?: DateTime.Utc;
}) => Effect.Effect<ReadonlyArray<ProjectionLimitRecoveryCandidate>, ProjectionStoreV2Error>;
/** Epoch ms of the next armed reset or snooze end after `now`, when a waiting thread becomes a candidate. */
readonly getLimitRecoveryWakeAt: (
Expand Down Expand Up @@ -3879,6 +3883,11 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
AND (
(
${booleanInt(options.switchAccountsSince !== undefined)}
AND julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) >= julianday(${DateTime.formatIso(options.switchAccountsSince ?? options.now)})
AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)})
)
OR (
json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id
AND json_extract(t.payload_json, '$.limitRecovery.resetAt') IS json_extract(item.payload_json, '$.failure.resetAt')
AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1
Expand Down Expand Up @@ -3916,6 +3925,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
if (summary.lastErrorClass !== "usage_limit") continue;
candidates.push({
id: thread.id,
projectId: thread.projectId,
modelSelection: thread.modelSelection,
status: "failed",
lastErrorClass: summary.lastErrorClass,
usageLimitResetAt: summary.usageLimitResetAt,
Expand Down
90 changes: 90 additions & 0 deletions apps/server/src/orchestration-v2/ThreadMessageIntake.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,16 @@ import {
EventId,
GuestMessageId,
MessageId,
ProjectId,
ProviderDriverKind,
ProviderInstanceId,
RuntimeRequestId,
ThreadId,
TurnItemId,
type OrchestrationV2Command,
type OrchestrationV2ServerCommand,
type OrchestrationV2StoredEvent,
type ServerProvider,
type UserInputAttachments,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
Expand All @@ -27,6 +31,7 @@ import {
resolveAttachmentPath,
} from "../attachmentStore.ts";
import * as ServerConfig from "../config.ts";
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
import * as ServerSettings from "../serverSettings.ts";
import {
OrchestratorCommandPreviouslyRejectedError,
Expand All @@ -40,6 +45,9 @@ const layerIntakeTest = ServerConfig.layerTest(process.cwd(), {
}).pipe(
Layer.provideMerge(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))),
Layer.provideMerge(ServerSettings.layerTest()),
Layer.provideMerge(
Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
),
);

const layerFailingDispatchHandler =
Expand Down Expand Up @@ -426,3 +434,85 @@ it.effect("sends a guest message with the guest's files, never ones the client n
expect(command.attachments).toEqual([guestFile]);
}).pipe(Effect.provide(layerIntakeTest)),
);

it.effect("sends on another account of the same provider when the chosen one is out of usage", () =>
Effect.gen(function* () {
const claude = (id: string, usedPercent: number): ServerProvider => ({
instanceId: ProviderInstanceId.make(id),
driver: ProviderDriverKind.make("claudeAgent"),
enabled: true,
installed: true,
version: null,
status: "ready",
auth: { status: "authenticated" },
checkedAt: "2026-09-13T00:00:00.000Z",
models: [{ slug: "claude-opus-5-5", name: "Opus 5.5", isCustom: false, capabilities: null }],
slashCommands: [],
skills: [],
usageLimits: {
checkedAt: "2026-09-13T00:00:00.000Z",
windows: [
{
id: "five_hour",
kind: "session",
label: "Session",
usedPercent,
resetsAt: "2099-01-01T00:00:00.000Z",
},
],
},
});
const captured: OrchestrationV2ServerCommand[] = [];
const selection = {
instanceId: ProviderInstanceId.make("claude_work"),
model: "claude-opus-5-5",
};
yield* dispatchCommand({
type: "message.dispatch",
commandId: CommandId.make("send-limited"),
threadId: ThreadId.make("thread-limited"),
messageId: MessageId.make("message-limited"),
text: "go",
attachments: [],
dispatchMode: { type: "start_immediately" },
createdBy: "user",
creationSource: "web",
}).pipe(
Effect.provide(
Layer.mergeAll(
Layer.mock(ThreadManagementService.ThreadManagementService)({
dispatch: layerFailingDispatchHandler(captured),
getThreadSnapshotWindow: () =>
Effect.succeed({
projection: {
thread: {
projectId: ProjectId.make("project-limited"),
modelSelection: selection,
},
},
} as never),
}),
Layer.mock(ProviderRegistry.ProviderRegistry)({
getProviders: Effect.succeed([
claude("claude_work", 100),
claude("claude_personal", 30),
]),
}),
ServerSettings.layerTest({
fallbackToAvailableAccounts: true,
providerInstances: Object.fromEntries(
["claude_work", "claude_personal"].map((id) => [
ProviderInstanceId.make(id),
{ driver: ProviderDriverKind.make("claudeAgent"), enabled: true },
]),
),
}),
),
),
Effect.result,
);
expect(captured[0]).toMatchObject({
modelSelection: { instanceId: "claude_personal", model: "claude-opus-5-5" },
});
}).pipe(Effect.provide(layerIntakeTest)),
);
62 changes: 59 additions & 3 deletions apps/server/src/orchestration-v2/ThreadMessageIntake.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,19 @@
import { remapComposerContextAttachments } from "@t3tools/shared/composerContextReferences";
import { appendUserInputAttachmentPaths } from "../provider/userInputAttachments.ts";
import type { ChatAttachment, OrchestrationV2Command } from "@t3tools/contracts";
import type {
ChatAttachment,
ModelSelection,
OrchestrationV2Command,
ProjectId,
} from "@t3tools/contracts";
import { availableAccountSelection } from "@t3tools/shared/usageLimits";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Schema from "effect/Schema";
import { resolveAttachmentPath } from "../attachmentStore.ts";
import * as ServerConfig from "../config.ts";
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
import * as ServerSettings from "../serverSettings.ts";
import * as Orchestrator from "./Orchestrator.ts";

import * as AttachmentClaims from "./AttachmentClaims.ts";
Expand Down Expand Up @@ -81,11 +90,53 @@ const withGuestAttachments = Effect.fn("ThreadMessageIntake.withGuestAttachments
return { ...command, attachments: message?.attachments ?? [] };
});

/** Null unless the user lets sends move off an account that is out of usage. */
const readAccountChoice = Effect.fn("ThreadMessageIntake.readAccountChoice")(function* () {
const settings = yield* ServerSettings.ServerSettingsService.pipe(
Effect.flatMap((service) => service.getSettings),
Effect.orElseSucceed(() => null),
);
if (!settings?.fallbackToAvailableAccounts) return null;
const providers = yield* ProviderRegistry.ProviderRegistry.pipe(
Effect.flatMap((registry) => registry.getProviders),
);
const now = DateTime.toEpochMillis(yield* DateTime.now);
return (selection: ModelSelection, projectId: ProjectId) =>
availableAccountSelection({ selection, providers, now, settings, projectId }) ?? selection;
});

const withAvailableAccount = Effect.fn("ThreadMessageIntake.withAvailableAccount")(function* (
command: Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }>,
) {
// Steering stays on the active run's account; server continuations validate their own.
if (
command.dispatchMode.type === "steer_active" ||
command.dispatchMode.type === "restart_active" ||
command.usageLimitContinuationOfRunId !== undefined ||
command.restartContinuationOfRunId !== undefined
) {
return command;
}
const choose = yield* readAccountChoice();
if (choose === null) return command;
const threads = yield* ThreadManagement.ThreadManagementService;
const window = yield* threads
.getThreadSnapshotWindow(command.threadId, { rowLimit: 1 })
.pipe(Effect.orElseSucceed(() => null));
if (window === null) return command;
const { thread } = window.projection;
const selection = command.modelSelection ?? thread.modelSelection;
const next = choose(selection, thread.projectId);
return next === selection ? command : { ...command, modelSelection: next };
});

export const dispatchCommand = Effect.fn("ThreadMessageIntake.dispatchCommand")(function* (
incoming: OrchestrationV2Command,
) {
const command =
incoming.type === "message.dispatch" ? yield* withGuestAttachments(incoming) : incoming;
incoming.type === "message.dispatch"
? yield* withAvailableAccount(yield* withGuestAttachments(incoming))
: incoming;
const threads = yield* ThreadManagement.ThreadManagementService;
if (command.type === "runtime-request.respond" && command.attachmentsByQuestionId) {
const config = yield* ServerConfig.ServerConfig;
Expand Down Expand Up @@ -207,9 +258,14 @@ export const sendToThread = Effect.fn("ThreadMessageIntake.sendToThread")(functi
});

export const launchThread = Effect.fn("ThreadMessageIntake.launchThread")(function* (
input: ThreadLaunch.ThreadLaunchInput,
requested: ThreadLaunch.ThreadLaunchInput,
) {
const launches = yield* ThreadLaunch.ThreadLaunchService;
const choose = yield* readAccountChoice();
const input =
choose === null
? requested
: { ...requested, modelSelection: choose(requested.modelSelection, requested.projectId) };
yield* AttachmentClaims.validateAttachmentLimits(input.initialMessage?.attachments ?? []);
if (!input.initialMessage?.attachments.some(AttachmentClaims.attachmentIsPendingUpload)) {
return yield* launches.launch(input);
Expand Down
84 changes: 82 additions & 2 deletions apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
import { assert, describe, it } from "@effect/vitest";
import { RunId, ThreadId, type OrchestrationV2DomainEvent } from "@t3tools/contracts";
import {
DEFAULT_SERVER_SETTINGS,
ProjectId,
ProviderDriverKind,
ProviderInstanceId,
RunId,
ThreadId,
type OrchestrationV2DomainEvent,
type ServerProvider,
} from "@t3tools/contracts";
import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
Expand All @@ -8,13 +17,14 @@ import * as Queue from "effect/Queue";
import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";

import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
import * as Scheduler from "../scheduling/Scheduler.ts";
import * as ServerSettings from "../serverSettings.ts";
import * as Orchestrator from "./Orchestrator.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as ThreadManagement from "./ThreadManagementService.ts";
import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts";
import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
import { limitRecoveryCommand, limitSwitchCommand } from "./UsageLimitRecoveryWorker.ts";

it.effect("idle sweeps skip SQL until the next wake time or a relevant event", () =>
Effect.gen(function* () {
Expand All @@ -39,6 +49,7 @@ it.effect("idle sweeps skip SQL until the next wake time or a relevant event", (
),
}),
Layer.mock(ThreadManagement.ThreadManagementService)({}),
Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
Layer.mock(Orchestrator.OrchestratorV2)({
streamDomainEvents: Stream.fromQueue(events),
}),
Expand Down Expand Up @@ -77,6 +88,11 @@ const failedThread = (failedAgoMs: number, resetAgoMs: number) => {
const failedAt = DateTime.makeUnsafe(nowMs - failedAgoMs);
const candidate: ProjectionStore.ProjectionLimitRecoveryCandidate = {
id: ThreadId.make("thread:limited"),
projectId: ProjectId.make("project:limited"),
modelSelection: {
instanceId: ProviderInstanceId.make("claude_work"),
model: "claude-opus-5-5",
},
status: "failed",
lastErrorClass: "usage_limit",
usageLimitResetAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - resetAgoMs)),
Expand Down Expand Up @@ -116,3 +132,67 @@ describe("limitRecoveryCommand arming", () => {
assert.isNull(limitRecoveryCommand(thread, true, nowMs));
});
});

describe("limitSwitchCommand", () => {
const claude = (id: string, usedPercent: number): ServerProvider => ({
instanceId: ProviderInstanceId.make(id),
driver: ProviderDriverKind.make("claudeAgent"),
enabled: true,
installed: true,
version: null,
status: "ready",
auth: { status: "authenticated" },
checkedAt: "2026-10-19T11:00:00.000Z",
models: [{ slug: "claude-opus-5-5", name: "Opus 5.5", isCustom: false, capabilities: null }],
slashCommands: [],
skills: [],
usageLimits: {
checkedAt: "2026-10-19T11:00:00.000Z",
windows: [{ id: "five_hour", kind: "session", label: "Session", usedPercent }],
},
});
const settings = {
...DEFAULT_SERVER_SETTINGS,
fallbackToAvailableAccounts: true,
providerInstances: {
[ProviderInstanceId.make("claude_work")]: {
driver: ProviderDriverKind.make("claudeAgent"),
enabled: true,
},
[ProviderInstanceId.make("claude_personal")]: {
driver: ProviderDriverKind.make("claudeAgent"),
enabled: true,
},
},
};
// The work account's bars are stale: the turn that just failed is the proof it is spent.
const providers = [claude("claude_work", 40), claude("claude_personal", 20)];
const input = { providers, settings, nowMs, limitedUntil: new Map<ProviderInstanceId, number>() };
const justLimited = failedThread(60_000, -HOUR_MS);

it("continues the same model on another account with usage left", () => {
assert.deepInclude(limitSwitchCommand(justLimited, input), {
type: "message.dispatch",
manualContinuationOfRunId: RunId.make("run:limited"),
modelSelection: {
instanceId: ProviderInstanceId.make("claude_personal"),
model: "claude-opus-5-5",
},
});
});

it.each([
["the setting is off", justLimited, { ...input, settings: DEFAULT_SERVER_SETTINGS }],
["the limit was hit long ago", failedThread(HOUR_MS, -HOUR_MS), input],
[
"the other account hit its own limit here and has not reset",
justLimited,
{
...input,
limitedUntil: new Map([[ProviderInstanceId.make("claude_personal"), nowMs + HOUR_MS]]),
},
],
])("waits when %s", (_name, thread, options) => {
assert.isNull(limitSwitchCommand(thread, options));
});
});
Loading
Loading