diff --git a/AGENTS.md b/AGENTS.md index 609bcbd7c9d88..71908d09fc786 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,6 +72,17 @@ Rules that apply regardless of approach: - **`git rerere` is enabled in this repo** (`rerere.enabled=true`), so a conflict resolved once is replayed automatically the next time the same hunk collides. Never resolve a conflict by discarding one side wholesale just to make it go away — a bad resolution gets replayed too. Verify what rerere auto-resolved before committing a merge. - **Test upstream merges on `expbkmain` first**, never directly on `bkmain`. Reset `expbkmain` from `bkmain` so its conflicts are exactly the ones `bkmain` will see, verify at expbkt3.dev, then promote. +## Server code must scale with bkt3's database + +bkt3 is far bigger than a typical install: a 16 GB `statev2.sqlite`, about 1,800 threads (half of them archived), 20+ live provider sessions and several Claude accounts that are often full. The server runs on one event loop with synchronous `node:sqlite`, so every query, decode and subprocess spawn on the main thread pauses every client. Code that is instant on a dev database froze bkt3 for 2-13 s at a time (#164 in September 2026, then #325/#326, #328 and #338 in October). Before merging server code, check it against these rules: + +1. **No all-threads reads on a hot path.** A full shell snapshot (`getShellSnapshot`, `readShellSnapshot`) or a loop over every thread costs about 2 s on bkt3. Never run one per event, per turn end, per connection, per account change, or on a timer. Read the narrowest thing that answers the question: one thread (`getThreadShell`), live sessions from memory (`ProviderSessionManager.listLive`, which is optional, so keep a fallback), or an indexed query that selects the few rows you need. Full snapshots are for rare, user-triggered requests only. +2. **Cheap checks first.** Run in-memory guards (is there room elsewhere, is the feature on, has anything changed) before any database read, so the common no-op case costs nothing. `ClaudeAccountsService.maybeRebalance` read every session twice per turn end before it checked whether another account had room (#338). +3. **Every query seeks an index.** `run_id IN (...) OR (? = 1 AND run_id IS NULL)` silently scans (#326; use `INDEXED BY` or split the query). `orchestration_events` lookups need `aggregate_kind` with the stream id (#164). Check new queries with `EXPLAIN QUERY PLAN` against a copy of real data (see Test data). +4. **No subprocess per thread.** Linux `fork()` of the ~1.5 GB server costs about 30 ms of main thread, even for a cached git call. A sweep that spawns per thread is a freeze: rotate idle threads, cache results, and spawn only for threads that changed (#325). +5. **Count calls in tests.** A focused test for a hot-path change asserts how many reads a trigger costs (for example, a turn end on a full account makes zero `listSessions` calls), not only the result. Wall-clock checks belong on a real-data copy, not in CI. +6. **Check production after the deploy.** Count `event loop stalled` warnings with `journalctl -u t3-nextbkt3.service`, probe `GET /.well-known/t3/environment` every 250 ms, and compare with the hours before the deploy. The server trace keeps only a few minutes, so capture a stall while it happens. + ## What makes T3 Code special? We have over 500,000 users who love T3 Code. It's important we maintain the things they love as we continue to iterate on the product. Here's a brief list of the things we can never compromise on. diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 51ea3d0c24635..5ed15a4c2ce82 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -97,6 +97,8 @@ import { } from "./activeSubagentCount.expbkt3.ts"; import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp"; +// T3-CUSTOM(expbkt3): full shell reads use the reader worker. +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts"; import { isThreadHistoryUserTurn, @@ -6516,7 +6518,12 @@ export const layer: Layer.Layer = const getShellSnapshot: ProjectionStoreV2Shape["getShellSnapshot"] = (options) => sql .withTransaction(readShellSnapshotRows(options)) - .pipe(Effect.mapError(shellSnapshotReadError), Effect.flatMap(decodeShellSnapshot)); + // T3-CUSTOM(expbkt3): full shell reads run on the SQLite reader worker; inside an open write transaction they stay on the writer. + .pipe( + NodeSqliteClient.readOnly, + Effect.mapError(shellSnapshotReadError), + Effect.flatMap(decodeShellSnapshot), + ); const getThreadShell: ProjectionStoreV2Shape["getThreadShell"] = (threadId) => sql diff --git a/apps/server/src/wsForkHandlers.threadUsage.test.ts b/apps/server/src/wsForkHandlers.threadUsage.test.ts new file mode 100644 index 0000000000000..6b464e853c0e6 --- /dev/null +++ b/apps/server/src/wsForkHandlers.threadUsage.test.ts @@ -0,0 +1,85 @@ +/** + * T3-CUSTOM(expbkt3): `threadUsage.get` reads only the requested thread. + * + * Every open thread view polls this RPC, so it must never read the full shell + * snapshot: on bkt3 that is ~1,800 threads per call. + */ +import { + ThreadId, + WS_METHODS, + type OrchestrationThreadShell, + type ThreadUsage, +} from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import { makeForkWsHandlers, type ForkWsHandlerDeps } from "./wsForkHandlers.ts"; + +const threadId = ThreadId.make("thread-usage"); + +function makeHandlers(shell: Option.Option) { + const shellReads: Array = []; + const usageReads: Array<{ + readonly sessionIds: ReadonlyArray; + readonly sinceMs: number; + }> = []; + const deps = { + observeRpcEffect: (_method, effect) => effect, + requireThreadAccess: () => Effect.void, + projectionSnapshotQuery: { + getShellSnapshot: () => Effect.die("threadUsage.get must not read the full shell"), + getThreadShellById: (id: ThreadId) => + Effect.sync(() => { + shellReads.push(id); + return shell; + }), + } as unknown as ForkWsHandlerDeps["projectionSnapshotQuery"], + usage: { + readThreadUsage: (input: { + readonly sessionIds: ReadonlyArray; + readonly sinceMs: number; + }) => + Effect.sync(() => { + usageReads.push({ sessionIds: input.sessionIds, sinceMs: input.sinceMs }); + return {} as ThreadUsage; + }), + } as unknown as ForkWsHandlerDeps["usage"], + } satisfies Partial; + return { + handlers: makeForkWsHandlers(deps as unknown as ForkWsHandlerDeps), + shellReads, + usageReads, + }; +} + +const shellFor = (archivedAt: string | null) => + ({ + id: threadId, + createdAt: "2026-10-21T10:00:00.000Z", + archivedAt, + session: { providerThreadId: "provider-thread-1" }, + }) as unknown as OrchestrationThreadShell; + +describe("threadUsage.get", () => { + it.effect("reads one thread shell and passes its provider session", () => + Effect.gen(function* () { + const { handlers, shellReads, usageReads } = makeHandlers(Option.some(shellFor(null))); + yield* handlers[WS_METHODS.threadUsageGet]({ threadId, timeZone: "UTC" }); + expect(shellReads).toEqual([threadId]); + expect(usageReads).toEqual([ + { sessionIds: ["provider-thread-1"], sinceMs: Date.parse("2026-10-21T10:00:00.000Z") }, + ]); + }), + ); + + it.effect("keeps the empty result for archived and missing threads", () => + Effect.gen(function* () { + for (const shell of [Option.some(shellFor("2026-10-21T11:00:00.000Z")), Option.none()]) { + const { handlers, usageReads } = makeHandlers(shell); + yield* handlers[WS_METHODS.threadUsageGet]({ threadId, timeZone: "UTC" }); + expect(usageReads).toEqual([{ sessionIds: [], sinceMs: 0 }]); + } + }), + ); +}); diff --git a/apps/server/src/wsForkHandlers.ts b/apps/server/src/wsForkHandlers.ts index 107a24f1a42ad..bf48422a42a8d 100644 --- a/apps/server/src/wsForkHandlers.ts +++ b/apps/server/src/wsForkHandlers.ts @@ -444,7 +444,9 @@ export const makeForkWsHandlers = ({ // T3-CUSTOM(expbkt3): END // T3-CUSTOM(expbkt3): BEGIN per-thread API-level cost. Thread-scoped read: // the figures go only to someone who can already read that thread. The - // provider session behind the thread comes from the shell projection. + // provider session behind the thread comes from that thread's shell row + // alone: a full shell snapshot here cost every open thread view a read of + // all threads each minute. [WS_METHODS.threadUsageGet]: (input) => observeRpcEffect( WS_METHODS.threadUsageGet, @@ -458,7 +460,7 @@ export const makeForkWsHandlers = ({ }), ), Effect.andThen( - projectionSnapshotQuery.getShellSnapshot().pipe( + projectionSnapshotQuery.getThreadShellById(input.threadId).pipe( Effect.mapError( (cause) => new UsageReadError({ @@ -469,8 +471,11 @@ export const makeForkWsHandlers = ({ ), ), ), - Effect.flatMap((snapshot) => { - const thread = snapshot.threads.find((candidate) => candidate.id === input.threadId); + Effect.flatMap((shell) => { + // Archived threads keep the old full-snapshot result: no session, no start time. + const thread = Option.getOrUndefined( + Option.filter(shell, (candidate) => candidate.archivedAt === null), + ); const providerThreadId = thread?.session?.providerThreadId ?? null; return usage.readThreadUsage({ threadId: input.threadId,