From 6db3ecd72e52d7a53a9c1a6c0795f0bdaddf136d Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:14:11 -0700 Subject: [PATCH 1/5] feat(web): `vp run stress` loads the app with concurrent streaming threads and records how it holds up Performance work had no way to put the app under realistic concurrent load. The rig builds the web client, starts an isolated server with the Replay provider, streams several long threads that stop for approvals, and drives a busy user in headless Chromium. It records frame times, long animation frames with their scripts, a CPU profile by module and function, React renders, heap and DOM growth, and page health into a report for daily runs. Co-Authored-By: Claude Opus 5.5 --- apps/server/scripts/dev-replay-agent.test.ts | 11 +- apps/server/scripts/dev-replay-agent.ts | 14 +- .../scripts/dev-replay/stressTranscript.ts | 216 ++++++++++ apps/web/stress/README.md | 25 ++ apps/web/stress/analyze.ts | 117 ++++++ apps/web/stress/probe.ts | 125 ++++++ apps/web/stress/run.ts | 382 ++++++++++++++++++ knip.jsonc | 6 +- package.json | 1 + 9 files changed, 888 insertions(+), 9 deletions(-) create mode 100644 apps/server/scripts/dev-replay/stressTranscript.ts create mode 100644 apps/web/stress/README.md create mode 100644 apps/web/stress/analyze.ts create mode 100644 apps/web/stress/probe.ts create mode 100644 apps/web/stress/run.ts diff --git a/apps/server/scripts/dev-replay-agent.test.ts b/apps/server/scripts/dev-replay-agent.test.ts index c3abef65161b7..ee322c9f48fbf 100644 --- a/apps/server/scripts/dev-replay-agent.test.ts +++ b/apps/server/scripts/dev-replay-agent.test.ts @@ -6,7 +6,12 @@ import * as NodeURL from "node:url"; import { describe, expect, it } from "@effect/vitest"; -import { parseReplayTurns, SCENARIOS, streamChunks } from "./dev-replay-agent.ts"; +import { + parseReplayTurns, + scenarioTranscript, + SCENARIOS, + streamChunks, +} from "./dev-replay-agent.ts"; interface AgentMessage { readonly id?: number; @@ -89,8 +94,8 @@ describe("dev replay agent", () => { expect(new Set(toolCallIds).size).toBe(6); }); - it.each(SCENARIOS)("$id plays a turn without session setup frames", ({ file }) => { - const turns = parseReplayTurns(NodeFS.readFileSync(new URL(file, import.meta.url), "utf8")); + it.each(SCENARIOS)("$id plays a turn without session setup frames", (scenario) => { + const turns = parseReplayTurns(scenarioTranscript(scenario)); expect(turns.length).toBeGreaterThan(0); const updates = turns.flatMap((turn) => turn.frames.flatMap(({ frame }) => diff --git a/apps/server/scripts/dev-replay-agent.ts b/apps/server/scripts/dev-replay-agent.ts index b6eea893ad6b7..68663f5418508 100644 --- a/apps/server/scripts/dev-replay-agent.ts +++ b/apps/server/scripts/dev-replay-agent.ts @@ -10,6 +10,8 @@ import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeReadline from "node:readline"; +import { stressTranscript } from "./dev-replay/stressTranscript.ts"; + interface LogicalFrame { readonly kind: "notification" | "request" | "response"; readonly method: string; @@ -64,6 +66,8 @@ export const SCENARIOS = [ name: "Multi turn", file: `${FIXTURES}/multi_turn/registry_transcript.ndjson`, }, + // Generated, so `vp run stress` can load the app without committing a large recording. + { id: "stress", name: "Stress", generate: stressTranscript }, ] as const; // Session setup and vendor state stay out: they would replace the replay agent's own models. @@ -94,6 +98,12 @@ function isReplayedFrame(frame: LogicalFrame): boolean { return typeof update?.sessionUpdate === "string" && REPLAYED_UPDATES.has(update.sessionUpdate); } +export function scenarioTranscript(scenario: (typeof SCENARIOS)[number]): string { + return "generate" in scenario + ? scenario.generate() + : NodeFS.readFileSync(new URL(scenario.file, import.meta.url), "utf8"); +} + /** Splits a recorded transcript into the turns that follow each `session/prompt`. */ export function parseReplayTurns(ndjson: string): ReadonlyArray { const turns: Array<{ frames: Array; stopReason: string }> = []; @@ -177,9 +187,7 @@ function runAgent(): void { const cached = turnsByScenario.get(scenarioId); if (cached !== undefined) return cached; const scenario = SCENARIOS.find((candidate) => candidate.id === scenarioId) ?? SCENARIOS[0]; - const turns = parseReplayTurns( - NodeFS.readFileSync(new URL(scenario.file, import.meta.url), "utf8"), - ); + const turns = parseReplayTurns(scenarioTranscript(scenario)); turnsByScenario.set(scenarioId, turns); return turns; }; diff --git a/apps/server/scripts/dev-replay/stressTranscript.ts b/apps/server/scripts/dev-replay/stressTranscript.ts new file mode 100644 index 0000000000000..895e9b95c1522 --- /dev/null +++ b/apps/server/scripts/dev-replay/stressTranscript.ts @@ -0,0 +1,216 @@ +/** + * A seeded, long transcript for `vp run stress`: every turn thinks, plans, reads, runs a + * command with long output, asks to edit, and answers with a long Markdown reply. + */ + +const TURNS = 4; +const REPLY_SECTIONS = 14; + +function seeded(seed: number) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const WORDS = + "the timeline renders each row once while the stream appends text so finished blocks keep their layout and the list measures only what grew between frames when a reply streams the sidebar still updates every running thread".split( + " ", + ); +const FILES = [ + "apps/web/src/components/ChatMarkdown.tsx", + "apps/web/src/components/chat/MessagesTimeline.tsx", + "apps/server/src/orchestration-v2/Orchestrator.ts", + "packages/client-runtime/src/state/threadDetail.ts", + "apps/web/src/components/pullRequest/PullRequestCodeTab.tsx", +]; + +function generator(seed: number) { + const random = seeded(seed); + const pick = (values: ReadonlyArray) => values[Math.floor(random() * values.length)]!; + const words = (count: number) => Array.from({ length: count }, () => pick(WORDS)).join(" "); + const sentence = () => { + const text = words(8 + Math.floor(random() * 14)); + return `${text[0]!.toUpperCase()}${text.slice(1)}.`; + }; + const paragraph = () => + Array.from({ length: 2 + Math.floor(random() * 4) }, () => { + const roll = random(); + const text = sentence(); + if (roll < 0.2) return text.replace(/ (\w+) /, " `$1` "); + if (roll < 0.35) return text.replace(/ (\w+ \w+) /, " **$1** "); + if (roll < 0.45) + return `${text} See [${pick(FILES)}](${pick(FILES)}#L${10 + Math.floor(random() * 400)}).`; + if (roll < 0.5) return `${text} Details in [the docs](https://example.com/${pick(WORDS)}).`; + return text; + }).join(" "); + const code = () => + [ + "```ts", + ...Array.from({ length: 6 + Math.floor(random() * 14) }, (_, line) => + line % 4 === 0 + ? `export function ${pick(WORDS)}${line}(input: number): number {` + : line % 4 === 3 + ? "}" + : ` return input * ${line} + ${Math.floor(random() * 100)}; // ${words(4)}`, + ), + "```", + ].join("\n"); + const table = () => + [ + "| File | Change | Lines |", + "| --- | --- | ---: |", + ...Array.from( + { length: 3 + Math.floor(random() * 5) }, + () => `| \`${pick(FILES)}\` | ${words(4)} | ${Math.floor(random() * 300)} |`, + ), + ].join("\n"); + const list = (ordered: boolean) => + Array.from( + { length: 3 + Math.floor(random() * 4) }, + (_, index) => `${ordered ? `${index + 1}.` : "-"} ${sentence()}`, + ).join("\n"); + const reply = () => + Array.from({ length: REPLY_SECTIONS }, (_, section) => + [ + `## ${sentence().slice(0, 40)} ${section + 1}`, + paragraph(), + pick([code, table, () => list(false), () => list(true), () => `> ${sentence()}`])(), + paragraph(), + ].join("\n\n"), + ).join("\n\n"); + const terminal = () => + Array.from( + { length: 120 }, + (_, line) => + ` ✓ ${pick(FILES).replace(".tsx", ".test.tsx")} (${line} tests) ${Math.floor(random() * 90)}ms`, + ).join("\n"); + return { paragraph, reply, terminal, words }; +} + +export function stressTranscript(): string { + const lines: Array = [ + { type: "transcript_start", provider: "acpRegistry", scenario: "dev_replay_stress" }, + ]; + const update = (value: Record, afterMs?: number) => + lines.push({ + type: "emit_inbound", + frame: { + kind: "notification", + method: "session/update", + params: { sessionId: "dev-replay-session", update: value }, + }, + ...(afterMs === undefined ? {} : { afterMs }), + }); + const plan = (done: number) => ({ + sessionUpdate: "plan", + entries: ["Read the affected files", "Edit the renderer", "Run the tests"].map( + (content, index) => ({ + content, + priority: "high", + status: index < done ? "completed" : index === done ? "in_progress" : "pending", + }), + ), + }); + for (let turn = 0; turn < TURNS; turn++) { + const text = generator(turn + 1); + lines.push({ + type: "expect_outbound", + frame: { kind: "request", method: "session/prompt", params: "" }, + }); + update({ + sessionUpdate: "agent_thought_chunk", + content: { type: "text", text: text.paragraph() }, + }); + update(plan(0)); + for (const [index, path] of FILES.slice(0, 3).entries()) { + const id = `read-${index}`; + update({ + sessionUpdate: "tool_call", + toolCallId: id, + kind: "read", + title: `Read ${path}`, + status: "pending", + rawInput: { path: `/${path}` }, + }); + update({ + sessionUpdate: "tool_call_update", + toolCallId: id, + status: "completed", + content: [{ type: "content", content: { type: "text", text: text.paragraph() } }], + }); + } + update(plan(1)); + update({ + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: `${text.paragraph()}\n\n` }, + }); + const edit = { + toolCallId: "edit", + kind: "edit", + title: `Edit ${FILES[0]}`, + rawInput: { path: `/${FILES[0]}` }, + }; + const diff = { + type: "diff", + path: `/${FILES[0]}`, + oldText: `${text.words(30)}\n`, + newText: `${text.words(30)}\n${text.words(30)}\n`, + }; + update({ sessionUpdate: "tool_call", ...edit, status: "pending", content: [diff] }); + lines.push({ + type: "emit_inbound", + frame: { + kind: "request", + method: "session/request_permission", + params: { + sessionId: "dev-replay-session", + toolCall: edit, + options: [ + { optionId: "allow-once", name: "Yes", kind: "allow_once" }, + { optionId: "reject-once", name: "No", kind: "reject_once" }, + ], + }, + }, + }); + lines.push({ + type: "expect_outbound", + frame: { kind: "response", method: "session/request_permission", result: "" }, + }); + update({ + sessionUpdate: "tool_call_update", + toolCallId: "edit", + status: "completed", + content: [diff], + }); + update(plan(2)); + update({ + sessionUpdate: "tool_call", + toolCallId: "test", + kind: "execute", + title: "vp test run", + status: "in_progress", + rawInput: { command: "vp test run" }, + }); + update( + { + sessionUpdate: "tool_call_update", + toolCallId: "test", + status: "completed", + content: [{ type: "content", content: { type: "text", text: text.terminal() } }], + }, + 1200, + ); + update(plan(3)); + update({ sessionUpdate: "agent_message_chunk", content: { type: "text", text: text.reply() } }); + lines.push({ + type: "emit_inbound", + frame: { kind: "response", method: "session/prompt", result: { stopReason: "end_turn" } }, + }); + } + return lines.map((line) => JSON.stringify(line)).join("\n"); +} diff --git a/apps/web/stress/README.md b/apps/web/stress/README.md new file mode 100644 index 0000000000000..e1d1f7b3a897b --- /dev/null +++ b/apps/web/stress/README.md @@ -0,0 +1,25 @@ +# Stress rig + +`vp run stress` puts a production build of T3 Code under concurrent load and records how the page holds up. It finds performance work; it is not a CI gate. + +## What a run does + +1. Builds `apps/web/dist` in production mode without minification, so profiles name real functions. `--skip-build` reuses the last build. +2. Starts a server on a fresh home in `.t3/stress/home` (never your real data) with the `Replay (dev)` provider and a scratch git project. +3. Starts `--threads` threads (default 5) on the Replay `stress` scenario, each with `--turns-per-thread` turns queued (default 4). Every turn thinks, plans, reads, asks for an edit approval, prints long terminal output, and streams a long Markdown reply through the server's default paragraph streaming. +4. Opens headless Chromium, pairs, and for `--duration` seconds (default 60) acts like a busy user: switches threads, approves edits, types in the composer, opens and closes the right panel, and scrolls. +5. Writes `report.md`, `summary.json`, `profile.cpuprofile` (open in Chrome DevTools), screenshots, and `server.log` to `.t3/stress/runs/