From f06570814ba9edbfb710c253dcfddd45005056e6 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 11 Oct 2026 04:28:32 +0200 Subject: [PATCH 1/4] test(core): measure what Code Mode runs hold, and run its executor on plain workerd Each Workspace object now holds the entrypoint and env stubs of its code runs in one measured registry, released when the run ends, and reports what runs hold. A new smoke run serves the same executor with a Worker Loader on plain workerd and checks the module contract, its failures, limits, cancellation, streamed output and the output budget there, and that the runtime records cpu_limit_unsupported. CI runs it next to the on-prem smoke run. --- .github/workflows/ci.yml | 2 + apps/core/src/agent.ts | 14 +- apps/core/src/code-mode.ts | 58 ++++++- apps/core/src/workspace.ts | 31 ++++ apps/core/test/agent.test.ts | 46 ++++++ apps/core/test/workerd/code-mode-worker.ts | 133 +++++++++++++++ package.json | 1 + scripts/workerd-code-mode.ts | 183 +++++++++++++++++++++ 8 files changed, 462 insertions(+), 6 deletions(-) create mode 100644 apps/core/test/workerd/code-mode-worker.ts create mode 100644 scripts/workerd-code-mode.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 165505da..ce452fca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,6 +125,8 @@ jobs: - args: ["--frozen-lockfile"] - run: vp run --filter @grasp-os/web build - run: vp run smoke:workerd + # Code Mode's executor, with a Worker Loader, on plain workerd. + - run: vp run smoke:code-mode # Builds a release bundle as main will publish it (core, connect, the # frontend's static files and D1 migrations, with its manifest) and checks diff --git a/apps/core/src/agent.ts b/apps/core/src/agent.ts index 1c625b7a..3e74fef2 100644 --- a/apps/core/src/agent.ts +++ b/apps/core/src/agent.ts @@ -15,7 +15,7 @@ import { z } from "zod"; import type { AgentApi, AgentScope } from "./agent-scope.ts"; import { describeRun, runCode, sourceHash } from "./code-mode.ts"; -import type { CodeRun } from "./code-mode.ts"; +import type { CodeRun, HeldHandles } from "./code-mode.ts"; import type { CodeRunStatus } from "./code-run-statuses.ts"; import type { AgentModel } from "./models.ts"; import type { CodeRunOpening } from "./workspace.ts"; @@ -226,6 +226,8 @@ export interface CodeRuns { * only while the run is open, for the code step it runs for. */ output: (runId: string, toolCallId: string, line: string) => void; + /** Where each run's entrypoint and stubs are held while it runs. */ + handles: HeldHandles; } /** @@ -253,7 +255,10 @@ const runOpen = async ( sourceChars: code.length, }); const env = Object.fromEntries( - apis.map((api) => [api.name, api.stub({ ...scope, runId })]) + apis.map((api) => [ + api.name, + runs.handles.hold(api.stub({ ...scope, runId })), + ]) ); // `runCode` never throws; should anything else, the run still closes. let ran: CodeRunStatus = "crashed"; @@ -263,6 +268,7 @@ const runOpen = async ( run = await runCode(loader, code, env, { signal, timeoutMs: codeTimeoutMs, + handles: runs.handles, onOutput: (line) => { onOutput(runId, line); }, @@ -270,6 +276,10 @@ const runOpen = async ( ran = run.status; } finally { ended = await runs.close(runId, ran); + // The run's stubs go with it. + for (const stub of Object.values(env)) { + runs.handles.release(stub); + } } // A run a revocation closed, as it ran or as it ended, hands on nothing // of what it did: not what it returned, nor what it logged, which could diff --git a/apps/core/src/code-mode.ts b/apps/core/src/code-mode.ts index bef5d94b..46a695ba 100644 --- a/apps/core/src/code-mode.ts +++ b/apps/core/src/code-mode.ts @@ -430,6 +430,41 @@ const parsed = ( const notAResult = "The run returned more than it may, or not a result."; +/** + * The handles code runs hold: each run's entrypoint into its isolate, and + * the stubs of its env, from when the run starts until it lets go of them + * (`release`). One per Workspace object, so an object that is reset takes + * its own with it. Measured, so a test sees that repeated runs leave none + * behind rather than trusting the cleanup. + */ +export class HeldHandles { + readonly #held = new Set(); + + /** How many handles runs hold now. */ + get size(): number { + return this.#held.size; + } + + /** Takes `handle` as held by a run, until {@link release}. */ + hold(handle: T): T { + this.#held.add(handle); + return handle; + } + + /** Lets go of a run's handle: disposed, where it can be, and forgotten. */ + release(handle: object): void { + const dispose: unknown = Reflect.get(handle, Symbol.dispose); + if (typeof dispose === "function") { + try { + Reflect.apply(dispose, handle, []); + } catch { + // Already disposed, or its isolate is gone: let go all the same. + } + } + this.#held.delete(handle); + } +} + /** How a run is limited and followed, besides its code and env. */ export interface RunOptions { /** Cancels the run. */ @@ -441,6 +476,8 @@ export interface RunOptions { * a line after that is dropped, whenever the isolate sends it. */ onOutput?: (line: string) => void; + /** Where the run's entrypoint is held while it runs. */ + handles?: HeldHandles; } /** @@ -454,10 +491,16 @@ export const runCode = async ( loader: WorkerLoader, code: string, env: Readonly>, - { signal, timeoutMs = codeLimits.timeoutMs, onOutput }: RunOptions = {} + { + signal, + timeoutMs = codeLimits.timeoutMs, + onOutput, + handles = new HeldHandles(), + }: RunOptions = {} ): Promise => { const limit = deadline(timeoutMs, signal); let over = false; + let entry: Fetcher | undefined; // What the isolate streams is checked like what it returns: a string, // no longer than a line may be, and only while the run is open. const stream = (line: unknown): void => { @@ -482,9 +525,13 @@ export const runCode = async ( env, limits: { ...sandbox.limits, subRequests: codeLimits.subRequests }, }); - const entry = worker.getEntrypoint(); + const harnessEntry = handles.hold(worker.getEntrypoint()); + entry = harnessEntry; const verified = parsed( - await Promise.race([entry.verify(stream), whenAborted(limit.signal)]) + await Promise.race([ + harnessEntry.verify(stream), + whenAborted(limit.signal), + ]) ); // An outcome that isn't one the harness returns came from the code, // which may change anything in its isolate: it didn't load (verify) @@ -501,7 +548,7 @@ export const runCode = async ( } phase = "run"; const ran = parsed( - await Promise.race([entry.run(stream), whenAborted(limit.signal)]) + await Promise.race([harnessEntry.run(stream), whenAborted(limit.signal)]) ); if (ran === undefined) { return ended("threw", notAResult); @@ -543,6 +590,9 @@ export const runCode = async ( } finally { over = true; limit.clear(); + if (entry !== undefined) { + handles.release(entry); + } } }; diff --git a/apps/core/src/workspace.ts b/apps/core/src/workspace.ts index 54607e53..cb96e115 100644 --- a/apps/core/src/workspace.ts +++ b/apps/core/src/workspace.ts @@ -74,6 +74,7 @@ import { codeTimeoutMs, cutMarker, cutToBytes, + HeldHandles, redact, utf8Bytes, } from "./code-mode.ts"; @@ -399,6 +400,9 @@ export class Workspace extends DurableObject { */ readonly #codeRuns = new Map(); + /** What this object's code runs hold while they run (code-mode.ts). */ + readonly #handles = new HeldHandles(); + /** * What the code step running now in each chat has logged, as its * watchers see it before its result: for one run (`runId`), its UTF-8 @@ -894,6 +898,7 @@ export class Workspace extends DurableObject { output: (runId, callId, line) => { this.codeRunOutput(chat.id, runId, callId, line); }, + handles: this.#handles, }, loader: this.env.LOADER, codeTimeoutMs: codeTimeoutMs(this.env), @@ -1183,6 +1188,32 @@ export class Workspace extends DurableObject { return "revoked"; } + /** + * What code runs hold in this object now, measured: the handles runs + * hold (their entrypoints and env stubs), open and closed leases, and + * outputs shown. After any number of runs, none open: no handles, no + * open leases, no output, and at most {@link endedRunsKept} closed. + */ + codeRunRetention(): { + handles: number; + open: number; + closed: number; + outputs: number; + } { + let open = 0; + for (const lease of this.#codeRuns.values()) { + if (!("closed" in lease)) { + open += 1; + } + } + return { + handles: this.#handles.size, + open, + closed: this.#codeRuns.size - open, + outputs: this.#outputs.size, + }; + } + /** * Takes a line a code run of the chat logged, for its watchers as the * code step `callId`'s output: counted and bounded here, whatever the diff --git a/apps/core/test/agent.test.ts b/apps/core/test/agent.test.ts index 6644efda..00940b5a 100644 --- a/apps/core/test/agent.test.ts +++ b/apps/core/test/agent.test.ts @@ -2511,3 +2511,49 @@ describe("chat agent code output", () => { }); }); }); + +/** Orders text alphabetically. */ +const byText = (a: string, b: string): number => a.localeCompare(b, "en"); + +describe("chat agent code run retention", () => { + it("leaves nothing held after many runs, whether they complete, throw or fail to load", async () => { + const kinds = [ + "export default async () => { console.log('ran'); return 1; };", + "export default async () => { throw new Error('nope'); };", + "export default async () => {", + ]; + const turns = 4; + const replies = Array.from({ length: turns }, (_, turn) => [ + codeStep(kinds[0] ?? "", 5), + codeStep(kinds[1 + (turn % 2)] ?? "", 5), + says("Done."), + ]).flat(); + const { stub, chat, ask } = await newChat(...replies); + + for (let turn = 0; turn < turns; turn += 1) { + // oxlint-disable-next-line no-await-in-loop -- one turn at a time per chat + await ask("Run them."); + } + + // Measured in the object: what its runs hold now, not what their + // cleanup is meant to do. + await expect( + runInDurableObject(stub, (instance) => instance.codeRunRetention()) + ).resolves.toStrictEqual({ + handles: 0, + open: 0, + closed: turns * 10, + outputs: 0, + }); + const records = await runRecords(stub, chat.id); + expect( + records.map(({ status }) => String(status)).toSorted(byText) + ).toStrictEqual( + [ + ...Array.from({ length: 20 }, () => "completed"), + ...Array.from({ length: 10 }, () => "load_failed"), + ...Array.from({ length: 10 }, () => "threw"), + ].toSorted(byText) + ); + }); +}); diff --git a/apps/core/test/workerd/code-mode-worker.ts b/apps/core/test/workerd/code-mode-worker.ts new file mode 100644 index 00000000..245543fd --- /dev/null +++ b/apps/core/test/workerd/code-mode-worker.ts @@ -0,0 +1,133 @@ +import { WorkerEntrypoint, env, exports } from "cloudflare:workers"; + +import { + codeLimits, + codeRuntime, + cutMarker, + describeRun, + runCode, + utf8Bytes, +} from "../../src/code-mode.ts"; +import type { CodeRun } from "../../src/code-mode.ts"; + +// Code Mode's executor on plain workerd (scripts/workerd-code-mode.ts): +// the same `runCode` core runs, with a Worker Loader and a capability of +// this Worker's own as the code's env, and no Wrangler or Miniflare. Each +// case runs one module and says how it ended; the script checks them. + +/** Calls the capability took, across every case. */ +let calls = 0; + +/** A capability the code is given, as core gives its APIs: a loopback stub. */ +export class Caps extends WorkerEntrypoint { + // oxlint-disable-next-line class-methods-use-this -- an RPC method, called on the entrypoint + async double(value: number): Promise { + calls += 1; + await scheduler.wait(1); + return value * 2; + } +} + +interface Case { + code: string; + timeoutMs?: number; + /** Cancels the run this long after it starts. */ + cancelAfterMs?: number; +} + +const cases: Record = { + complete: { + code: "const twice = (env, n) => env.caps.double(n); export default async (self, env, ctx) => ({ self, ctx: Object.keys(ctx), sum: (await Promise.all([twice(env, 1), twice(env, 2)])).reduce((a, b) => a + b, 0) });", + }, + network: { + code: "export default async () => (await fetch('https://example.com')).status;", + }, + syntax: { code: "export default async () => {" }, + notAFunction: { code: "export default 42;" }, + throwsAsItLoads: { + code: "console.log('loading'); throw new Error('broken module'); export default async () => 1;", + }, + throws: { code: "export default async () => { throw new Error('nope'); };" }, + oversized: { + code: `export default async () => 1; // ${"x".repeat(codeLimits.sourceChars)}`, + }, + neverSettles: { + code: "export default async () => { await new Promise(() => {}); };", + }, + waitsPastItsTime: { + code: "export default async () => { await scheduler.wait(60_000); };", + timeoutMs: 1000, + }, + cancelled: { + code: "export default async () => { await scheduler.wait(60_000); };", + cancelAfterMs: 300, + }, + flood: { + code: "export default async () => { for (let i = 0; i < 5000; i++) console.log('é'.repeat(200)); throw new Error('the end'); };", + }, +}; + +/** Whether `value` is a stub a loaded Worker can be given in its env. */ +const isFetcher = (value: unknown): value is Fetcher => + (typeof value === "object" || typeof value === "function") && + value !== null && + "fetch" in value; + +/** A loopback stub of {@link Caps}, as core makes its APIs' stubs. */ +const capability = (): Fetcher => { + const make: unknown = Reflect.get(exports, "Caps"); + if (typeof make !== "function") { + throw new TypeError("This Worker exports no Caps"); + } + const stub: unknown = Reflect.apply(make, exports, [{}]); + if (!isFetcher(stub)) { + throw new TypeError("Caps gave no stub"); + } + return stub; +}; + +/** One case: how it ended, what the model would read, what streamed. */ +const runCase = async ({ code, timeoutMs, cancelAfterMs }: Case) => { + const cancel = new AbortController(); + if (cancelAfterMs !== undefined) { + setTimeout(() => { + cancel.abort(); + }, cancelAfterMs); + } + const streamed: string[] = []; + const run: CodeRun = await runCode( + env.LOADER, + code, + { caps: capability() }, + { + signal: cancel.signal, + ...(timeoutMs === undefined ? {} : { timeoutMs }), + onOutput: (line) => { + streamed.push(line); + }, + } + ); + const text = describeRun(run); + return { + status: run.status, + text: text.slice(0, 400), + bytes: utf8Bytes(text), + cut: text.includes(cutMarker), + streamed: streamed.length, + }; +}; + +export default { + async fetch(): Promise { + const results: Record = {}; + for (const [name, each] of Object.entries(cases)) { + // oxlint-disable-next-line no-await-in-loop -- one run at a time, as a chat runs them + results[name] = await runCase(each); + } + return Response.json({ + runtime: codeRuntime({ RUNTIME_PROFILE: "workerd" }), + calls, + results, + }); + }, +}; diff --git a/package.json b/package.json index b4f2a08a..f0713767 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dev": "node scripts/dev.ts", "e2e": "playwright test", "smoke:workerd": "node scripts/workerd-smoke.ts", + "smoke:code-mode": "node scripts/workerd-code-mode.ts", "check:docs": "node scripts/check-docs.ts", "check:migrations": "node scripts/check-migrations.ts", "check:wrangler-configs": "node scripts/check-wrangler-configs.ts", diff --git a/scripts/workerd-code-mode.ts b/scripts/workerd-code-mode.ts new file mode 100644 index 00000000..75d0a165 --- /dev/null +++ b/scripts/workerd-code-mode.ts @@ -0,0 +1,183 @@ +/** + * Code Mode on plain workerd: bundles the executor core runs + * (apps/core/src/code-mode.ts) into a small Worker + * (apps/core/test/workerd/code-mode-worker.ts), serves it on plain workerd + * with a Worker Loader (no Wrangler, no Miniflare), and checks every case + * ends as it does in core: the module contract, its failures, the limits, + * cancellation, streamed output and the output budget. + * + * Plain workerd enforces no CPU limit on a loaded Worker, and its runs + * record `cpu_limit_unsupported`. Worse than unbounded: a synchronous busy + * loop holds the thread the host's own wall-clock timer would fire on, so + * no case here runs one. The CPU limit is proven on the deployed profile. + */ +import { execFileSync, spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; + +import { z } from "zod"; + +const ATTEMPTS = 100; +const RETRY_DELAY_MS = 100; +const out = mkdtempSync(path.join(tmpdir(), "grasp-os-code-mode-")); +const worker = path.join( + import.meta.dirname, + "../apps/core/test/workerd/code-mode-worker.ts" +); + +execFileSync( + "esbuild", + [ + worker, + "--bundle", + "--format=esm", + "--platform=neutral", + "--main-fields=module,main", + "--external:cloudflare:*", + `--outfile=${path.join(out, "worker.js")}`, + ], + { stdio: "inherit" } +); + +const freePort = async (): Promise => { + const probe = createServer().listen(0, "127.0.0.1"); + await once(probe, "listening"); + const address = probe.address(); + probe.close(); + await once(probe, "close"); + if (address === null || typeof address === "string") { + throw new Error("no free port for workerd"); + } + return address.port; +}; +const PORT = await freePort(); + +writeFileSync( + path.join(out, "config.capnp"), + `using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [(name = "main", worker = .main)], + sockets = [(name = "http", address = "127.0.0.1:${PORT}", http = (), service = "main")], +); + +const main :Workerd.Worker = ( + modules = [(name = "worker.js", esModule = embed "worker.js")], + compatibilityDate = "2026-09-15", + bindings = [(name = "LOADER", workerLoader = ())], +); +` +); + +/** What each case must come to, and the runtime the run records. */ +const expected = { + runtime: { + runtime: "workerd", + cpuLimit: "cpu_limit_unsupported", + }, + calls: 2, + statuses: { + complete: "completed", + network: "threw", + syntax: "load_failed", + notAFunction: "load_failed", + throwsAsItLoads: "load_failed", + throws: "threw", + oversized: "refused", + neverSettles: "wall_expired", + waitsPastItsTime: "wall_expired", + cancelled: "cancelled", + flood: "threw", + }, +}; + +/** What the Worker answers: the runtime, capability calls, each case. */ +const answerSchema = z.object({ + runtime: z.record(z.string(), z.string()), + calls: z.number(), + results: z.record( + z.string(), + z.object({ + status: z.string(), + text: z.string(), + bytes: z.number(), + cut: z.boolean(), + streamed: z.number(), + }) + ), +}); + +const server = spawn( + "workerd", + ["serve", "--experimental", path.join(out, "config.capnp")], + { stdio: "inherit" } +); + +const fetchResults = async (): Promise> => { + for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) { + if (server.exitCode !== null) { + throw new Error(`workerd exited with ${server.exitCode}`); + } + try { + // oxlint-disable-next-line no-await-in-loop -- polled until workerd answers + const response = await fetch(`http://127.0.0.1:${PORT}/`); + // oxlint-disable-next-line no-await-in-loop -- the one answer + return answerSchema.parse(await response.json()); + } catch { + // oxlint-disable-next-line no-await-in-loop -- polled until workerd answers + await sleep(RETRY_DELAY_MS); + } + } + throw new Error("workerd never answered"); +}; + +try { + const { runtime, calls, results } = await fetchResults(); + const problems: string[] = []; + for (const [name, status] of Object.entries(expected.statuses)) { + if (results[name]?.status !== status) { + problems.push( + `${name}: ${results[name]?.status ?? "missing"} (${results[name]?.text ?? ""}), expected ${status}` + ); + } + } + if (calls !== expected.calls) { + problems.push(`capability calls: ${calls}, expected ${expected.calls}`); + } + if ( + runtime.runtime !== expected.runtime.runtime || + runtime.cpuLimit !== expected.runtime.cpuLimit + ) { + problems.push(`runtime: ${JSON.stringify(runtime)}`); + } + const { complete, flood, throwsAsItLoads } = results; + if ( + complete?.text !== 'Returned:\n{"self":null,"ctx":["waitUntil"],"sum":6}' + ) { + problems.push(`complete: ${complete?.text}`); + } + if (!(throwsAsItLoads?.text.startsWith("Logs:\nloading") ?? false)) { + problems.push(`throwsAsItLoads kept no log: ${throwsAsItLoads?.text}`); + } + if ( + flood === undefined || + flood.bytes > 64 * 1024 || + !flood.cut || + flood.streamed === 0 + ) { + problems.push(`flood: ${JSON.stringify(flood)}`); + } + if (problems.length > 0) { + throw new Error(`Code Mode on plain workerd:\n${problems.join("\n")}`); + } + console.info( + `Code Mode runs on plain workerd: ${Object.keys(expected.statuses).length} cases (${JSON.stringify(runtime)})` + ); +} finally { + server.kill(); + rmSync(out, { force: true, recursive: true }); +} From 3149fe7ac25d63f6b3c3d13ac8dd14fc1bb20e42 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 11 Oct 2026 05:57:41 +0200 Subject: [PATCH 2/4] fix(core): bundle the Code Mode smoke run with Wrangler, and claim only what is read The plain-workerd smoke run bundles its Worker with the Wrangler dry run the on-prem smoke run uses, so a frozen install is all it needs. It retries only a refused connection, within an overall timeout, and checks that a flood keeps its ending and that a run past its time ends at its limit. The handle registry goes: it counted core bookkeeping and disposed nothing, and workerd gives no signal of the isolates or RPC sessions a run leaves (it never calls a loopback entrypoint dispose). What is measured after many runs is core own state: no open lease and no output. --- apps/core/src/agent.ts | 14 +--- apps/core/src/code-mode.ts | 58 ++--------------- apps/core/src/workspace.ts | 22 ++----- apps/core/test/agent.test.ts | 11 ++-- apps/core/test/workerd/code-mode-worker.ts | 3 + scripts/workerd-code-mode.ts | 76 ++++++++++++++++------ 6 files changed, 77 insertions(+), 107 deletions(-) diff --git a/apps/core/src/agent.ts b/apps/core/src/agent.ts index 3e74fef2..1c625b7a 100644 --- a/apps/core/src/agent.ts +++ b/apps/core/src/agent.ts @@ -15,7 +15,7 @@ import { z } from "zod"; import type { AgentApi, AgentScope } from "./agent-scope.ts"; import { describeRun, runCode, sourceHash } from "./code-mode.ts"; -import type { CodeRun, HeldHandles } from "./code-mode.ts"; +import type { CodeRun } from "./code-mode.ts"; import type { CodeRunStatus } from "./code-run-statuses.ts"; import type { AgentModel } from "./models.ts"; import type { CodeRunOpening } from "./workspace.ts"; @@ -226,8 +226,6 @@ export interface CodeRuns { * only while the run is open, for the code step it runs for. */ output: (runId: string, toolCallId: string, line: string) => void; - /** Where each run's entrypoint and stubs are held while it runs. */ - handles: HeldHandles; } /** @@ -255,10 +253,7 @@ const runOpen = async ( sourceChars: code.length, }); const env = Object.fromEntries( - apis.map((api) => [ - api.name, - runs.handles.hold(api.stub({ ...scope, runId })), - ]) + apis.map((api) => [api.name, api.stub({ ...scope, runId })]) ); // `runCode` never throws; should anything else, the run still closes. let ran: CodeRunStatus = "crashed"; @@ -268,7 +263,6 @@ const runOpen = async ( run = await runCode(loader, code, env, { signal, timeoutMs: codeTimeoutMs, - handles: runs.handles, onOutput: (line) => { onOutput(runId, line); }, @@ -276,10 +270,6 @@ const runOpen = async ( ran = run.status; } finally { ended = await runs.close(runId, ran); - // The run's stubs go with it. - for (const stub of Object.values(env)) { - runs.handles.release(stub); - } } // A run a revocation closed, as it ran or as it ended, hands on nothing // of what it did: not what it returned, nor what it logged, which could diff --git a/apps/core/src/code-mode.ts b/apps/core/src/code-mode.ts index 46a695ba..bef5d94b 100644 --- a/apps/core/src/code-mode.ts +++ b/apps/core/src/code-mode.ts @@ -430,41 +430,6 @@ const parsed = ( const notAResult = "The run returned more than it may, or not a result."; -/** - * The handles code runs hold: each run's entrypoint into its isolate, and - * the stubs of its env, from when the run starts until it lets go of them - * (`release`). One per Workspace object, so an object that is reset takes - * its own with it. Measured, so a test sees that repeated runs leave none - * behind rather than trusting the cleanup. - */ -export class HeldHandles { - readonly #held = new Set(); - - /** How many handles runs hold now. */ - get size(): number { - return this.#held.size; - } - - /** Takes `handle` as held by a run, until {@link release}. */ - hold(handle: T): T { - this.#held.add(handle); - return handle; - } - - /** Lets go of a run's handle: disposed, where it can be, and forgotten. */ - release(handle: object): void { - const dispose: unknown = Reflect.get(handle, Symbol.dispose); - if (typeof dispose === "function") { - try { - Reflect.apply(dispose, handle, []); - } catch { - // Already disposed, or its isolate is gone: let go all the same. - } - } - this.#held.delete(handle); - } -} - /** How a run is limited and followed, besides its code and env. */ export interface RunOptions { /** Cancels the run. */ @@ -476,8 +441,6 @@ export interface RunOptions { * a line after that is dropped, whenever the isolate sends it. */ onOutput?: (line: string) => void; - /** Where the run's entrypoint is held while it runs. */ - handles?: HeldHandles; } /** @@ -491,16 +454,10 @@ export const runCode = async ( loader: WorkerLoader, code: string, env: Readonly>, - { - signal, - timeoutMs = codeLimits.timeoutMs, - onOutput, - handles = new HeldHandles(), - }: RunOptions = {} + { signal, timeoutMs = codeLimits.timeoutMs, onOutput }: RunOptions = {} ): Promise => { const limit = deadline(timeoutMs, signal); let over = false; - let entry: Fetcher | undefined; // What the isolate streams is checked like what it returns: a string, // no longer than a line may be, and only while the run is open. const stream = (line: unknown): void => { @@ -525,13 +482,9 @@ export const runCode = async ( env, limits: { ...sandbox.limits, subRequests: codeLimits.subRequests }, }); - const harnessEntry = handles.hold(worker.getEntrypoint()); - entry = harnessEntry; + const entry = worker.getEntrypoint(); const verified = parsed( - await Promise.race([ - harnessEntry.verify(stream), - whenAborted(limit.signal), - ]) + await Promise.race([entry.verify(stream), whenAborted(limit.signal)]) ); // An outcome that isn't one the harness returns came from the code, // which may change anything in its isolate: it didn't load (verify) @@ -548,7 +501,7 @@ export const runCode = async ( } phase = "run"; const ran = parsed( - await Promise.race([harnessEntry.run(stream), whenAborted(limit.signal)]) + await Promise.race([entry.run(stream), whenAborted(limit.signal)]) ); if (ran === undefined) { return ended("threw", notAResult); @@ -590,9 +543,6 @@ export const runCode = async ( } finally { over = true; limit.clear(); - if (entry !== undefined) { - handles.release(entry); - } } }; diff --git a/apps/core/src/workspace.ts b/apps/core/src/workspace.ts index cb96e115..86112b16 100644 --- a/apps/core/src/workspace.ts +++ b/apps/core/src/workspace.ts @@ -74,7 +74,6 @@ import { codeTimeoutMs, cutMarker, cutToBytes, - HeldHandles, redact, utf8Bytes, } from "./code-mode.ts"; @@ -400,9 +399,6 @@ export class Workspace extends DurableObject { */ readonly #codeRuns = new Map(); - /** What this object's code runs hold while they run (code-mode.ts). */ - readonly #handles = new HeldHandles(); - /** * What the code step running now in each chat has logged, as its * watchers see it before its result: for one run (`runId`), its UTF-8 @@ -898,7 +894,6 @@ export class Workspace extends DurableObject { output: (runId, callId, line) => { this.codeRunOutput(chat.id, runId, callId, line); }, - handles: this.#handles, }, loader: this.env.LOADER, codeTimeoutMs: codeTimeoutMs(this.env), @@ -1189,17 +1184,13 @@ export class Workspace extends DurableObject { } /** - * What code runs hold in this object now, measured: the handles runs - * hold (their entrypoints and env stubs), open and closed leases, and - * outputs shown. After any number of runs, none open: no handles, no - * open leases, no output, and at most {@link endedRunsKept} closed. + * What code runs keep in this object now, read from its own state: + * leases open and closed, and outputs held. After any number of runs, + * none open and no output, and at most {@link endedRunsKept} closed. + * What the runtime holds for them (isolates, RPC sessions) isn't here: + * workerd offers no signal of it to read. */ - codeRunRetention(): { - handles: number; - open: number; - closed: number; - outputs: number; - } { + codeRunRetention(): { open: number; closed: number; outputs: number } { let open = 0; for (const lease of this.#codeRuns.values()) { if (!("closed" in lease)) { @@ -1207,7 +1198,6 @@ export class Workspace extends DurableObject { } } return { - handles: this.#handles.size, open, closed: this.#codeRuns.size - open, outputs: this.#outputs.size, diff --git a/apps/core/test/agent.test.ts b/apps/core/test/agent.test.ts index 00940b5a..22e10fbb 100644 --- a/apps/core/test/agent.test.ts +++ b/apps/core/test/agent.test.ts @@ -2516,10 +2516,10 @@ describe("chat agent code output", () => { const byText = (a: string, b: string): number => a.localeCompare(b, "en"); describe("chat agent code run retention", () => { - it("leaves nothing held after many runs, whether they complete, throw or fail to load", async () => { + it("keeps no open lease or output after many runs, whether they complete, throw or fail to load", async () => { const kinds = [ - "export default async () => { console.log('ran'); return 1; };", - "export default async () => { throw new Error('nope'); };", + "export default async (self, env) => { console.log((await env.chat.info()).chatId); return 1; };", + "export default async (self, env) => { await env.chat.info(); throw new Error('nope'); };", "export default async () => {", ]; const turns = 4; @@ -2535,12 +2535,11 @@ describe("chat agent code run retention", () => { await ask("Run them."); } - // Measured in the object: what its runs hold now, not what their - // cleanup is meant to do. + // Read from the object's own state after the runs, not inferred from + // their cleanup. What the runtime holds for them can't be read. await expect( runInDurableObject(stub, (instance) => instance.codeRunRetention()) ).resolves.toStrictEqual({ - handles: 0, open: 0, closed: turns * 10, outputs: 0, diff --git a/apps/core/test/workerd/code-mode-worker.ts b/apps/core/test/workerd/code-mode-worker.ts index 245543fd..9c3cc4aa 100644 --- a/apps/core/test/workerd/code-mode-worker.ts +++ b/apps/core/test/workerd/code-mode-worker.ts @@ -95,6 +95,7 @@ const runCase = async ({ code, timeoutMs, cancelAfterMs }: Case) => { }, cancelAfterMs); } const streamed: string[] = []; + const started = Date.now(); const run: CodeRun = await runCode( env.LOADER, code, @@ -111,6 +112,8 @@ const runCase = async ({ code, timeoutMs, cancelAfterMs }: Case) => { return { status: run.status, text: text.slice(0, 400), + ending: text.slice(-200), + ms: Date.now() - started, bytes: utf8Bytes(text), cut: text.includes(cutMarker), streamed: streamed.length, diff --git a/scripts/workerd-code-mode.ts b/scripts/workerd-code-mode.ts index 75d0a165..4db1ecff 100644 --- a/scripts/workerd-code-mode.ts +++ b/scripts/workerd-code-mode.ts @@ -24,23 +24,32 @@ import { z } from "zod"; const ATTEMPTS = 100; const RETRY_DELAY_MS = 100; const out = mkdtempSync(path.join(tmpdir(), "grasp-os-code-mode-")); -const worker = path.join( - import.meta.dirname, - "../apps/core/test/workerd/code-mode-worker.ts" -); +const core = path.join(import.meta.dirname, "../apps/core"); +const worker = path.join(core, "test/workerd/code-mode-worker.ts"); +// Bundled as core's own deploy bundles (workerd-smoke.ts): Wrangler's dry +// run, from a config of the Worker's own, so the bundle is what Wrangler +// would ship and nothing but the frozen install is needed. +const config = path.join(out, "wrangler.jsonc"); +writeFileSync( + config, + JSON.stringify({ + name: "code-mode-smoke", + main: worker, + compatibility_date: "2026-09-15", + }) +); execFileSync( - "esbuild", + "wrangler", [ - worker, - "--bundle", - "--format=esm", - "--platform=neutral", - "--main-fields=module,main", - "--external:cloudflare:*", - `--outfile=${path.join(out, "worker.js")}`, + "deploy", + "--dry-run", + "--outdir", + path.join(out, "bundle"), + "--config", + config, ], - { stdio: "inherit" } + { cwd: core, stdio: "inherit" } ); const freePort = async (): Promise => { @@ -66,7 +75,7 @@ const config :Workerd.Config = ( ); const main :Workerd.Worker = ( - modules = [(name = "worker.js", esModule = embed "worker.js")], + modules = [(name = "worker.js", esModule = embed "bundle/code-mode-worker.js")], compatibilityDate = "2026-09-15", bindings = [(name = "LOADER", workerLoader = ())], ); @@ -106,6 +115,8 @@ const answerSchema = z.object({ text: z.string(), bytes: z.number(), cut: z.boolean(), + ending: z.string(), + ms: z.number(), streamed: z.number(), }) ), @@ -117,22 +128,43 @@ const server = spawn( { stdio: "inherit" } ); +/** The longest the cases may take together, start to answer. */ +const ANSWER_TIMEOUT_MS = 120_000; + +/** + * What the Worker answers, once workerd takes the request: a refused + * connection (workerd still starting) is tried again, until it answers or + * {@link ANSWER_TIMEOUT_MS} passes; anything else fails the run at once. + */ const fetchResults = async (): Promise> => { - for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) { + const until = Date.now() + ANSWER_TIMEOUT_MS; + for (let attempt = 0; ; attempt += 1) { if (server.exitCode !== null) { throw new Error(`workerd exited with ${server.exitCode}`); } try { // oxlint-disable-next-line no-await-in-loop -- polled until workerd answers - const response = await fetch(`http://127.0.0.1:${PORT}/`); + const response = await fetch(`http://127.0.0.1:${PORT}/`, { + signal: AbortSignal.timeout(Math.max(1, until - Date.now())), + }); + if (!response.ok) { + throw new Error(`the Worker answered ${response.status}`); + } // oxlint-disable-next-line no-await-in-loop -- the one answer return answerSchema.parse(await response.json()); - } catch { + } catch (error) { + const refused = + error instanceof TypeError && + error.cause instanceof Error && + "code" in error.cause && + error.cause.code === "ECONNREFUSED"; + if (!refused || attempt >= ATTEMPTS || Date.now() >= until) { + throw error; + } // oxlint-disable-next-line no-await-in-loop -- polled until workerd answers await sleep(RETRY_DELAY_MS); } } - throw new Error("workerd never answered"); }; try { @@ -167,10 +199,16 @@ try { flood === undefined || flood.bytes > 64 * 1024 || !flood.cut || - flood.streamed === 0 + flood.streamed === 0 || + !flood.ending.includes("Error (threw):\nError: the end") ) { problems.push(`flood: ${JSON.stringify(flood)}`); } + const { waitsPastItsTime } = results; + // Its limit is a second: let go at it, not at its minute-long wait. + if ((waitsPastItsTime?.ms ?? Infinity) > 5000) { + problems.push(`waitsPastItsTime took ${waitsPastItsTime?.ms} ms`); + } if (problems.length > 0) { throw new Error(`Code Mode on plain workerd:\n${problems.join("\n")}`); } From 1499b35e3a99f0cfd032aa5d7cbcc90cb4ab9266 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 11 Oct 2026 06:13:28 +0200 Subject: [PATCH 3/4] test(core): check the Code Mode smoke run streams no more lines than a run may log A run that logs five thousand lines streams exactly the thousand a run may log, and a flood never streams more. The header says Wrangler only bundles the Worker. --- apps/core/test/workerd/code-mode-worker.ts | 3 +++ scripts/workerd-code-mode.ts | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/core/test/workerd/code-mode-worker.ts b/apps/core/test/workerd/code-mode-worker.ts index 9c3cc4aa..b4254b03 100644 --- a/apps/core/test/workerd/code-mode-worker.ts +++ b/apps/core/test/workerd/code-mode-worker.ts @@ -62,6 +62,9 @@ const cases: Record = { code: "export default async () => { await scheduler.wait(60_000); };", cancelAfterMs: 300, }, + manyLines: { + code: "export default async () => { for (let i = 0; i < 5000; i++) console.log(i); return 'done'; };", + }, flood: { code: "export default async () => { for (let i = 0; i < 5000; i++) console.log('é'.repeat(200)); throw new Error('the end'); };", }, diff --git a/scripts/workerd-code-mode.ts b/scripts/workerd-code-mode.ts index 4db1ecff..96fae0f3 100644 --- a/scripts/workerd-code-mode.ts +++ b/scripts/workerd-code-mode.ts @@ -1,8 +1,9 @@ /** * Code Mode on plain workerd: bundles the executor core runs * (apps/core/src/code-mode.ts) into a small Worker - * (apps/core/test/workerd/code-mode-worker.ts), serves it on plain workerd - * with a Worker Loader (no Wrangler, no Miniflare), and checks every case + * (apps/core/test/workerd/code-mode-worker.ts) with Wrangler's dry run, + * serves it on plain workerd with a Worker Loader (Wrangler only bundles; + * no Wrangler dev server, no Miniflare), and checks every case * ends as it does in core: the module contract, its failures, the limits, * cancellation, streamed output and the output budget. * @@ -22,6 +23,8 @@ import { setTimeout as sleep } from "node:timers/promises"; import { z } from "zod"; const ATTEMPTS = 100; +/** Most lines a run may log (`codeLimits.logLines` in code-mode.ts). */ +const LOG_LINES = 1000; const RETRY_DELAY_MS = 100; const out = mkdtempSync(path.join(tmpdir(), "grasp-os-code-mode-")); const core = path.join(import.meta.dirname, "../apps/core"); @@ -100,6 +103,7 @@ const expected = { neverSettles: "wall_expired", waitsPastItsTime: "wall_expired", cancelled: "cancelled", + manyLines: "completed", flood: "threw", }, }; @@ -200,11 +204,17 @@ try { flood.bytes > 64 * 1024 || !flood.cut || flood.streamed === 0 || + // Never more lines than a run may log, whatever it wrote. + flood.streamed > LOG_LINES || !flood.ending.includes("Error (threw):\nError: the end") ) { problems.push(`flood: ${JSON.stringify(flood)}`); } - const { waitsPastItsTime } = results; + const { manyLines, waitsPastItsTime } = results; + // Five thousand lines written: never more streamed than a run may log. + if (manyLines === undefined || manyLines.streamed !== LOG_LINES) { + problems.push(`manyLines streamed ${manyLines?.streamed} lines`); + } // Its limit is a second: let go at it, not at its minute-long wait. if ((waitsPastItsTime?.ms ?? Infinity) > 5000) { problems.push(`waitsPastItsTime took ${waitsPastItsTime?.ms} ms`); From 0fc1867427bc10b9e38b33cb0021889f03f903f2 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 11 Oct 2026 15:31:08 +0200 Subject: [PATCH 4/4] test(core): check the closed-lease cap, and clean up after a failed Code Mode smoke run A new test closes more leases than a Workspace object keeps and checks that the oldest go. The smoke run makes its folder and bundles inside the scope that removes it and stops workerd, and reports a signal that ended workerd, whose exit code stays null. --- apps/core/src/workspace.ts | 2 +- apps/core/test/agent.test.ts | 29 ++++++++ scripts/workerd-code-mode.ts | 134 ++++++++++++++++++++--------------- 3 files changed, 107 insertions(+), 58 deletions(-) diff --git a/apps/core/src/workspace.ts b/apps/core/src/workspace.ts index 86112b16..96b7edf8 100644 --- a/apps/core/src/workspace.ts +++ b/apps/core/src/workspace.ts @@ -188,7 +188,7 @@ export const maxChatChars = 4_000_000; * Most ended code runs an object remembers, to refuse and log a call from * one: a few turns' worth (at most 30 runs each). */ -const endedRunsKept = 1000; +export const endedRunsKept = 1000; /** * Most code run records a chat keeps, newest first: a long chat's run diff --git a/apps/core/test/agent.test.ts b/apps/core/test/agent.test.ts index 22e10fbb..aa7cb4ca 100644 --- a/apps/core/test/agent.test.ts +++ b/apps/core/test/agent.test.ts @@ -29,6 +29,7 @@ import { utf8Bytes, } from "../src/code-mode.ts"; import { workspace } from "../src/durable-objects.ts"; +import { endedRunsKept } from "../src/workspace.ts"; import { chatOf, codeResults, @@ -2516,6 +2517,34 @@ describe("chat agent code output", () => { const byText = (a: string, b: string): number => a.localeCompare(b, "en"); describe("chat agent code run retention", () => { + it("keeps no more closed leases than it may, however many runs end", async () => { + const { stub, chat } = await newChat(); + const chatId = chatIdSchema.parse(chat.id); + const runs = endedRunsKept + 50; + + // Leases opened and closed as runs do, past the most it keeps. + const kept = await runInDurableObject(stub, (instance) => { + const ids: string[] = []; + for (let run = 0; run < runs; run += 1) { + const id = instance.openCodeRun(chatId, testRunOpening); + instance.closeCodeRun(chatId, id, "completed"); + ids.push(id); + } + return { + retention: instance.codeRunRetention(), + // The oldest are forgotten, and answer as ended, unlogged. + oldest: instance.callFromCodeRun(chatId, ids[0] ?? ""), + newest: instance.callFromCodeRun(chatId, ids.at(-1) ?? ""), + }; + }); + + expect(kept).toStrictEqual({ + retention: { open: 0, closed: endedRunsKept, outputs: 0 }, + oldest: { call: "ended", first: false }, + newest: { call: "ended", first: true }, + }); + }); + it("keeps no open lease or output after many runs, whether they complete, throw or fail to load", async () => { const kinds = [ "export default async (self, env) => { console.log((await env.chat.info()).chatId); return 1; };", diff --git a/scripts/workerd-code-mode.ts b/scripts/workerd-code-mode.ts index 96fae0f3..dcc67897 100644 --- a/scripts/workerd-code-mode.ts +++ b/scripts/workerd-code-mode.ts @@ -13,6 +13,7 @@ * no case here runs one. The CPU limit is proven on the deployed profile. */ import { execFileSync, spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; import { once } from "node:events"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { createServer } from "node:net"; @@ -26,35 +27,9 @@ const ATTEMPTS = 100; /** Most lines a run may log (`codeLimits.logLines` in code-mode.ts). */ const LOG_LINES = 1000; const RETRY_DELAY_MS = 100; -const out = mkdtempSync(path.join(tmpdir(), "grasp-os-code-mode-")); const core = path.join(import.meta.dirname, "../apps/core"); const worker = path.join(core, "test/workerd/code-mode-worker.ts"); -// Bundled as core's own deploy bundles (workerd-smoke.ts): Wrangler's dry -// run, from a config of the Worker's own, so the bundle is what Wrangler -// would ship and nothing but the frozen install is needed. -const config = path.join(out, "wrangler.jsonc"); -writeFileSync( - config, - JSON.stringify({ - name: "code-mode-smoke", - main: worker, - compatibility_date: "2026-09-15", - }) -); -execFileSync( - "wrangler", - [ - "deploy", - "--dry-run", - "--outdir", - path.join(out, "bundle"), - "--config", - config, - ], - { cwd: core, stdio: "inherit" } -); - const freePort = async (): Promise => { const probe = createServer().listen(0, "127.0.0.1"); await once(probe, "listening"); @@ -66,25 +41,6 @@ const freePort = async (): Promise => { } return address.port; }; -const PORT = await freePort(); - -writeFileSync( - path.join(out, "config.capnp"), - `using Workerd = import "/workerd/workerd.capnp"; - -const config :Workerd.Config = ( - services = [(name = "main", worker = .main)], - sockets = [(name = "http", address = "127.0.0.1:${PORT}", http = (), service = "main")], -); - -const main :Workerd.Worker = ( - modules = [(name = "worker.js", esModule = embed "bundle/code-mode-worker.js")], - compatibilityDate = "2026-09-15", - bindings = [(name = "LOADER", workerLoader = ())], -); -` -); - /** What each case must come to, and the runtime the run records. */ const expected = { runtime: { @@ -126,12 +82,6 @@ const answerSchema = z.object({ ), }); -const server = spawn( - "workerd", - ["serve", "--experimental", path.join(out, "config.capnp")], - { stdio: "inherit" } -); - /** The longest the cases may take together, start to answer. */ const ANSWER_TIMEOUT_MS = 120_000; @@ -140,15 +90,19 @@ const ANSWER_TIMEOUT_MS = 120_000; * connection (workerd still starting) is tried again, until it answers or * {@link ANSWER_TIMEOUT_MS} passes; anything else fails the run at once. */ -const fetchResults = async (): Promise> => { +const fetchResults = async ( + port: number, + stopped: () => string | undefined +): Promise> => { const until = Date.now() + ANSWER_TIMEOUT_MS; for (let attempt = 0; ; attempt += 1) { - if (server.exitCode !== null) { - throw new Error(`workerd exited with ${server.exitCode}`); + const why = stopped(); + if (why !== undefined) { + throw new Error(`workerd ${why}`); } try { // oxlint-disable-next-line no-await-in-loop -- polled until workerd answers - const response = await fetch(`http://127.0.0.1:${PORT}/`, { + const response = await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(Math.max(1, until - Date.now())), }); if (!response.ok) { @@ -162,6 +116,11 @@ const fetchResults = async (): Promise> => { error.cause instanceof Error && "code" in error.cause && error.cause.code === "ECONNREFUSED"; + // workerd may have died with the request: say how, not the reset. + const died = stopped(); + if (died !== undefined) { + throw new Error(`workerd ${died}`, { cause: error }); + } if (!refused || attempt >= ATTEMPTS || Date.now() >= until) { throw error; } @@ -171,8 +130,69 @@ const fetchResults = async (): Promise> => { } }; +// Everything after the temporary folder is made runs inside the scope +// that removes it, and stops workerd, however the run ends: a bundle that +// fails leaves nothing behind. +const out = mkdtempSync(path.join(tmpdir(), "grasp-os-code-mode-")); +let server: ChildProcess | undefined; +/** How workerd stopped, once it has: its exit code or its signal. */ +let stopped: string | undefined; try { - const { runtime, calls, results } = await fetchResults(); + // Bundled as core's own deploy bundles (workerd-smoke.ts): Wrangler's dry + // run, from a config of the Worker's own, so the bundle is what Wrangler + // would ship and nothing but the frozen install is needed. + const config = path.join(out, "wrangler.jsonc"); + writeFileSync( + config, + JSON.stringify({ + name: "code-mode-smoke", + main: worker, + compatibility_date: "2026-09-15", + }) + ); + execFileSync( + "wrangler", + [ + "deploy", + "--dry-run", + "--outdir", + path.join(out, "bundle"), + "--config", + config, + ], + { cwd: core, stdio: "inherit" } + ); + + const port = await freePort(); + writeFileSync( + path.join(out, "config.capnp"), + `using Workerd = import "/workerd/workerd.capnp"; + + const config :Workerd.Config = ( + services = [(name = "main", worker = .main)], + sockets = [(name = "http", address = "127.0.0.1:${port}", http = (), service = "main")], + ); + + const main :Workerd.Worker = ( + modules = [(name = "worker.js", esModule = embed "bundle/code-mode-worker.js")], + compatibilityDate = "2026-09-15", + bindings = [(name = "LOADER", workerLoader = ())], + ); + ` + ); + + server = spawn( + "workerd", + ["serve", "--experimental", path.join(out, "config.capnp")], + { stdio: "inherit" } + ); + // The exit code stays null when workerd dies from a signal: say which. + server.once("exit", (code, signal) => { + stopped = + signal === null ? `exited with ${code}` : `was killed by ${signal}`; + }); + + const { runtime, calls, results } = await fetchResults(port, () => stopped); const problems: string[] = []; for (const [name, status] of Object.entries(expected.statuses)) { if (results[name]?.status !== status) { @@ -226,6 +246,6 @@ try { `Code Mode runs on plain workerd: ${Object.keys(expected.statuses).length} cases (${JSON.stringify(runtime)})` ); } finally { - server.kill(); + server?.kill(); rmSync(out, { force: true, recursive: true }); }