diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 165505da6..ce452fca4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,6 +125,8 @@ jobs: - args: ["--frozen-lockfile"] - run: vp run --filter @grasp-os/web build - run: vp run smoke:workerd + # Code Mode's executor, with a Worker Loader, on plain workerd. + - run: vp run smoke:code-mode # Builds a release bundle as main will publish it (core, connect, the # frontend's static files and D1 migrations, with its manifest) and checks diff --git a/apps/core/src/workspace.ts b/apps/core/src/workspace.ts index 54607e530..96b7edf80 100644 --- a/apps/core/src/workspace.ts +++ b/apps/core/src/workspace.ts @@ -188,7 +188,7 @@ export const maxChatChars = 4_000_000; * Most ended code runs an object remembers, to refuse and log a call from * one: a few turns' worth (at most 30 runs each). */ -const endedRunsKept = 1000; +export const endedRunsKept = 1000; /** * Most code run records a chat keeps, newest first: a long chat's run @@ -1183,6 +1183,27 @@ export class Workspace extends DurableObject { return "revoked"; } + /** + * What code runs keep in this object now, read from its own state: + * leases open and closed, and outputs held. After any number of runs, + * none open and no output, and at most {@link endedRunsKept} closed. + * What the runtime holds for them (isolates, RPC sessions) isn't here: + * workerd offers no signal of it to read. + */ + codeRunRetention(): { open: number; closed: number; outputs: number } { + let open = 0; + for (const lease of this.#codeRuns.values()) { + if (!("closed" in lease)) { + open += 1; + } + } + return { + open, + closed: this.#codeRuns.size - open, + outputs: this.#outputs.size, + }; + } + /** * Takes a line a code run of the chat logged, for its watchers as the * code step `callId`'s output: counted and bounded here, whatever the diff --git a/apps/core/test/agent.test.ts b/apps/core/test/agent.test.ts index 6644efdaa..aa7cb4ca9 100644 --- a/apps/core/test/agent.test.ts +++ b/apps/core/test/agent.test.ts @@ -29,6 +29,7 @@ import { utf8Bytes, } from "../src/code-mode.ts"; import { workspace } from "../src/durable-objects.ts"; +import { endedRunsKept } from "../src/workspace.ts"; import { chatOf, codeResults, @@ -2511,3 +2512,76 @@ describe("chat agent code output", () => { }); }); }); + +/** Orders text alphabetically. */ +const byText = (a: string, b: string): number => a.localeCompare(b, "en"); + +describe("chat agent code run retention", () => { + it("keeps no more closed leases than it may, however many runs end", async () => { + const { stub, chat } = await newChat(); + const chatId = chatIdSchema.parse(chat.id); + const runs = endedRunsKept + 50; + + // Leases opened and closed as runs do, past the most it keeps. + const kept = await runInDurableObject(stub, (instance) => { + const ids: string[] = []; + for (let run = 0; run < runs; run += 1) { + const id = instance.openCodeRun(chatId, testRunOpening); + instance.closeCodeRun(chatId, id, "completed"); + ids.push(id); + } + return { + retention: instance.codeRunRetention(), + // The oldest are forgotten, and answer as ended, unlogged. + oldest: instance.callFromCodeRun(chatId, ids[0] ?? ""), + newest: instance.callFromCodeRun(chatId, ids.at(-1) ?? ""), + }; + }); + + expect(kept).toStrictEqual({ + retention: { open: 0, closed: endedRunsKept, outputs: 0 }, + oldest: { call: "ended", first: false }, + newest: { call: "ended", first: true }, + }); + }); + + it("keeps no open lease or output after many runs, whether they complete, throw or fail to load", async () => { + const kinds = [ + "export default async (self, env) => { console.log((await env.chat.info()).chatId); return 1; };", + "export default async (self, env) => { await env.chat.info(); throw new Error('nope'); };", + "export default async () => {", + ]; + const turns = 4; + const replies = Array.from({ length: turns }, (_, turn) => [ + codeStep(kinds[0] ?? "", 5), + codeStep(kinds[1 + (turn % 2)] ?? "", 5), + says("Done."), + ]).flat(); + const { stub, chat, ask } = await newChat(...replies); + + for (let turn = 0; turn < turns; turn += 1) { + // oxlint-disable-next-line no-await-in-loop -- one turn at a time per chat + await ask("Run them."); + } + + // Read from the object's own state after the runs, not inferred from + // their cleanup. What the runtime holds for them can't be read. + await expect( + runInDurableObject(stub, (instance) => instance.codeRunRetention()) + ).resolves.toStrictEqual({ + open: 0, + closed: turns * 10, + outputs: 0, + }); + const records = await runRecords(stub, chat.id); + expect( + records.map(({ status }) => String(status)).toSorted(byText) + ).toStrictEqual( + [ + ...Array.from({ length: 20 }, () => "completed"), + ...Array.from({ length: 10 }, () => "load_failed"), + ...Array.from({ length: 10 }, () => "threw"), + ].toSorted(byText) + ); + }); +}); diff --git a/apps/core/test/workerd/code-mode-worker.ts b/apps/core/test/workerd/code-mode-worker.ts new file mode 100644 index 000000000..b4254b036 --- /dev/null +++ b/apps/core/test/workerd/code-mode-worker.ts @@ -0,0 +1,139 @@ +import { WorkerEntrypoint, env, exports } from "cloudflare:workers"; + +import { + codeLimits, + codeRuntime, + cutMarker, + describeRun, + runCode, + utf8Bytes, +} from "../../src/code-mode.ts"; +import type { CodeRun } from "../../src/code-mode.ts"; + +// Code Mode's executor on plain workerd (scripts/workerd-code-mode.ts): +// the same `runCode` core runs, with a Worker Loader and a capability of +// this Worker's own as the code's env, and no Wrangler or Miniflare. Each +// case runs one module and says how it ended; the script checks them. + +/** Calls the capability took, across every case. */ +let calls = 0; + +/** A capability the code is given, as core gives its APIs: a loopback stub. */ +export class Caps extends WorkerEntrypoint { + // oxlint-disable-next-line class-methods-use-this -- an RPC method, called on the entrypoint + async double(value: number): Promise { + calls += 1; + await scheduler.wait(1); + return value * 2; + } +} + +interface Case { + code: string; + timeoutMs?: number; + /** Cancels the run this long after it starts. */ + cancelAfterMs?: number; +} + +const cases: Record = { + complete: { + code: "const twice = (env, n) => env.caps.double(n); export default async (self, env, ctx) => ({ self, ctx: Object.keys(ctx), sum: (await Promise.all([twice(env, 1), twice(env, 2)])).reduce((a, b) => a + b, 0) });", + }, + network: { + code: "export default async () => (await fetch('https://example.com')).status;", + }, + syntax: { code: "export default async () => {" }, + notAFunction: { code: "export default 42;" }, + throwsAsItLoads: { + code: "console.log('loading'); throw new Error('broken module'); export default async () => 1;", + }, + throws: { code: "export default async () => { throw new Error('nope'); };" }, + oversized: { + code: `export default async () => 1; // ${"x".repeat(codeLimits.sourceChars)}`, + }, + neverSettles: { + code: "export default async () => { await new Promise(() => {}); };", + }, + waitsPastItsTime: { + code: "export default async () => { await scheduler.wait(60_000); };", + timeoutMs: 1000, + }, + cancelled: { + code: "export default async () => { await scheduler.wait(60_000); };", + cancelAfterMs: 300, + }, + manyLines: { + code: "export default async () => { for (let i = 0; i < 5000; i++) console.log(i); return 'done'; };", + }, + flood: { + code: "export default async () => { for (let i = 0; i < 5000; i++) console.log('é'.repeat(200)); throw new Error('the end'); };", + }, +}; + +/** Whether `value` is a stub a loaded Worker can be given in its env. */ +const isFetcher = (value: unknown): value is Fetcher => + (typeof value === "object" || typeof value === "function") && + value !== null && + "fetch" in value; + +/** A loopback stub of {@link Caps}, as core makes its APIs' stubs. */ +const capability = (): Fetcher => { + const make: unknown = Reflect.get(exports, "Caps"); + if (typeof make !== "function") { + throw new TypeError("This Worker exports no Caps"); + } + const stub: unknown = Reflect.apply(make, exports, [{}]); + if (!isFetcher(stub)) { + throw new TypeError("Caps gave no stub"); + } + return stub; +}; + +/** One case: how it ended, what the model would read, what streamed. */ +const runCase = async ({ code, timeoutMs, cancelAfterMs }: Case) => { + const cancel = new AbortController(); + if (cancelAfterMs !== undefined) { + setTimeout(() => { + cancel.abort(); + }, cancelAfterMs); + } + const streamed: string[] = []; + const started = Date.now(); + const run: CodeRun = await runCode( + env.LOADER, + code, + { caps: capability() }, + { + signal: cancel.signal, + ...(timeoutMs === undefined ? {} : { timeoutMs }), + onOutput: (line) => { + streamed.push(line); + }, + } + ); + const text = describeRun(run); + return { + status: run.status, + text: text.slice(0, 400), + ending: text.slice(-200), + ms: Date.now() - started, + bytes: utf8Bytes(text), + cut: text.includes(cutMarker), + streamed: streamed.length, + }; +}; + +export default { + async fetch(): Promise { + const results: Record = {}; + for (const [name, each] of Object.entries(cases)) { + // oxlint-disable-next-line no-await-in-loop -- one run at a time, as a chat runs them + results[name] = await runCase(each); + } + return Response.json({ + runtime: codeRuntime({ RUNTIME_PROFILE: "workerd" }), + calls, + results, + }); + }, +}; diff --git a/package.json b/package.json index b4f2a08a5..f0713767e 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dev": "node scripts/dev.ts", "e2e": "playwright test", "smoke:workerd": "node scripts/workerd-smoke.ts", + "smoke:code-mode": "node scripts/workerd-code-mode.ts", "check:docs": "node scripts/check-docs.ts", "check:migrations": "node scripts/check-migrations.ts", "check:wrangler-configs": "node scripts/check-wrangler-configs.ts", diff --git a/scripts/workerd-code-mode.ts b/scripts/workerd-code-mode.ts new file mode 100644 index 000000000..dcc67897c --- /dev/null +++ b/scripts/workerd-code-mode.ts @@ -0,0 +1,251 @@ +/** + * Code Mode on plain workerd: bundles the executor core runs + * (apps/core/src/code-mode.ts) into a small Worker + * (apps/core/test/workerd/code-mode-worker.ts) with Wrangler's dry run, + * serves it on plain workerd with a Worker Loader (Wrangler only bundles; + * no Wrangler dev server, no Miniflare), and checks every case + * ends as it does in core: the module contract, its failures, the limits, + * cancellation, streamed output and the output budget. + * + * Plain workerd enforces no CPU limit on a loaded Worker, and its runs + * record `cpu_limit_unsupported`. Worse than unbounded: a synchronous busy + * loop holds the thread the host's own wall-clock timer would fire on, so + * no case here runs one. The CPU limit is proven on the deployed profile. + */ +import { execFileSync, spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; + +import { z } from "zod"; + +const ATTEMPTS = 100; +/** Most lines a run may log (`codeLimits.logLines` in code-mode.ts). */ +const LOG_LINES = 1000; +const RETRY_DELAY_MS = 100; +const core = path.join(import.meta.dirname, "../apps/core"); +const worker = path.join(core, "test/workerd/code-mode-worker.ts"); + +const freePort = async (): Promise => { + const probe = createServer().listen(0, "127.0.0.1"); + await once(probe, "listening"); + const address = probe.address(); + probe.close(); + await once(probe, "close"); + if (address === null || typeof address === "string") { + throw new Error("no free port for workerd"); + } + return address.port; +}; +/** What each case must come to, and the runtime the run records. */ +const expected = { + runtime: { + runtime: "workerd", + cpuLimit: "cpu_limit_unsupported", + }, + calls: 2, + statuses: { + complete: "completed", + network: "threw", + syntax: "load_failed", + notAFunction: "load_failed", + throwsAsItLoads: "load_failed", + throws: "threw", + oversized: "refused", + neverSettles: "wall_expired", + waitsPastItsTime: "wall_expired", + cancelled: "cancelled", + manyLines: "completed", + flood: "threw", + }, +}; + +/** What the Worker answers: the runtime, capability calls, each case. */ +const answerSchema = z.object({ + runtime: z.record(z.string(), z.string()), + calls: z.number(), + results: z.record( + z.string(), + z.object({ + status: z.string(), + text: z.string(), + bytes: z.number(), + cut: z.boolean(), + ending: z.string(), + ms: z.number(), + streamed: z.number(), + }) + ), +}); + +/** The longest the cases may take together, start to answer. */ +const ANSWER_TIMEOUT_MS = 120_000; + +/** + * What the Worker answers, once workerd takes the request: a refused + * connection (workerd still starting) is tried again, until it answers or + * {@link ANSWER_TIMEOUT_MS} passes; anything else fails the run at once. + */ +const fetchResults = async ( + port: number, + stopped: () => string | undefined +): Promise> => { + const until = Date.now() + ANSWER_TIMEOUT_MS; + for (let attempt = 0; ; attempt += 1) { + const why = stopped(); + if (why !== undefined) { + throw new Error(`workerd ${why}`); + } + try { + // oxlint-disable-next-line no-await-in-loop -- polled until workerd answers + const response = await fetch(`http://127.0.0.1:${port}/`, { + signal: AbortSignal.timeout(Math.max(1, until - Date.now())), + }); + if (!response.ok) { + throw new Error(`the Worker answered ${response.status}`); + } + // oxlint-disable-next-line no-await-in-loop -- the one answer + return answerSchema.parse(await response.json()); + } catch (error) { + const refused = + error instanceof TypeError && + error.cause instanceof Error && + "code" in error.cause && + error.cause.code === "ECONNREFUSED"; + // workerd may have died with the request: say how, not the reset. + const died = stopped(); + if (died !== undefined) { + throw new Error(`workerd ${died}`, { cause: error }); + } + if (!refused || attempt >= ATTEMPTS || Date.now() >= until) { + throw error; + } + // oxlint-disable-next-line no-await-in-loop -- polled until workerd answers + await sleep(RETRY_DELAY_MS); + } + } +}; + +// Everything after the temporary folder is made runs inside the scope +// that removes it, and stops workerd, however the run ends: a bundle that +// fails leaves nothing behind. +const out = mkdtempSync(path.join(tmpdir(), "grasp-os-code-mode-")); +let server: ChildProcess | undefined; +/** How workerd stopped, once it has: its exit code or its signal. */ +let stopped: string | undefined; +try { + // Bundled as core's own deploy bundles (workerd-smoke.ts): Wrangler's dry + // run, from a config of the Worker's own, so the bundle is what Wrangler + // would ship and nothing but the frozen install is needed. + const config = path.join(out, "wrangler.jsonc"); + writeFileSync( + config, + JSON.stringify({ + name: "code-mode-smoke", + main: worker, + compatibility_date: "2026-09-15", + }) + ); + execFileSync( + "wrangler", + [ + "deploy", + "--dry-run", + "--outdir", + path.join(out, "bundle"), + "--config", + config, + ], + { cwd: core, stdio: "inherit" } + ); + + const port = await freePort(); + writeFileSync( + path.join(out, "config.capnp"), + `using Workerd = import "/workerd/workerd.capnp"; + + const config :Workerd.Config = ( + services = [(name = "main", worker = .main)], + sockets = [(name = "http", address = "127.0.0.1:${port}", http = (), service = "main")], + ); + + const main :Workerd.Worker = ( + modules = [(name = "worker.js", esModule = embed "bundle/code-mode-worker.js")], + compatibilityDate = "2026-09-15", + bindings = [(name = "LOADER", workerLoader = ())], + ); + ` + ); + + server = spawn( + "workerd", + ["serve", "--experimental", path.join(out, "config.capnp")], + { stdio: "inherit" } + ); + // The exit code stays null when workerd dies from a signal: say which. + server.once("exit", (code, signal) => { + stopped = + signal === null ? `exited with ${code}` : `was killed by ${signal}`; + }); + + const { runtime, calls, results } = await fetchResults(port, () => stopped); + const problems: string[] = []; + for (const [name, status] of Object.entries(expected.statuses)) { + if (results[name]?.status !== status) { + problems.push( + `${name}: ${results[name]?.status ?? "missing"} (${results[name]?.text ?? ""}), expected ${status}` + ); + } + } + if (calls !== expected.calls) { + problems.push(`capability calls: ${calls}, expected ${expected.calls}`); + } + if ( + runtime.runtime !== expected.runtime.runtime || + runtime.cpuLimit !== expected.runtime.cpuLimit + ) { + problems.push(`runtime: ${JSON.stringify(runtime)}`); + } + const { complete, flood, throwsAsItLoads } = results; + if ( + complete?.text !== 'Returned:\n{"self":null,"ctx":["waitUntil"],"sum":6}' + ) { + problems.push(`complete: ${complete?.text}`); + } + if (!(throwsAsItLoads?.text.startsWith("Logs:\nloading") ?? false)) { + problems.push(`throwsAsItLoads kept no log: ${throwsAsItLoads?.text}`); + } + if ( + flood === undefined || + flood.bytes > 64 * 1024 || + !flood.cut || + flood.streamed === 0 || + // Never more lines than a run may log, whatever it wrote. + flood.streamed > LOG_LINES || + !flood.ending.includes("Error (threw):\nError: the end") + ) { + problems.push(`flood: ${JSON.stringify(flood)}`); + } + const { manyLines, waitsPastItsTime } = results; + // Five thousand lines written: never more streamed than a run may log. + if (manyLines === undefined || manyLines.streamed !== LOG_LINES) { + problems.push(`manyLines streamed ${manyLines?.streamed} lines`); + } + // Its limit is a second: let go at it, not at its minute-long wait. + if ((waitsPastItsTime?.ms ?? Infinity) > 5000) { + problems.push(`waitsPastItsTime took ${waitsPastItsTime?.ms} ms`); + } + if (problems.length > 0) { + throw new Error(`Code Mode on plain workerd:\n${problems.join("\n")}`); + } + console.info( + `Code Mode runs on plain workerd: ${Object.keys(expected.statuses).length} cases (${JSON.stringify(runtime)})` + ); +} finally { + server?.kill(); + rmSync(out, { force: true, recursive: true }); +}