diff --git a/apps/core/src/agent.ts b/apps/core/src/agent.ts index cd92fc1c..2a53d96b 100644 --- a/apps/core/src/agent.ts +++ b/apps/core/src/agent.ts @@ -11,10 +11,11 @@ import { toToolDeclaration } from "@earendil-works/pi-ai/utils/transcript"; import { agentErrors } from "@grasp-os/shared/agent"; import type { CatalogSkill } from "@grasp-os/shared/knowledge"; import type { Memory } from "@grasp-os/shared/memory"; +import { z } from "zod"; import type { AgentApi, AgentScope } from "./agent-scope.ts"; import { describeRun, runCode } from "./code-mode.ts"; -import type { CodeRun } from "./code-mode.ts"; +import type { CodeRun, CodeRunStatus } from "./code-mode.ts"; import type { AgentModel } from "./models.ts"; // The agent loop: Pi (pi-agent-core) in Code Mode. The model answers, or @@ -40,10 +41,10 @@ export const maxRunsPerTurn = 30; const instructions = `You are the Grasp assistant, in a chat with one person. You answer their questions, and you look things up or act for them by writing code. You act on their behalf and can do only what they may do, as far as you were given access in this workspace: when something is refused, say so, and don't try another way around it. -To use an API, call the \`executeCode\` tool with a JavaScript module whose default export is an async function of \`env\`: +To use an API, call the \`executeCode\` tool with a complete JavaScript module whose default export is an async function of \`(self, env, ctx)\`. \`env\` holds the APIs; \`self\` is null and \`ctx\` gives you nothing you need. Helper functions in the module are fine: \`\`\`js -export default async (env) => { +export default async (self, env, ctx) => { const info = await env.chat.info(); return info.now; }; @@ -179,12 +180,33 @@ const systemUpdates = ( : [{ role: "system", content: "", sections: changed, timestamp }]; }; -const codeParameters = Type.Object({ - code: Type.String({ - description: - "A complete JavaScript module whose default export is an async function of `env`: `export default async (env) => { ... }`.", - }), -}); +const codeParameters = Type.Object( + { + code: Type.String({ + description: + "A complete JavaScript module whose default export is an async function of `(self, env, ctx)`: `export default async (self, env, ctx) => { ... }`.", + }), + }, + { additionalProperties: false } +); + +/** + * The tool's input, checked before Pi's own validation: only `code`, a + * string. Refused with a short message rather than Pi's, which repeats the + * whole input back to the model. Its length is checked by the host as the + * run starts (`runCode`), so an oversized module is a run that was refused. + */ +const codeInputSchema = z.strictObject({ code: z.string() }); + +const codeInput = (args: unknown): { code: string } => { + const input = codeInputSchema.safeParse(args); + if (!input.success) { + throw new Error( + "executeCode takes one field, `code`: the module as a string. Nothing else is accepted." + ); + } + return input.data; +}; /** * The code runs of a chat that are running now. A stub answers only while @@ -203,7 +225,8 @@ const runOpen = async ( scope, runs, loader, - }: Pick, + codeTimeoutMs, + }: Pick, code: string, signal: AbortSignal | undefined ): Promise => { @@ -212,7 +235,7 @@ const runOpen = async ( apis.map((api) => [api.name, api.stub({ ...scope, runId })]) ); try { - return await runCode(loader, code, env, signal); + return await runCode(loader, code, env, signal, codeTimeoutMs); } finally { runs.close(runId); } @@ -224,26 +247,31 @@ const executeCode = ({ scope, runs, loader, -}: Pick): AgentTool< - typeof codeParameters -> => ({ + codeTimeoutMs, +}: Pick< + Turn, + "apis" | "scope" | "runs" | "loader" | "codeTimeoutMs" +>): AgentTool => ({ name: "executeCode", label: "Run code", description: "Runs a JavaScript module in a sandbox with the chat's APIs as `env`, and returns what its default export returns, what it logs, or what it throws.", parameters: codeParameters, + prepareArguments: codeInput, executionMode: "sequential", execute: async (_toolCallId, { code }, signal) => { - const run = await runOpen({ apis, scope, runs, loader }, code, signal); - if (!run.ok) { - // pi hands a thrown error's message to the model as a failed result. - throw new Error(describeRun(run)); - } - // Only the text the model reads is kept: it is bounded, and the run's - // own output (which the transcript stores) needn't be. + const run = await runOpen( + { apis, scope, runs, loader, codeTimeoutMs }, + code, + signal + ); + // Only the text the model reads is kept, with how the run ended: it is + // bounded, and the run's own output (which the transcript stores) + // needn't be. A run that didn't complete is a failed result. return { content: [{ type: "text", text: describeRun(run) }], - details: undefined, + details: { status: run.status }, + isError: run.status !== "completed", }; }, }); @@ -271,6 +299,8 @@ export interface Turn { scope: Omit; runs: CodeRuns; loader: WorkerLoader; + /** Each code run's wall-clock limit (`codeTimeoutMs`). */ + codeTimeoutMs: number; /** Cancels the turn: the request or code run in flight stops. */ signal: AbortSignal; /** Keeps a finished message: called in order, as each one finishes. */ @@ -546,6 +576,7 @@ export const runTurn = async ({ scope, runs, loader, + codeTimeoutMs, signal: cancelled, keep, write, @@ -559,7 +590,7 @@ export const runTurn = async ({ { role: "user", content: question, timestamp: Date.now() }, ]; const progress: Progress = { steps: 0, runs: 0 }; - const tools = [executeCode({ apis, scope, runs, loader })]; + const tools = [executeCode({ apis, scope, runs, loader, codeTimeoutMs })]; const room = { chars: requestChars(model.inputTokens), tools }; // A question that doesn't fit next to the rest of what the request // carries is refused here, before anything is kept or sent: the provider diff --git a/apps/core/src/code-mode.ts b/apps/core/src/code-mode.ts index 6acc7e9b..f56e79da 100644 --- a/apps/core/src/code-mode.ts +++ b/apps/core/src/code-mode.ts @@ -15,19 +15,30 @@ import { sandbox } from "./sandbox.ts"; // back as data, and core bounds it again: the code runs in the same // isolate as the harness, so nothing the harness does is trusted. // +// The code is a complete ES module whose default export is an async +// function of `(self, env, ctx)`: `self` is null (no originating-session +// capability is offered in this release), `env` holds the stubs, and `ctx` +// is the run's own, with nothing of core's in it. Two modules load: the +// harness, which core wrote, and the code. The harness's `verify` imports +// the code, so what its module does as it loads (it is untrusted too) runs +// under the same limits and is told apart from a failure of the function; +// `run` then calls it. +// // The CPU limit is enforced by Cloudflare's runtime, not by plain workerd: // on-prem, without that limit, a run could spin forever, so the agent // needs a limit of its own there before it is used. /** - * How far one run may go beyond App code's CPU limit: calls out of the - * isolate (its API calls, which the Workspace object counts on every call, - * `callFromCodeRun`; the runtime's own cap too, where it enforces one: - * plain workerd doesn't), and wall-clock time, which also ends a run that - * waits forever. Memory is the runtime's per-isolate limit; Worker Loader - * has no setting for less. + * How far one run may go beyond App code's CPU limit: the source the + * model may send, calls out of the isolate (its API calls, which the + * Workspace object counts on every call, `callFromCodeRun`; the runtime's + * own cap too, where it enforces one: plain workerd doesn't), and + * wall-clock time, which also ends a run that waits forever, counted from + * before the code loads. Memory is the runtime's per-isolate limit; Worker + * Loader has no setting for less. */ export const codeLimits = { + sourceChars: 200_000, subRequests: 100, timeoutMs: 30_000, /** Most characters of logs and result that go back to the model. */ @@ -36,198 +47,444 @@ export const codeLimits = { logLines: 1000, } as const; +/** + * How a run ended, as the host saw it, never as the code says: it + * returned (`completed`), threw (`threw`, or sent back what the harness + * never returns), was refused before it loaded (`refused`: its source is + * over the limit), its module didn't load or export a function + * (`load_failed`, including anything the runtime ended while it loaded, + * short of its CPU time), the runtime stopped it for CPU time + * (`cpu_exceeded`) or anything else once it ran (`crashed`: memory, say), + * it ran past its wall-clock time (`wall_expired`, a wait that can never + * settle too), or the turn was cancelled (`cancelled`). Only the host + * sets one: the harness returns every outcome of the code, so the code + * can't make a call reject, nor pick the message a status is read from. + */ +export type CodeRunStatus = + | "completed" + | "threw" + | "refused" + | "load_failed" + | "cpu_exceeded" + | "crashed" + | "wall_expired" + | "cancelled"; + /** * Most characters of each part of a run that core takes from the isolate: * more than the model reads, so it can be told what was cut. */ const room = 2 * codeLimits.outputChars; +/** + * A run's wall-clock limit: {@link codeLimits}' `timeoutMs`, or less where + * tests set `CODE_RUN_TIMEOUT_MS`; it is never set in wrangler.jsonc, and + * never by the model. + */ +export const codeTimeoutMs = ( + env: Pick +): number => { + const set = Number(env.CODE_RUN_TIMEOUT_MS); + return Number.isInteger(set) && set > 0 && set < codeLimits.timeoutMs + ? set + : codeLimits.timeoutMs; +}; + +/** What the code must export, as the model is told when it doesn't. */ +const exportNeeded = + "The module's default export must be an async function: export default async (self, env, ctx) => { ... }"; + /** * Runs in the isolate: takes what it needs of the language first, removes - * the Cache API, and only then loads the code, which can't change what the - * harness already holds. Calls the code's default export with the env, and - * returns its result, its logs and what it threw. An env name the code - * wasn't given is refused by name, instead of failing later as `undefined`. + * the Cache API and takes over `console`, and only then loads the code, + * which can't change what the harness already holds. `verify` imports the + * code and checks its default export; `run` calls it with `(null, env, + * ctx)`. Both return what was logged so far, from the code's module and + * function alike. An env name the code wasn't given is refused by name, + * instead of failing later as `undefined`. */ const harness = ` import { WorkerEntrypoint } from "cloudflare:workers"; +// Everything the harness uses of the language, taken before the code +// loads: the code can replace any global or prototype after, never these. const room = ${room}; const lines = ${codeLimits.logLines}; -const slice = Function.prototype.call.bind(String.prototype.slice); +const exportNeeded = ${JSON.stringify(exportNeeded)}; +const call = Function.prototype.call; +const bind = (method) => call.bind(method); +const slice = bind(String.prototype.slice); +const map = bind(Array.prototype.map); +const join = bind(Array.prototype.join); +const keys = Object.keys; +const freeze = Object.freeze; +const has = Reflect.has; +const get = Reflect.get; const stringify = JSON.stringify; const toText = String; -const push = Function.prototype.call.bind(Array.prototype.push); -const tooLarge = new Error("too large"); +const ProxyOf = Proxy; +const ErrorOf = Error; +const tooLarge = {}; +const undescribable = "(an error that can't be described)"; Object.defineProperty(globalThis, "caches", { value: undefined }); -// A value as text: serializing stops as soon as it would pass room, -// instead of building all of a huge value first. +// A value as text, never throwing, whatever the value does: serializing +// stops as soon as it would pass room, instead of building all of a huge +// value first. const render = (value) => { if (typeof value === "string") { return value; } let size = 0; + try { + const json = stringify(value, (key, item) => { + size += key.length + (typeof item === "string" ? item.length : 8); + if (size > room) { + throw tooLarge; + } + return item; + }); + return json === undefined ? toText(value) : json; + } catch (error) { + if (error === tooLarge) { + return "(a value too large to show, over " + room + " characters)"; + } + try { + return toText(value); + } catch { + return "(a value that can't be shown)"; + } + } +}; + +// Never longer than room, and never throwing, whatever the value makes +// of itself (a \`toString\` of its own, say). +const show = (value) => { + try { + return slice(toText(render(value)), 0, room); + } catch { + return "(a value that can't be shown)"; + } +}; + +// What the code threw, as text: its stack when it has one, never +// throwing (a thrown Proxy's traps may throw anything). +const describe = (error) => { + try { + const stack = + typeof error === "object" && error !== null ? get(error, "stack") : undefined; + return show(typeof stack === "string" ? stack : error); + } catch { + return undescribable; + } +}; + +// What the code logs, as it loads and as it runs: one run per isolate. +// Kept as the JSON of its lines, built from strings only: no array or +// object of the harness's that a prototype the code changed could reach +// (a setter on \`Array.prototype[0]\` fires on an array's [[Set]]). +let logsJson = ""; +let logCount = 0; +let left = room; +const write = (...parts) => { + if (left > 0 && logCount < lines) { + let line = ""; + for (let index = 0; index < parts.length; index += 1) { + const part = show(parts[index]); + line = line === "" ? part : line + " " + part; + } + line = slice(line, 0, left); + left -= line.length + 1; + const quoted = stringify(line); + logsJson = logCount === 0 ? quoted : logsJson + "," + quoted; + logCount += 1; + } +}; + +// An outcome as the one string a call returns: its fields are all +// primitives, put together as JSON by hand, so nothing the code changed +// runs as it is built or as the runtime sends it. A string can't make +// the call reject. +const outcome = (ok, loaded, field, text) => { try { return ( - stringify(value, (key, item) => { - size += key.length + (typeof item === "string" ? item.length : 8); - if (size > room) { - throw tooLarge; - } - return item; - }) ?? toText(value) + '{"ok":' + (ok ? "true" : "false") + + ',"loaded":' + (loaded ? "true" : "false") + + ',"logs":[' + logsJson + "]" + + (typeof text === "string" ? "," + stringify(field) + ":" + stringify(text) : "") + + "}" ); - } catch (error) { - return error === tooLarge - ? "(a value too large to show, over " + room + " characters)" - : toText(value); + } catch { + return '{"ok":false,"loaded":' + (loaded ? "true" : "false") + ',"logs":[],"error":"(an outcome that could not be sent)"}'; } }; +for (const level of ["log", "info", "warn", "error", "debug"]) { + console[level] = write; +} -// Never longer than room, whatever the value makes of itself (a -// \`toString\` of its own, say). -const show = (value) => slice(toText(render(value)), 0, room); +// The code's default export, once \`verify\` found it a function. +let code; -export default class extends WorkerEntrypoint { - async run() { - const logs = []; - let left = room; - const write = (...parts) => { - if (left > 0 && logs.length < lines) { - let line = ""; - for (const part of parts) { - line = line === "" ? show(part) : line + " " + show(part); - } - line = slice(line, 0, left); - left -= line.length + 1; - push(logs, line); +// Every outcome is returned, never thrown: a call that rejects was ended +// by the runtime, so the code can't choose how its run ends. +class Harness extends WorkerEntrypoint { + async verify() { + try { + const { default: found } = await import("code.js"); + if (typeof found !== "function") { + return outcome(false, false, "error", exportNeeded); } - }; - for (const level of ["log", "info", "warn", "error", "debug"]) { - console[level] = write; + code = found; + return outcome(true, true); + } catch (error) { + return outcome(false, false, "error", describe(error)); + } + } + + async run() { + if (code === undefined) { + return outcome(false, false, "error", exportNeeded); } - const given = Object.keys(this.env); - const env = new Proxy(this.env, { - get(target, name) { - if (typeof name === "string" && name !== "then" && !(name in target)) { - throw new Error( - "This chat has no API named env." + name + ". It has: " + - (given.length === 0 ? "none" : given.map((api) => "env." + api).join(", ")) + "." - ); - } - return target[name]; - }, - }); try { - const { default: code } = await import("code.js"); - if (typeof code !== "function") { - throw new TypeError("The module's default export must be an async function: export default async (env) => { ... }"); - } - const value = await code(env); - // An own \`then\`, so an \`Object.prototype.then\` the code set can't - // make the result a thenable that resolves to something else. - return { then: undefined, ok: true, logs, result: value === undefined ? undefined : show(value) }; + const given = keys(this.env); + const env = new ProxyOf(this.env, { + get(target, name) { + if (typeof name === "string" && name !== "then" && !has(target, name)) { + throw new ErrorOf( + "This chat has no API named env." + name + ". It has: " + + (given.length === 0 ? "none" : join(map(given, (api) => "env." + api), ", ")) + "." + ); + } + return get(target, name); + }, + }); + // The run's own context: waiting on more work extends nothing of + // core's (the run's stubs answer only while the host keeps it open). + const own = this.ctx; + const ctx = freeze({ waitUntil: (promise) => own.waitUntil(promise) }); + const value = await code(null, env, ctx); + return outcome(true, true, "result", value === undefined ? undefined : show(value)); } catch (error) { - return { - then: undefined, - ok: false, - logs, - error: error instanceof Error ? show(error.stack ?? toText(error)) : show(error), - }; + return outcome(false, true, "error", describe(error)); } } } + +// Frozen before the code loads: it can't replace \`verify\` or \`run\`, even +// by importing this module. +freeze(Harness.prototype); +freeze(Harness); + +export default Harness; `; +const logsSchema = z.array(z.string().max(room)).max(codeLimits.logLines); + +/** All of what the isolate sends: the logs share one room, the rest another. */ +const withinRoom = ({ + logs, + result, + error, +}: { + logs: string[]; + result?: string; + error?: string; +}): boolean => + logs.reduce((total, line) => total + line.length, 0) + + (result?.length ?? 0) + + (error?.length ?? 0) <= + 2 * room; + /** * What the isolate sends back, bounded here whatever the harness did: the * agent's code runs in the same isolate and can change anything there. */ -const runSchema = z +const outcomeSchema = z .object({ ok: z.boolean(), - logs: z.array(z.string().max(room)).max(codeLimits.logLines), + /** Whether the code's function was there to call (`verify` found it). */ + loaded: z.boolean(), + logs: logsSchema, result: z.string().max(room).optional(), error: z.string().max(room).optional(), }) - // All of it: the logs share one room, the result or error has another. - .refine( - ({ logs, result, error }) => - logs.reduce((total, line) => total + line.length, 0) + - (result?.length ?? 0) + - (error?.length ?? 0) <= - 2 * room, - { message: "More than a run may send back" } - ); - -/** One run of the agent's code. */ -export type CodeRun = z.infer; + .refine(withinRoom, { message: "More than a run may send back" }); + +/** One run of the agent's code, as the host ended it. */ +export interface CodeRun { + status: CodeRunStatus; + logs: string[]; + result?: string; + error?: string; +} /** * The harness's entrypoint, as core calls it. What it says it returns is * checked, not trusted. */ interface Harness extends WorkerEntrypoint { - run: () => Promise; + verify: () => Promise; + run: () => Promise; } -const failed = (error: string): CodeRun => ({ ok: false, logs: [], error }); +/** + * The message the runtime ends a run with when it used up its CPU time. + * Matched only on a call that rejected, which the harness never makes + * happen (it returns every outcome), and whole: the code can't make a + * call reject, nor choose the message. + */ +const cpuExceeded = /^Worker exceeded CPU time limit\.?$/u; + +/** + * A message the runtime ends a run with when its code waits on nothing + * that can ever settle (no timer, no call, no I/O): it would wait past its + * wall-clock time, so it ends as if it had. + */ +const hung = + "The Workers runtime canceled this request because it detected that your Worker's code had hung"; + +const ended = (status: CodeRunStatus, error: string): CodeRun => ({ + status, + logs: [], + error, +}); + +/** + * The longest outcome string the harness can send for output within its + * limits: every character of the logs and the result or error escaped as + * JSON at its worst (six characters, as for \u0001 or a lone surrogate), + * each log line's quotes and comma, and the field names and punctuation. + * Past it, the string isn't the harness's; the decoded limits are checked + * again once it is parsed (`outcomeSchema`). + */ +const maxOutcomeChars = 6 * 2 * room + 3 * codeLimits.logLines + 256; + +/** What the isolate sent, or `undefined` for anything it may not send. */ +const parsed = ( + outcome: unknown +): z.infer | undefined => { + if (typeof outcome !== "string" || outcome.length > maxOutcomeChars) { + return undefined; + } + let json: unknown; + try { + json = JSON.parse(outcome); + } catch { + return undefined; + } + const checked = outcomeSchema.safeParse(json); + return checked.success ? checked.data : undefined; +}; + +const notAResult = "The run returned more than it may, or not a result."; /** * Runs `code`, an ES module whose default export is an async function of - * `env`, in a fresh locked isolate with `env` as its only way out. Never - * throws: code that fails to load, throws, runs out of time or is - * cancelled comes back as a failed run. + * `(self, env, ctx)`, in a fresh locked isolate with `env` as its only way + * out. Never throws: every way a run can end is a {@link CodeRunStatus}. + * The wall-clock limit is set before the isolate is asked to load the + * code, and covers loading it, its module's own code and the call. */ export const runCode = async ( loader: WorkerLoader, code: string, env: Readonly>, - signal?: AbortSignal + signal?: AbortSignal, + timeoutMs: number = codeLimits.timeoutMs ): Promise => { - const worker = loader.load({ - ...sandbox, - mainModule: "harness.js", - modules: { "harness.js": harness, "code.js": code }, - env, - limits: { ...sandbox.limits, subRequests: codeLimits.subRequests }, - }); - const limit = deadline(codeLimits.timeoutMs, signal); + const limit = deadline(timeoutMs, signal); + // The phase it is in, for a failure the runtime throws: a module that + // doesn't compile fails as it loads, before `run` is ever called. + let phase: "load" | "run" = "load"; try { - const outcome: unknown = await Promise.race([ - worker.getEntrypoint().run(), - whenAborted(limit.signal), - ]); - const parsed = runSchema.safeParse(outcome); - return parsed.success - ? parsed.data - : failed("The run returned more than it may, or not a result."); + if (code.length > codeLimits.sourceChars) { + return ended( + "refused", + `The module is ${code.length} characters long; a run takes at most ${codeLimits.sourceChars}.` + ); + } + const worker = loader.load({ + ...sandbox, + mainModule: "harness.js", + modules: { "harness.js": harness, "code.js": code }, + env, + limits: { ...sandbox.limits, subRequests: codeLimits.subRequests }, + }); + const entry = worker.getEntrypoint(); + const verified = parsed( + await Promise.race([entry.verify(), whenAborted(limit.signal)]) + ); + // An outcome that isn't one the harness returns came from the code, + // which may change anything in its isolate: it didn't load (verify) + // or threw (run), never a status the host sets. + if (verified === undefined) { + return ended("load_failed", notAResult); + } + if (!verified.ok) { + return { + status: "load_failed", + logs: verified.logs, + error: verified.error ?? exportNeeded, + }; + } + phase = "run"; + const ran = parsed( + await Promise.race([entry.run(), whenAborted(limit.signal)]) + ); + if (ran === undefined) { + return ended("threw", notAResult); + } + const { ok, loaded, ...output } = ran; + if (ok) { + return { status: "completed", ...output }; + } + // Its function wasn't there to call (the isolate started over since + // `verify`): it never ran. + return { status: loaded ? "threw" : "load_failed", ...output }; } catch (error) { if (limit.stopped() === "cancelled") { - return failed("The run was cancelled."); + return ended("cancelled", "The run was cancelled."); } if (limit.stopped() === "timeout") { - return failed( - `The run took longer than ${codeLimits.timeoutMs / 1000} seconds, so it was stopped.` + return ended( + "wall_expired", + `The run took longer than ${timeoutMs / 1000} seconds, so it was stopped.` ); } - // The code didn't load (a syntax error, say), or the runtime stopped - // it (CPU time, memory). Its message is about the code, never core. - return failed( - (error instanceof Error ? error.message : String(error)).slice(0, room) - ); + // A rejection is the runtime's, never the code's (the harness returns + // every outcome): the module didn't compile (a syntax error, say) as + // it loaded, or the runtime stopped it (CPU time, memory, a hang). + // Its message is about the code, never core. + const message = ( + error instanceof Error ? error.message : String(error) + ).slice(0, room); + if (cpuExceeded.test(message)) { + return ended("cpu_exceeded", message); + } + if (message.startsWith(hung)) { + return ended( + "wall_expired", + "The run waits for something that can never come, so it was stopped." + ); + } + return ended(phase === "load" ? "load_failed" : "crashed", message); } finally { limit.clear(); } }; /** - * A run as the model reads it: its logs, then its result or error, at most - * `outputChars` of it. Built only that far, however much the run sent. + * A run as the model reads it: its logs, then its result or, for a run + * that didn't complete, its error under its status, at most `outputChars` + * of it. Built only that far, however much the run sent. */ -export const describeRun = ({ ok, logs, result, error }: CodeRun): string => { +export const describeRun = ({ + status, + logs, + result, + error, +}: CodeRun): string => { const max = codeLimits.outputChars; let text = ""; // One character past `max` at most: enough to know it was cut. @@ -243,11 +500,11 @@ export const describeRun = ({ ok, logs, result, error }: CodeRun): string => { if (logs.length > 0) { part("Logs", logs); } - if (ok && result !== undefined) { + if (status === "completed" && result !== undefined) { part("Returned", [result]); } - if (!ok) { - part("Error", [error ?? "The run failed."]); + if (status !== "completed") { + part(`Error (${status})`, [error ?? "The run failed."]); } if (text === "") { return "(no logs, returned nothing)"; diff --git a/apps/core/src/env.d.ts b/apps/core/src/env.d.ts index c2e59b1b..491d5c55 100644 --- a/apps/core/src/env.d.ts +++ b/apps/core/src/env.d.ts @@ -52,6 +52,8 @@ interface __BaseEnv_Env { * (src/workflows/connector-events.ts). */ CONNECTOR_EVENTS_PER_RUN?: string; + /** Tests only: a shorter wall-clock limit for one code run (src/code-mode.ts). */ + CODE_RUN_TIMEOUT_MS?: string; /** Tests only: a shorter limit for one call into an App (src/app.ts). */ APP_CALL_TIMEOUT_MS?: string; /** Tests only: a shorter bound on one build of a save (src/save-builds.ts). */ diff --git a/apps/core/src/workspace.ts b/apps/core/src/workspace.ts index d23b1cc8..bf6a8cea 100644 --- a/apps/core/src/workspace.ts +++ b/apps/core/src/workspace.ts @@ -62,7 +62,7 @@ import { revokeChatPermissions } from "./chat-connections.ts"; import { chatMessageOf, partialOf } from "./chat-messages.ts"; import { ChatWatch } from "./chat-watch.ts"; import type { ChatListener, ChatState } from "./chat-watch.ts"; -import { codeLimits } from "./code-mode.ts"; +import { codeLimits, codeTimeoutMs } from "./code-mode.ts"; import { migrateOnWake } from "./db/migrate.ts"; import migrations from "./db/workspace/migrations/migrations.js"; import { @@ -775,6 +775,7 @@ export class Workspace extends DurableObject { }, }, loader: this.env.LOADER, + codeTimeoutMs: codeTimeoutMs(this.env), signal: cancel.signal, keep: (message) => { if (message.role === "assistant") { diff --git a/apps/core/test/agent-apps.test.ts b/apps/core/test/agent-apps.test.ts index 3b836a33..2013c925 100644 --- a/apps/core/test/agent-apps.test.ts +++ b/apps/core/test/agent-apps.test.ts @@ -163,7 +163,7 @@ const failuresById = (text: string | undefined) => /** Code that makes each call in turn: what it returned, or its message. */ const tryEach = (calls: Record): string => - `export default async (env) => { + `export default async (self, env) => { const tried = async (call) => { try { return await call(); } catch (error) { return error.message; } }; const results = {}; ${Object.entries(calls) @@ -309,7 +309,7 @@ describe("a chat's Apps and workflows", () => { it("show where a run failed, but of a restricted App's only a platform code", async () => { const { app, chat, grant } = await setUp(({ app: made }) => { - const runsOf = `export default async (env) => (await env.workflows.runs(${JSON.stringify(made)}, "report")).filter(({ failure }) => failure !== null).map(({ id, failure }) => ({ id, failure }));`; + const runsOf = `export default async (self, env) => (await env.workflows.runs(${JSON.stringify(made)}, "report")).filter(({ failure }) => failure !== null).map(({ id, failure }) => ({ id, failure }));`; return [ codeStep(runsOf), says("Before."), diff --git a/apps/core/test/agent-blueprints.test.ts b/apps/core/test/agent-blueprints.test.ts index f410b8f2..5ceb3531 100644 --- a/apps/core/test/agent-blueprints.test.ts +++ b/apps/core/test/agent-blueprints.test.ts @@ -27,7 +27,7 @@ const returned = (text: string | undefined): unknown => /** Code that makes each call in turn: what it returned, or its message. */ const tryEach = (calls: Record): string => - `export default async (env) => { + `export default async (self, env) => { const tried = async (call) => { try { return await call(); } catch (error) { return error.message; } }; const results = {}; ${Object.entries(calls) @@ -102,7 +102,7 @@ describe( () => { it("creates one as its person may, asking for what the blueprint declares, as the agent", async () => { const { person, chat, ledgerBlueprint } = await setUp(() => [ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const listed = await env.build.blueprints(); const { id } = listed.find(({ name }) => name === "Ledger"); const created = await env.build.createFromBlueprint(id, { name: "My ledger" }); diff --git a/apps/core/test/agent-builds.test.ts b/apps/core/test/agent-builds.test.ts index b94c1bd5..cd18c193 100644 --- a/apps/core/test/agent-builds.test.ts +++ b/apps/core/test/agent-builds.test.ts @@ -71,7 +71,7 @@ export default workflowTests(definition, [ const findApp = `const [app] = (await env.apps.list()).filter(({ name }) => name === ${JSON.stringify(appName)});`; /** Creates the App, writes the restyled screen and a workflow, and checks. */ -const firstTry = `export default async (env) => { +const firstTry = `export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": restyled, ...intake })}); const check = await env.build.check(app.id); @@ -85,7 +85,7 @@ const firstTry = `export default async (env) => { };`; /** Writes the fixed screen, checks again, and dry-runs the workflow. */ -const repair = `export default async (env) => { +const repair = `export default async (self, env) => { ${findApp} const written = await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": fixed })}); const check = await env.build.check(app.id); @@ -108,7 +108,7 @@ const returned = (text: string | undefined): unknown => /** Code that makes each call in turn: what it returned, or its message. */ const tryEach = (calls: Record, before = ""): string => - `export default async (env) => { + `export default async (self, env) => { ${before} const tried = async (call) => { try { return await call(); } catch (error) { return error.message; } }; const results = {}; @@ -364,13 +364,13 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("keeps the draft apart from what a builder commits meanwhile, both ways", async () => { const ledger = `const [app] = (await env.apps.list()).filter(({ name }) => name === "Ledger");`; const { builder, existing, chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { ${ledger} await env.build.write(app.id, { "notes.md": "the agent's" }); return await env.build.files(app.id); };`), says("Written."), - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { ${ledger} const draft = await env.build.files(app.id); return { notes: draft.files["notes.md"], builder: draft.files["builder.md"] ?? null }; @@ -424,7 +424,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { }); it("stops the repair loop after five failed checks in a row, until the next question", async () => { - const checks = `export default async (env) => { + const checks = `export default async (self, env) => { ${findApp} const outcomes = []; for (let attempt = 0; attempt < 6; attempt += 1) { @@ -438,13 +438,13 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { return outcomes; };`; const { chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": restyled })}); };`), codeStep(checks), says("It still fails: the button is restyled."), - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { ${findApp} return (await env.build.check(app.id)).failedInARow; };`), @@ -471,7 +471,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("holds checks started at once to the repair loop's limit", async () => { const { chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": restyled })}); const tried = async () => { try { return (await env.build.check(app.id)).passed; } catch (error) { return error.message; } }; @@ -495,7 +495,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("caps the Apps a turn creates, counting none that wasn't created", async () => { const { chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const tried = async (call) => { try { return await call(); } catch (error) { return error.message; } }; // Refused as invalid: it creates nothing, and takes no App's place. const created = [await tried(async () => (await env.build.create({ name: "" })).name)]; @@ -522,7 +522,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("runs a draft's dry runs without a count of their own, and checks it afterwards", async () => { const { chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": fixed, ...intake })}); const runs = []; @@ -548,7 +548,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("keeps no change that leaves a file as the base has it", async () => { const ledger = `const [app] = (await env.apps.list()).filter(({ name }) => name === "Ledger");`; const { existing, chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { ${ledger} const steps = []; steps.push((await env.build.write(app.id, { "notes.md": "new", "gone.md": null })).changed); @@ -736,13 +736,13 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { }); it("fails a check whose build couldn't run, saying so, and passes it once the build can", async () => { - const check = `export default async (env) => { + const check = `export default async (self, env) => { ${findApp} const check = await env.build.check(app.id); return { passed: check.passed, screens: check.screens, failedInARow: check.failedInARow }; };`; const { chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": fixed })}); };`), @@ -801,7 +801,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("fails a check whose build doesn't finish within its bound, and proposes nothing", async () => { const { chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": fixed })}); const check = await env.build.check(app.id); @@ -863,7 +863,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("proposes a passing draft for review, and nothing goes live until a builder makes it current", async () => { const { person, chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": fixed, ...intake })}); const permission = await env.build.requestPermission(app.id, { @@ -985,7 +985,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("stands by a proposal once committed, when what follows it fails", async () => { const { person, chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": fixed, ...intake })}); const proposal = await env.build.propose(app.id, "A desk"); @@ -1043,7 +1043,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("proposes nothing that fails its checks", async () => { const { person, chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const app = await env.build.create({ name: ${JSON.stringify(appName)} }); await env.build.write(app.id, ${JSON.stringify({ "screens/desk.tsx": restyled })}); const proposal = await env.build.propose(app.id, "A desk"); @@ -1127,7 +1127,7 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { it("proposes over what builders committed since, and never undoes it", async () => { const ledger = `const [app] = (await env.apps.list()).filter(({ name }) => name === "Ledger");`; - const propose = `export default async (env) => { + const propose = `export default async (self, env) => { ${ledger} try { return (await env.build.propose(app.id, "Notes")).version; @@ -1136,14 +1136,14 @@ describe("building Apps from a chat", { timeout: 120_000 }, () => { } };`; const { builder, existing, chat, grant } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { ${ledger} await env.build.write(app.id, { "notes.md": "the agent's", "AGENTS.md": "# Ledger, by the agent\\n" }); };`), says("Written."), codeStep(propose), says("Someone changed it meanwhile."), - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { ${ledger} await env.build.write(app.id, { "AGENTS.md": "# Ledger\\n\\nWith notes.\\n" }); };`), diff --git a/apps/core/test/agent-chat.ts b/apps/core/test/agent-chat.ts index 01c0cd43..a2670f5e 100644 --- a/apps/core/test/agent-chat.ts +++ b/apps/core/test/agent-chat.ts @@ -144,3 +144,36 @@ export const codeResults = async (stub: WorkspaceStub, chatId: string) => { : [] ); }; + +/** How each code step of the chat ended, as the host recorded it. */ +export const codeStatuses = async (stub: WorkspaceStub, chatId: string) => { + const messages = await transcript(stub, chatId); + return messages.flatMap((message) => { + if (message.role !== "toolResult") { + return []; + } + const details: unknown = message.details; + return [ + typeof details === "object" && details !== null && "status" in details + ? details.status + : undefined, + ]; + }); +}; + +/** + * Sets `vars` in the object's env, as `pointAtGateway` points its gateway: + * a shorter code run limit (`CODE_RUN_TIMEOUT_MS`), say. + */ +export const setObjectEnv = async ( + stub: WorkspaceStub, + vars: Record +) => { + await runInDurableObject(stub, (instance) => { + const objectEnv: unknown = Reflect.get(instance, "env"); + if (typeof objectEnv !== "object" || objectEnv === null) { + throw new TypeError("The Workspace object has no env"); + } + Object.assign(objectEnv, vars); + }); +}; diff --git a/apps/core/test/agent-connections.test.ts b/apps/core/test/agent-connections.test.ts index 1b0cbc40..6e4d66c4 100644 --- a/apps/core/test/agent-connections.test.ts +++ b/apps/core/test/agent-connections.test.ts @@ -78,7 +78,7 @@ const pendingOf = (id: string | undefined) => /** The model reads how the held call `id` ended, or why it can't. */ const outcomeOf = (id: string) => codeStep( - `export default async (env) => { try { return await env.connections.outcome(${JSON.stringify(id)}); } catch (error) { return error.message; } };` + `export default async (self, env) => { try { return await env.connections.outcome(${JSON.stringify(id)}); } catch (error) { return error.message; } };` ); /** What the chat's code steps returned, as the model read them. */ @@ -103,7 +103,7 @@ describe("a chat's connections", setUpTime, () => { it("hold a write, with no key of the model's, for the person to confirm, and send it once when they do", async () => { const { person, mail, chat, grant } = await setUp( codeStep( - `export default async (env) => await env.connections.call("MAIL", "mail.send", ${JSON.stringify(invoiceMail)});` + `export default async (self, env) => await env.connections.call("MAIL", "mail.send", ${JSON.stringify(invoiceMail)});` ), says("It waits for you to confirm it.") ); @@ -151,7 +151,7 @@ describe("a chat's connections", setUpTime, () => { it("list and call only the connections the agent was granted that its person may use", async () => { const { person, admin, mail, chat, grant } = await setUp( codeStep( - `export default async (env) => { + `export default async (self, env) => { const tried = async (name) => { try { return (await env.connections.call(name, "mail.search", { query: "invoice" })).output; } catch (error) { return error.message; } }; const listed = await env.connections.list(); return { @@ -279,7 +279,7 @@ describe("a chat's connections", setUpTime, () => { it("audit a call refused before connect once, by core, and one connect took by connect alone", async () => { const { mail, chat, grant } = await setUp( codeStep( - `export default async (env) => { + `export default async (self, env) => { const tried = async (action) => { try { await env.connections.call("MAIL", action, { query: "invoice" }); return "ok"; } catch (error) { return error.message; } }; return { search: await tried("mail.search"), @@ -332,7 +332,7 @@ describe("a chat's connections", setUpTime, () => { // keys don't make two actions. const send = (key: string) => codeStep( - `export default async (env) => await env.connections.call("MAIL", "mail.send", ${JSON.stringify(invoiceMail)}, { idempotencyKey: "${key}" });` + `export default async (self, env) => await env.connections.call("MAIL", "mail.send", ${JSON.stringify(invoiceMail)}, { idempotencyKey: "${key}" });` ); const { person, mail, chat, grant } = await setUp( send("first"), @@ -375,7 +375,7 @@ describe("a chat's connections", setUpTime, () => { it("read how a held call ended by its ID: a read held in a restricted chat, once its person confirmed it, and only in the chat that asked", async () => { const { person, mail, chat, grant } = await setUp( codeStep( - `export default async (env) => { + `export default async (self, env) => { const held = await env.connections.call("MAIL", "mail.search", { query: "invoice" }); return { held: held.output, now: await env.connections.outcome(held.pending.id) }; };` @@ -457,7 +457,7 @@ describe("a chat's connections", setUpTime, () => { const chat = await chatOf( person.userId, codeStep( - `export default async (env) => await env.connections.call("INVOICES", "mail.search", { query: "invoice" });` + `export default async (self, env) => await env.connections.call("INVOICES", "mail.search", { query: "invoice" });` ), says("It waits for you.") ); @@ -491,7 +491,7 @@ describe("a chat's connections", setUpTime, () => { it("keep a chat that read an EU-only connection to EU models, in every later turn", async () => { const { mail, chat, grant } = await setUp( codeStep( - `export default async (env) => (await env.connections.call("MAIL", "mail.search", { query: "invoice" })).output;` + `export default async (self, env) => (await env.connections.call("MAIL", "mail.search", { query: "invoice" })).output;` ), says("Never sent.") ); diff --git a/apps/core/test/agent-instructions.test.ts b/apps/core/test/agent-instructions.test.ts index 04d565ea..18684eb2 100644 --- a/apps/core/test/agent-instructions.test.ts +++ b/apps/core/test/agent-instructions.test.ts @@ -105,7 +105,7 @@ describe("a chat's instructions and memory", setUpTime, () => { const { stub, chat, gateway, ask, agent } = await chatOf( person.userId, codeStep( - "export default async (env) => await env.memory.saveUser({ text: 'Prefers short answers.', ifVersion: 0 });" + "export default async (self, env) => await env.memory.saveUser({ text: 'Prefers short answers.', ifVersion: 0 });" ), says("Noted."), says("Kort."), @@ -312,7 +312,7 @@ describe("a chat's instructions and memory", setUpTime, () => { person.userId, says("Hi."), codeStep( - "export default async (env) => { try { await env.memory.saveUser({ text: 'Earns a lot.', ifVersion: 0 }); return 'saved'; } catch (error) { return error.message; } };" + "export default async (self, env) => { try { await env.memory.saveUser({ text: 'Earns a lot.', ifVersion: 0 }); return 'saved'; } catch (error) { return error.message; } };" ), says("Can't.") ); @@ -361,7 +361,7 @@ describe("a chat's instructions and memory", setUpTime, () => { const { ask, agent, stub, chat } = await chatOf( person.userId, codeStep( - `export default async (env) => (await env.knowledge.search(${JSON.stringify(marker)}, { limit: 50 })).hits.length;` + `export default async (self, env) => (await env.knowledge.search(${JSON.stringify(marker)}, { limit: 50 })).hits.length;` ), says("Found them.") ); diff --git a/apps/core/test/agent-knowledge.test.ts b/apps/core/test/agent-knowledge.test.ts index f9074e50..0b696d16 100644 --- a/apps/core/test/agent-knowledge.test.ts +++ b/apps/core/test/agent-knowledge.test.ts @@ -117,7 +117,7 @@ describe("a chat's Knowledge", setUpTime, () => { const { stub, chat, gateway, ask, agent } = await chatOf( person.userId, codeStep( - "export default async (env) => { const { hits } = await env.knowledge.search('parental leave'); const doc = await env.knowledge.read(hits[0].documentId, { section: hits[0].section }); return { title: doc.title, path: doc.path, text: doc.text }; };" + "export default async (self, env) => { const { hits } = await env.knowledge.search('parental leave'); const doc = await env.knowledge.read(hits[0].documentId, { section: hits[0].section }); return { title: doc.title, path: doc.path, text: doc.text }; };" ), says("Sixteen weeks (Handbook, leave.md).") ); @@ -203,7 +203,7 @@ describe("a chat's Knowledge", setUpTime, () => { const { stub, chat, ask, agent } = await chatOf( person.userId, codeStep( - `export default async (env) => { + `export default async (self, env) => { const tried = async (read) => { try { await read(); return "ok"; } catch (error) { return error.message; } }; const catalog = await env.knowledge.catalog(); const found = await env.knowledge.search("note"); @@ -274,7 +274,7 @@ describe("a chat's Knowledge", setUpTime, () => { const { stub, chat, ask, agent } = await chatOf( person.userId, codeStep( - `export default async (env) => (await env.knowledge.read(${JSON.stringify(noteId)})).title;` + `export default async (self, env) => (await env.knowledge.read(${JSON.stringify(noteId)})).title;` ), says("Note."), says("Still here.") @@ -309,7 +309,7 @@ describe("a chat's Knowledge", setUpTime, () => { const { stub, chat, gateway, ask, agent } = await chatOf( person.userId, codeStep( - `export default async (env) => (await env.knowledge.read(${JSON.stringify(payroll.noteId)})).title;` + `export default async (self, env) => (await env.knowledge.read(${JSON.stringify(payroll.noteId)})).title;` ), says("Never sent.") ); @@ -359,7 +359,7 @@ describe("a chat's Knowledge", setUpTime, () => { access: "everyone", }); const read = codeStep( - `export default async (env) => (await env.knowledge.read(${JSON.stringify(noteId)})).title;` + `export default async (self, env) => (await env.knowledge.read(${JSON.stringify(noteId)})).title;` ); const { stub, chat, ask, agent } = await chatOf( person.userId, @@ -450,7 +450,7 @@ describe("a chat's Knowledge", setUpTime, () => { const { stub, chat, ask, agent } = await chatOf( owner.userId, codeStep( - "export default async (env) => { const { signals } = await env.knowledge.signals(); return signals.map(({ kind, collection }) => [kind, collection.id]); };" + "export default async (self, env) => { const { signals } = await env.knowledge.signals(); return signals.map(({ kind, collection }) => [kind, collection.id]); };" ), says("One document is past its review date.") ); diff --git a/apps/core/test/agent.test.ts b/apps/core/test/agent.test.ts index 3b752de7..373839a1 100644 --- a/apps/core/test/agent.test.ts +++ b/apps/core/test/agent.test.ts @@ -20,12 +20,14 @@ import { workspace } from "../src/durable-objects.ts"; import { chatOf, codeResults, + codeStatuses, codeStep, gatewayConfig, model, pausedReply, pointAtGateway, says, + setObjectEnv, transcript, } from "./agent-chat.ts"; import type { WorkspaceStub } from "./agent-chat.ts"; @@ -121,6 +123,28 @@ const runStep = async (code: string) => { return result; }; +/** The runtime's own messages for a run out of CPU time, and a hung one. */ +const cpuMessage = "Worker exceeded CPU time limit."; +const hangMessage = + "The Workers runtime canceled this request because it detected that your Worker's code had hung and would never generate a response."; + +/** + * Code that makes the next value set at index 0 of any array a getter + * throwing `message`: an array the harness filled through [[Set]], then + * sent, would reject with the code's own message. + */ +const arrayTrap = (message: string): string => + `Object.defineProperty(Array.prototype, "0", { configurable: true, set(value) { Object.defineProperty(this, "0", { enumerable: true, configurable: true, get() { throw new Error(${JSON.stringify(message)}); } }); } });`; + +/** Runs `code` as the chat's only code step: what it gave, and how it ended. */ +const runEnded = async (code: string) => { + const { stub, chat, ask } = await newChat(codeStep(code), says("Done.")); + await ask("Run it."); + const [result] = await codeResults(stub, chat.id); + const [status] = await codeStatuses(stub, chat.id); + return { ...result, status }; +}; + /** The agent's model.call events for `userId`, once `count` have arrived. */ const modelCallsBy = async ( userId: string, @@ -188,7 +212,7 @@ describe("chat agent", () => { it("answers a question with a code step against the chat's API", async () => { const { stub, chat, personId, gateway, ask } = await newChat( codeStep( - "export default async (env) => (await env.chat.info()).personId;" + "export default async (self, env) => (await env.chat.info()).personId;" ), says("You are the person this chat belongs to.") ); @@ -401,7 +425,7 @@ describe("chat agent sandbox", () => { it("gets only its APIs in env, and can't import core's env or entrypoints", async () => { const result = await runStep( - "import { env as imported, exports } from 'cloudflare:workers'; export default async (env) => ({ given: Object.keys(env), imported: Object.keys(imported ?? {}), exports: Object.keys(exports ?? {}) });" + "import { env as imported, exports } from 'cloudflare:workers'; export default async (self, env) => ({ given: Object.keys(env), imported: Object.keys(imported ?? {}), exports: Object.keys(exports ?? {}) });" ); expect(result).toStrictEqual({ @@ -412,7 +436,7 @@ describe("chat agent sandbox", () => { it("names the API it doesn't have, and the ones it does", async () => { const result = await runStep( - "export default async (env) => await env.mailbox.send('hi');" + "export default async (self, env) => await env.mailbox.send('hi');" ); expect(result?.isError).toBeTruthy(); @@ -421,12 +445,247 @@ describe("chat agent sandbox", () => { ); }); - it("stops code that waits for something that never comes", async () => { - const result = await runStep( + it("ends code that waits on what can never settle as past its wall-clock time", async () => { + const result = await runEnded( "export default async () => { await new Promise(() => {}); return 'never'; };" ); - expect(result?.isError).toBeTruthy(); + expect(result).toStrictEqual({ + isError: true, + text: "Error (wall_expired):\nThe run waits for something that can never come, so it was stopped.", + status: "wall_expired", + }); + }); + + it("ends code still waiting at its wall-clock limit", async () => { + const { stub, chat, ask } = await newChat( + codeStep( + "export default async () => { await scheduler.wait(60_000); return 'late'; };" + ), + says("Done.") + ); + // Objects share their env: set back however the test ends. + await setObjectEnv(stub, { CODE_RUN_TIMEOUT_MS: "1000" }); + const started = Date.now(); + try { + await ask("Wait."); + } finally { + await setObjectEnv(stub, { CODE_RUN_TIMEOUT_MS: "" }); + } + + await expect(codeResults(stub, chat.id)).resolves.toStrictEqual([ + { + isError: true, + text: "Error (wall_expired):\nThe run took longer than 1 seconds, so it was stopped.", + }, + ]); + await expect(codeStatuses(stub, chat.id)).resolves.toStrictEqual([ + "wall_expired", + ]); + // At its limit, long before its wait would end. + expect(Date.now() - started).toBeLessThan(10_000); + }); + + it("runs a complete module: helpers, null self, its own ctx and parallel calls", async () => { + const result = await runEnded( + "const keys = (value) => Object.keys(value ?? {}); export default async (self, env, ctx) => { const [first, second] = await Promise.all([env.chat.info(), env.chat.info()]); return { self, ctx: keys(ctx), same: first.chatId === second.chatId }; };" + ); + + expect(result).toStrictEqual({ + isError: false, + text: `Returned:\n${JSON.stringify({ self: null, ctx: ["waitUntil"], same: true })}`, + status: "completed", + }); + }); + + it("refuses a module over the source limit before it loads", async () => { + // A module that would log as it loads: refused, it never does. + const code = `console.log("loaded"); export default async () => 1; // ${"x".repeat(codeLimits.sourceChars)}`; + const result = await runEnded(code); + + expect(result).toStrictEqual({ + isError: true, + text: `Error (refused):\nThe module is ${code.length} characters long; a run takes at most ${codeLimits.sourceChars}.`, + status: "refused", + }); + }); + + it("runs a module of exactly the source limit, and refuses one character more", async () => { + const lead = "export default async () => 'ran'; // "; + const exact = `${lead}${"x".repeat(codeLimits.sourceChars - lead.length)}`; + const { stub, chat, ask } = await newChat( + codeStep(exact), + codeStep(`${exact}x`), + says("Done.") + ); + + await ask("Run both."); + + expect(exact).toHaveLength(codeLimits.sourceChars); + await expect(codeStatuses(stub, chat.id)).resolves.toStrictEqual([ + "completed", + "refused", + ]); + }); + + it.each([ + [ + "a thrown Proxy whose traps throw the runtime's message", + "const forged = () => { throw new Error('Worker exceeded CPU time limit.'); }; export default async () => { throw new Proxy({}, { get: forged, getPrototypeOf: forged, has: forged, ownKeys: forged }); };", + "threw", + ], + [ + "an error that says the runtime's message", + "export default async () => { throw new Error('Worker exceeded CPU time limit.'); };", + "threw", + ], + [ + "the harness's built-ins replaced as the module loads", + "const forged = () => { throw new Error('Worker exceeded CPU time limit.'); }; Object.keys = forged; Array.prototype.map = forged; Array.prototype.join = forged; String.prototype.slice = forged; JSON.stringify = forged; Reflect.get = forged; Reflect.has = forged; globalThis.String = forged; export default async (self, env) => { await env.nothing; };", + "threw", + ], + [ + "the harness's methods replaced through an import of it", + "import Harness from 'harness.js'; const forged = async () => { throw new Error('Worker exceeded CPU time limit.'); }; try { Harness.prototype.run = forged; } catch {} try { Object.defineProperty(Harness.prototype, 'run', { value: forged }); } catch {} export default async () => 'mine';", + "completed", + ], + [ + "a setter on Array.prototype[0] set up as the module loads, throwing the CPU message", + `${arrayTrap(cpuMessage)} console.log('x'); export default async () => { console.log('y'); return 'mine'; };`, + "completed", + ], + [ + "a setter on Array.prototype[0] set up as the module loads, throwing the hang message", + `${arrayTrap(hangMessage)} console.log('x'); export default async () => { console.log('y'); return 'mine'; };`, + "completed", + ], + [ + "a setter on Array.prototype[0] set up in the function, throwing the CPU message", + `export default async () => { ${arrayTrap(cpuMessage)} console.log('x'); throw new Error('then this'); };`, + "threw", + ], + [ + "a setter on Array.prototype[0] set up in the function, throwing the hang message", + `export default async () => { ${arrayTrap(hangMessage)} console.log('x'); return 'mine'; };`, + "completed", + ], + [ + "a Proxy thrown as the module loads", + "const forged = () => { throw new Error('Worker exceeded CPU time limit.'); }; throw new Proxy({}, { get: forged, getPrototypeOf: forged }); export default async () => 1;", + "load_failed", + ], + [ + "the runtime's hang message thrown as the module loads", + 'throw new Error("The Workers runtime canceled this request because it detected that your Worker\'s code had hung and would never generate a response."); export default async () => 1;', + "load_failed", + ], + ])("can't forge how its run ended, with %s", async (_, code, status) => { + const result = await runEnded(code); + + expect(result.status).toBe(status); + }); + + it("lets no ctx.waitUntil work act through env after its run returned", async () => { + const { stub, chat, ask } = await newChat( + codeStep( + "export default async (self, env, ctx) => { ctx.waitUntil((async () => { await scheduler.wait(500); await env.chat.attachments(); })()); return 'returned'; };" + ), + says("Done.") + ); + + await ask("Return early."); + // Past when the work would call: refused at the lease, or never let + // reach core at all, and never answered. + await scheduler.wait(2000); + + await expect(codeStatuses(stub, chat.id)).resolves.toStrictEqual([ + "completed", + ]); + const events = await allEvents(); + expect( + events + .filter( + ({ action, detail }) => + action === "agent.call" && + detail.chat === chat.id && + detail.method === "chat.attachments" + ) + .map(({ detail }) => detail.outcome) + .filter((outcome) => outcome !== "refused") + ).toStrictEqual([]); + }); + + it.each([ + [ + "no default export", + "export const run = async () => 1;", + /default export must be an async function/u, + ], + [ + "a default export that isn't a function", + "export default 42;", + /default export must be an async function/u, + ], + [ + "a module that throws as it loads", + "console.log('before'); throw new Error('broken module'); export default async () => 1;", + /^Logs:\nbefore\n\nError \(load_failed\):\nError: broken module/u, + ], + [ + "a syntax error", + "export default async () => {", + /^Error \(load_failed\):\n.*Syntax/isu, + ], + ])( + "reports %s as a module that didn't load, without running it", + async (_, code, text) => { + const result = await runEnded(code); + + expect(result.status).toBe("load_failed"); + expect(result.isError).toBeTruthy(); + expect(result.text).toMatch(text); + } + ); + + it("reports what the function throws as thrown, after what the module logged as it loaded", async () => { + const result = await runEnded( + "console.log('loaded'); export default async () => { throw new Error('nope'); };" + ); + + expect(result.status).toBe("threw"); + expect(result.text).toMatch( + /^Logs:\nloaded\n\nError \(threw\):\nError: nope/u + ); + }); + + it("refuses input with fields besides code, without running it or repeating it back", async () => { + const { stub, chat, ask } = await newChat( + { + text: "", + toolCalls: [ + { + id: `call_${crypto.randomUUID()}`, + name: "executeCode", + arguments: { + code: "export default async () => 'ran';", + timeoutMs: 1, + }, + }, + ], + inputTokens: 200, + outputTokens: 40, + }, + says("Done.") + ); + + await ask("Run it."); + + await expect(codeResults(stub, chat.id)).resolves.toStrictEqual([ + { + isError: true, + text: "executeCode takes one field, `code`: the module as a string. Nothing else is accepted.", + }, + ]); }); it("keeps what the code logs and returns to what the model can read", async () => { @@ -439,6 +698,18 @@ describe("chat agent sandbox", () => { expect(result?.text).toMatch(/cut: longer than 32768 characters\)$/u); }); + it("keeps a run whose output is all characters JSON must escape, within its limits", async () => { + // Each \u0001 is six characters as JSON: the most a character grows. + const result = await runEnded( + "export default async () => { console.log('\\x01'.repeat(65_535)); return '\\x01'.repeat(65_536); };" + ); + + expect(result.status).toBe("completed"); + expect( + (result.text ?? "").startsWith(`Logs:\n${"\u0001".repeat(100)}`) + ).toBeTruthy(); + }); + it("shows a value too large to read as such, without building it all", async () => { const result = await runStep( "export default async () => Array.from({ length: 1_000_000 }, (_, i) => ({ i }));" @@ -456,7 +727,7 @@ describe("chat agent sandbox", () => { // runner: the cancel ends the run, not the wait. const { stub, chat, personId, ask } = await newChat( codeStep( - "export default async (env) => { await scheduler.wait(5_000); await env.chat.info(); await env.chat.info(); };" + "export default async (self, env) => { await scheduler.wait(5_000); await env.chat.info(); await env.chat.info(); };" ) ); @@ -531,7 +802,7 @@ describe("chat agent sandbox", () => { it("stops a run's API calls at the most one run may make, and audits that once", async () => { const { stub, chat, ask } = await newChat( codeStep( - "export default async (env) => { let answered = 0; let refused = 0; let message; for (let i = 0; i < 150; i++) { try { await env.chat.info(); answered += 1; } catch (error) { refused += 1; message = String(error.message); } } return { answered, refused, message }; };" + "export default async (self, env) => { let answered = 0; let refused = 0; let message; for (let i = 0; i < 150; i++) { try { await env.chat.info(); answered += 1; } catch (error) { refused += 1; message = String(error.message); } } return { answered, refused, message }; };" ), says("Done.") ); @@ -602,13 +873,6 @@ describe("chat agent sandbox", () => { expect(row?.longest).toBeLessThan(40_000); } ); - - it("reports code that doesn't load", async () => { - const result = await runStep("export default async () => {"); - - expect(result?.isError).toBeTruthy(); - expect(result?.text).toMatch(/Syntax/iu); - }); }); describe("chat agent turns", () => { @@ -632,7 +896,7 @@ describe("chat agent turns", () => { await expect(reply).resolves.toMatchObject({ outcome: "cancelled" }); await expect(codeResults(stub, chat.id)).resolves.toStrictEqual([ - { isError: true, text: "Error:\nThe run was cancelled." }, + { isError: true, text: "Error (cancelled):\nThe run was cancelled." }, ]); // No request after the cancelled step. expect(gateway.requests).toHaveLength(1); @@ -1113,7 +1377,9 @@ describe("chat agent after a restart", () => { it("continues the conversation with everything before it", async () => { const { stub, chat, ask } = await newChat( - codeStep("export default async (env) => (await env.chat.info()).chatId;"), + codeStep( + "export default async (self, env) => (await env.chat.info()).chatId;" + ), says("This chat's ID is known.") ); await ask("Which chat is this?"); diff --git a/apps/core/test/chat-connections.test.ts b/apps/core/test/chat-connections.test.ts index df34b261..021ba755 100644 --- a/apps/core/test/chat-connections.test.ts +++ b/apps/core/test/chat-connections.test.ts @@ -36,7 +36,7 @@ const setUpTime = { timeout: 90_000 }; /** A chat's code step running `expression`, returning why it was refused. */ const tries = (expression: string) => codeStep( - `export default async (env) => { try { return await ${expression}; } catch (error) { return error.message; } };` + `export default async (self, env) => { try { return await ${expression}; } catch (error) { return error.message; } };` ); /** Asking for connection `connectionId` as `binding`, with `actions`. */ @@ -496,7 +496,7 @@ describe("a connection the chat's agent asks for", setUpTime, () => { const chats = await personalChatsOf( person.userId, codeStep( - `export default async (env) => { + `export default async (self, env) => { const tried = async (ask) => { try { return (await env.connections.request({ reason: "To help.", actions: ["mail.search"], ...ask })).binding; } catch (error) { return error.message; } }; const available = await env.connections.available(); return { @@ -680,11 +680,11 @@ describe("a connection the chat's agent asks for", setUpTime, () => { const chats = await personalChatsOf( person.userId, codeStep( - `export default async (env) => await Promise.all([${["A", "B", "C", "D", "E", "F", "G"].map((binding) => ask(binding, ["mail.search"])).join(", ")}]);` + `export default async (self, env) => await Promise.all([${["A", "B", "C", "D", "E", "F", "G"].map((binding) => ask(binding, ["mail.search"])).join(", ")}]);` ), says("Some wait."), codeStep( - `export default async (env) => await Promise.all([${ask("AGAIN", ["mail.search"])}, ${ask("SEND", ["mail.send"])}]);` + `export default async (self, env) => await Promise.all([${ask("AGAIN", ["mail.search"])}, ${ask("SEND", ["mail.send"])}]);` ), says("Asked.") ); diff --git a/apps/core/test/chats.test.ts b/apps/core/test/chats.test.ts index 1c3850fe..6be9ad5b 100644 --- a/apps/core/test/chats.test.ts +++ b/apps/core/test/chats.test.ts @@ -358,7 +358,7 @@ describe("chats", () => { await answering( ann, codeStep( - 'export default async (env) => await env.connections.call("HELD_MAIL", "mail.send", { to: "ben@acme.test", subject: "Invoice" });' + 'export default async (self, env) => await env.connections.call("HELD_MAIL", "mail.send", { to: "ben@acme.test", subject: "Invoice" });' ), reply ); @@ -402,7 +402,7 @@ describe("chats", () => { await answering( ann, codeStep( - `export default async (env) => { for (const name of ${JSON.stringify(Object.keys(mails))}) { await env.connections.call(name, "mail.send", { to: "ben@acme.test", subject: name }); } };` + `export default async (self, env) => { for (const name of ${JSON.stringify(Object.keys(mails))}) { await env.connections.call(name, "mail.send", { to: "ben@acme.test", subject: name }); } };` ), says("They wait for you.") ); @@ -517,7 +517,7 @@ describe("chats", () => { await answering( ann, codeStep( - 'export default async (env) => await env.connections.call("RACED_MAIL", "mail.send", { to: "ben@acme.test", subject: "Raced" });' + 'export default async (self, env) => await env.connections.call("RACED_MAIL", "mail.send", { to: "ben@acme.test", subject: "Raced" });' ), says("It waits for you.") ); diff --git a/apps/core/test/dependencies.test.ts b/apps/core/test/dependencies.test.ts index 68d5c161..6d3134ad 100644 --- a/apps/core/test/dependencies.test.ts +++ b/apps/core/test/dependencies.test.ts @@ -1470,7 +1470,7 @@ describe("dependency approval", () => { const intent = intentFor(app, { [dates]: "^1.0.0" }); const chat = await chatOf( builder.userId, - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const tried = async (call) => { try { return await call(); } catch (error) { return error.message; } }; const { request } = await env.build.resolveDependencies(${JSON.stringify(app)}, ${JSON.stringify(intent)}); const decision = { approved: true, reviewed: { graphHash: request.graphHash, policyGeneration: request.policyGeneration } }; diff --git a/apps/core/test/packages.test.ts b/apps/core/test/packages.test.ts index 48cbf904..85c3af47 100644 --- a/apps/core/test/packages.test.ts +++ b/apps/core/test/packages.test.ts @@ -890,7 +890,7 @@ describe("a chat's agent", () => { await publish(plain(dates, "1.0.0")); const chat = await chatOf( builder.userId, - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const { request, lock } = await env.build.resolveDependencies(${JSON.stringify(app)}, ${JSON.stringify( intentFor(app, { [dates]: "^1.0.0" }) )}); diff --git a/apps/core/test/preview-repairs.test.ts b/apps/core/test/preview-repairs.test.ts index be3399a1..86d56fe9 100644 --- a/apps/core/test/preview-repairs.test.ts +++ b/apps/core/test/preview-repairs.test.ts @@ -114,14 +114,14 @@ export class App extends DurableObject { /** Finds the Invoice desk, and writes `files` into its draft. */ const write = ( files: Record -) => `export default async (env) => { +) => `export default async (self, env) => { const [app] = (await env.apps.list()).filter(({ name }) => name === "Invoice desk"); await env.build.write(app.id, ${JSON.stringify(files)}); return "written"; };`; /** Checks the draft, with what its preview reported. */ -const check = `export default async (env) => { +const check = `export default async (self, env) => { const [app] = (await env.apps.list()).filter(({ name }) => name === "Invoice desk"); const checked = await env.build.check(app.id); return { @@ -132,7 +132,7 @@ const check = `export default async (env) => { };`; /** Checks the draft, with what its server code wrote in the preview. */ -const checkLogs = `export default async (env) => { +const checkLogs = `export default async (self, env) => { const [app] = (await env.apps.list()).filter(({ name }) => name === "Invoice desk"); const { preview } = await env.build.check(app.id); return { @@ -333,7 +333,7 @@ describe("the repair loop, fed by the preview", { timeout: 180_000 }, () => { it("waits a moment for the preview of a write checked at once, and fails on what it ran into", async () => { const { builder, app, chatId, stub } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const [app] = (await env.apps.list()).filter(({ name }) => name === "Invoice desk"); await env.build.write(app.id, ${JSON.stringify({ "app/server.ts": server(false), @@ -382,7 +382,7 @@ describe("the repair loop, fed by the preview", { timeout: 180_000 }, () => { it("waits for no preview when the draft doesn't build, even with the panel open", async () => { const { builder, app, chatId, stub } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const [app] = (await env.apps.list()).filter(({ name }) => name === "Invoice desk"); await env.build.write(app.id, { "app/server.ts": "export class App {" }); const started = Date.now(); @@ -659,7 +659,7 @@ describe("the repair loop, fed by the preview", { timeout: 180_000 }, () => { it("lets the agent call its draft's server code in the preview, never the App's live data", async () => { const drafted = { "screens/desk.tsx": screen, "app/server.ts": mailing }; const { builder, app, chatId, stub, gateway } = await setUp([ - codeStep(`export default async (env) => { + codeStep(`export default async (self, env) => { const [app] = (await env.apps.list()).filter(({ name }) => name === "Invoice desk"); await env.build.write(app.id, ${JSON.stringify(drafted)}); const first = await env.build.call(app.id, "count"); diff --git a/apps/core/test/run-fixes.test.ts b/apps/core/test/run-fixes.test.ts index cf0aded3..7119e436 100644 --- a/apps/core/test/run-fixes.test.ts +++ b/apps/core/test/run-fixes.test.ts @@ -156,7 +156,9 @@ describe("asking the agent to fix a failed run", slow, () => { const run = await endedRun(owner, app, "careless"); const gateway = await answering( owner, - codeStep("export default async (env) => await env.chat.attachments();"), + codeStep( + "export default async (self, env) => await env.chat.attachments();" + ), says("The check throws for blocked customers."), says("Hello.") );