diff --git a/AGENTS.md b/AGENTS.md index 25cd84f..8575657 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,9 @@ This file applies to the whole repository. ## Use the right guide +- Agent libraries, models, or harness adapters: + [docs/configuration.md#agent-harnesses-and-models](docs/configuration.md#agent-harnesses-and-models) + and [docs/runbooks/extend.md](docs/runbooks/extend.md#add-an-agent-harness) - First deployment: [docs/runbooks/setup.md](docs/runbooks/setup.md) - Repositories, labels, models, permissions, limits, or snapshots: @@ -31,8 +34,10 @@ database. Two dependency rules are mandatory: -1. `app/runner/` imports nothing from the rest of `app/`. It is bundled into - a self-contained Sandbox program. +1. The Sandbox runner bundle (`app/runner/`, `app/harness/`, the agent + definitions in `app/agents/registry.ts`, and `app/schemas/`) imports no + host-only code: GitHub, Sandboxes, Workflows, claims, events, or config. + `tests/agents-catalog.test.ts` enforces it. 2. `app/gateway/` does not import GitHub authentication, Sandboxes, agents, or workflows. The gateway holds no GitHub credential and runs no model. @@ -93,8 +98,13 @@ strict output schemas. credentials or a model key to the gateway. - App permission, event, or label: update `scripts/lib/requirements.ts`, setup behavior, and doctor tests. -- `app/runner/` or `sandbox/`: refresh every repository snapshot after - deployment. +- Agent harness: declare the env var of its key as the adapter's + `credential`; setup and `doctor` derive the key list. It must give the + model only the factory's tools; add it to `HARNESSES` in + `tests/helpers/runner.ts` if it can run the conformance suite. +- `sandbox/`: refresh every repository snapshot after deployment. Changes to + `app/runner/`, `app/harness/`, or agents reach Sandboxes with the Workflows + deploy, because each Sandbox replaces a stale runner bundle when it opens. ## Verify diff --git a/README.md b/README.md index 8bf87f0..7cd4241 100644 --- a/README.md +++ b/README.md @@ -58,15 +58,16 @@ questions, the workflow stops and continues when a person replies. The built-in agents and tasks are examples of the template's extension model: -- Add or replace an agent with a Markdown prompt, a model tier, a bounded Zod - output schema, and the minimum tools it needs. +- Add or replace an agent with a Markdown prompt, a harness (agent library and + model), a bounded Zod output schema, and the minimum tools it needs. +- Run agents on another agent library by adding a harness adapter. - Define a GitHub task for work attached to an issue or pull request. - Add a plain Workflows task for maintenance or other work without a GitHub claim. - Route new GitHub events, labels, or commands through the authenticated gateway. -- Change repositories, verification commands, models, permissions, and limits - in `factory.config.ts`. +- Change repositories, verification commands, permissions, and limits in + `factory.config.ts`, and agent models in `app/agents/models.ts`. The Sandbox runner, claim lifecycle, GitHub client, formatting helpers, and publishing boundary are reusable building blocks. Start with the diff --git a/app/agents/define.ts b/app/agents/define.ts index 64d10c4..aa56637 100644 --- a/app/agents/define.ts +++ b/app/agents/define.ts @@ -1,9 +1,9 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { z } from "zod"; -import type { ModelTier } from "../config/index.js"; -import type { ThinkingLevel, ToolName } from "../runner/protocol.js"; -import { outputSchemaProblem } from "../runner/submit.js"; +import type { Harness } from "../harness/types.js"; +import type { ToolName } from "../runner/core/protocol.js"; +import { outputSchemaProblem } from "../runner/core/submit.js"; export type OutputSchema = z.ZodObject; @@ -11,36 +11,51 @@ export interface AgentSpec extends Omit, "prompt"> { /** Absolute path of the prompt file. */ promptPath: string; - /** The prompt file's contents, verbatim: the agent's system prompt. */ - prompt: string; + /** The prompt file's contents, verbatim: the agent's system prompt. Read on first use, on the host. */ + readonly prompt: string; /** `output` as JSON Schema (input side, so fields with defaults are optional): the runner's submit tool takes it. */ outputJsonSchema: Record; } const AGENTS_DIR = new URL("./", import.meta.url); -/** Reads the prompt file once, as a string (no frontmatter, no parsing), and converts the output schema. */ +/** + * Converts the output schema. Agent modules are also bundled into the sandbox runner, where prompt files don't + * exist, so the prompt is read lazily. + */ export function defineAgent( spec: AgentSpec, ): Agent { if (!Number.isInteger(spec.maxTurns) || spec.maxTurns <= 0) throw new Error(`agent ${spec.id}: maxTurns must be a positive integer`); const promptPath = fileURLToPath(new URL(spec.prompt, AGENTS_DIR)); - const prompt = readFileSync(promptPath, "utf8"); - if (!prompt.trim()) throw new Error(`agent ${spec.id}: prompt file ${promptPath} is empty`); const outputJsonSchema = z.toJSONSchema(spec.output, { io: "input" }) as Record; const problem = outputSchemaProblem(outputJsonSchema); if (problem) throw new Error(`agent ${spec.id}: ${problem}`); - return { ...spec, promptPath, prompt, outputJsonSchema }; + let prompt: string | undefined; + const { prompt: _relative, ...rest } = spec; + return { + ...rest, + promptPath, + outputJsonSchema, + get prompt() { + if (prompt === undefined) { + const text = readFileSync(promptPath, "utf8"); + if (!text.trim()) throw new Error(`agent ${spec.id}: prompt file ${promptPath} is empty`); + prompt = text; + } + return prompt; + }, + }; } diff --git a/app/agents/implementer/index.ts b/app/agents/implementer/index.ts index d72df1f..fa7ca38 100644 --- a/app/agents/implementer/index.ts +++ b/app/agents/implementer/index.ts @@ -1,5 +1,6 @@ import { Implementation } from "../../schemas/index.js"; import { defineAgent } from "../define.js"; +import { piOpus } from "../models.js"; /** * Implements the approved spec in the sandbox's working tree and runs the verify commands itself. A repair run @@ -8,7 +9,7 @@ import { defineAgent } from "../define.js"; export const implementer = defineAgent({ id: "implementer", prompt: "implementer/prompt.md", - model: "large", + harness: piOpus, tools: ["read", "ls", "bash", "edit", "write"], output: Implementation, maxTurns: 150, diff --git a/app/agents/index.ts b/app/agents/index.ts index 1175ef9..a32f4af 100644 --- a/app/agents/index.ts +++ b/app/agents/index.ts @@ -1,6 +1,8 @@ export { type Agent, type AgentSpec, defineAgent, type OutputSchema } from "./define.js"; export { implementer } from "./implementer/index.js"; +export { piOpus, piSonnet } from "./models.js"; export { prReviewer } from "./pr-reviewer/index.js"; +export { AGENTS } from "./registry.js"; export { AgentError, type AgentErrorCode, type AgentRun, type RunAgentOptions, runAgent } from "./run.js"; export { specWriter } from "./spec-writer/index.js"; export { triage } from "./triage/index.js"; diff --git a/app/agents/models.ts b/app/agents/models.ts new file mode 100644 index 0000000..7926b88 --- /dev/null +++ b/app/agents/models.ts @@ -0,0 +1,10 @@ +import { getBuiltinModel } from "@earendil-works/pi-ai/providers/all"; +import { pi } from "../harness/index.js"; + +/** + * The harnesses the built-in agents use. Change an agent's library or model here, or give an agent its own + * harness in its definition. Each value carries its library's own configuration, so a typo in a model ID is a + * type error. + */ +export const piSonnet = pi({ model: getBuiltinModel("anthropic", "claude-sonnet-5") }); +export const piOpus = pi({ model: getBuiltinModel("anthropic", "claude-opus-5") }); diff --git a/app/agents/pr-reviewer/index.ts b/app/agents/pr-reviewer/index.ts index e342c15..6f176b5 100644 --- a/app/agents/pr-reviewer/index.ts +++ b/app/agents/pr-reviewer/index.ts @@ -1,11 +1,12 @@ import { Review } from "../../schemas/index.js"; import { defineAgent } from "../define.js"; +import { piSonnet } from "../models.js"; /** Reviews a pull request across correctness, security, performance, and tests. Read-only, in the PR Reviewer's sandbox. */ export const prReviewer = defineAgent({ id: "pr-reviewer", prompt: "pr-reviewer/prompt.md", - model: "medium", + harness: piSonnet, tools: ["read", "ls", "bash"], output: Review, maxTurns: 100, diff --git a/app/agents/registry.ts b/app/agents/registry.ts new file mode 100644 index 0000000..675b092 --- /dev/null +++ b/app/agents/registry.ts @@ -0,0 +1,11 @@ +import type { Agent } from "./define.js"; +import { implementer } from "./implementer/index.js"; +import { prReviewer } from "./pr-reviewer/index.js"; +import { specWriter } from "./spec-writer/index.js"; +import { triage } from "./triage/index.js"; + +/** + * Every agent the factory runs. The sandbox runner is bundled from this list and runs an agent only if it is here. + * Bundled modules must not import host-only code (GitHub, Sandbox, Workflows, claims, events, config). + */ +export const AGENTS: readonly Agent[] = [triage, specWriter, implementer, prReviewer]; diff --git a/app/agents/run.ts b/app/agents/run.ts index 801f7af..c36a3c2 100644 --- a/app/agents/run.ts +++ b/app/agents/run.ts @@ -1,10 +1,8 @@ import { randomUUID } from "node:crypto"; import type { z } from "zod"; -import { modelFor } from "../config/index.js"; import { emit } from "../events/index.js"; -import { providerApiKey } from "../runner/keys.js"; -import { type RunnerErrorCode, RunnerEvent, type RunnerInput, type ToolName, type Usage } from "../runner/protocol.js"; -import { errorMessage } from "../runner/session.js"; +import { errorMessage } from "../runner/core/emit.js"; +import { PROTOCOL_VERSION, type RunnerErrorCode, RunnerEvent, type RunnerInput, type ToolName, type Usage } from "../runner/core/protocol.js"; import { APP_DIR, REPO_DIR, SCRATCH_DIR, type Sandbox } from "../sandbox/types.js"; import type { Agent, OutputSchema } from "./define.js"; @@ -55,7 +53,7 @@ export async function runAgent> { const { sandbox, ...base } = opts; - const { provider, model } = modelFor(agent.model); + const { harness } = agent; const timeoutMs = base.timeoutMs ?? DEFAULT_TIMEOUT_MS; const finish = (outcome: string, summary: Summary, message?: string) => emit({ @@ -66,7 +64,9 @@ export async function runAgent new AgentError(agent.id, code, text); try { const runnerInput: RunnerInput = { + protocol_version: PROTOCOL_VERSION, trace_id: base.traceId, agent_id: agent.id, system_prompt: agent.prompt, message, - model_provider: provider, - model, - thinking_level: agent.thinking ?? "medium", tools: [...agent.tools], output_schema: agent.outputJsonSchema, max_turns: agent.maxTurns, @@ -94,8 +92,8 @@ export async function runAgent; + /** From the runner's `started` event: what actually ran, which the host can't always know. */ + harnessVersion?: string; + model?: string; /** Lines that aren't valid RunnerEvents. */ invalidLines: number; } @@ -197,7 +198,10 @@ function summarize(jsonl: string): Summary { continue; } if (summary.terminal) continue; - if (event.type === "usage") { + if (event.type === "started") { + summary.harnessVersion = event.harness_version; + summary.model = event.model; + } else if (event.type === "usage") { const u = summary.usage; u.input_tokens += event.usage.input_tokens; u.output_tokens += event.usage.output_tokens; diff --git a/app/agents/spec-writer/index.ts b/app/agents/spec-writer/index.ts index b86ce81..d13202e 100644 --- a/app/agents/spec-writer/index.ts +++ b/app/agents/spec-writer/index.ts @@ -1,11 +1,12 @@ import { SpecDraft } from "../../schemas/index.js"; import { defineAgent } from "../define.js"; +import { piOpus } from "../models.js"; /** Researches the repository (read-only) and drafts or revises the spec. Runs in triage's sandbox after a "draft" decision. */ export const specWriter = defineAgent({ id: "spec-writer", prompt: "spec-writer/prompt.md", - model: "large", + harness: piOpus, tools: ["read", "ls", "bash"], output: SpecDraft, maxTurns: 60, diff --git a/app/agents/triage/index.ts b/app/agents/triage/index.ts index cea6321..302cefd 100644 --- a/app/agents/triage/index.ts +++ b/app/agents/triage/index.ts @@ -1,11 +1,12 @@ import { TriageDecision } from "../../schemas/index.js"; import { defineAgent } from "../define.js"; +import { piSonnet } from "../models.js"; /** Reads the repository (read-only) and decides whether the issue is ready for a spec. Runs in a sandbox at the base branch. */ export const triage = defineAgent({ id: "triage", prompt: "triage/prompt.md", - model: "medium", + harness: piSonnet, tools: ["read", "ls", "bash"], output: TriageDecision, maxTurns: 40, diff --git a/app/config/index.ts b/app/config/index.ts index 7bef46c..0d6a3b4 100644 --- a/app/config/index.ts +++ b/app/config/index.ts @@ -1,6 +1,6 @@ -import config, { type ModelTier, type RepoConfig } from "../../factory.config.js"; +import config, { type RepoConfig } from "../../factory.config.js"; -export { config, type ModelTier, type RepoConfig }; +export { config, type RepoConfig }; /** One repository a run acts on, resolved from the webhook that started it. */ export interface RepoContext { @@ -29,13 +29,6 @@ export function requireRepo(fullName: string, installationId: number): RepoConte return repo; } -export function modelFor(tier: ModelTier): { provider: string; model: string } { - const id = config.models[tier]; - const slash = id.indexOf("/"); - if (slash <= 0) throw new Error(`models.${tier} must be "provider/model", got "${id}"`); - return { provider: id.slice(0, slash), model: id.slice(slash + 1) }; -} - export function requireEnv(name: string): string { const value = process.env[name]?.trim(); if (!value) throw new Error(`${name} is not set`); diff --git a/app/harness/index.ts b/app/harness/index.ts new file mode 100644 index 0000000..a608afc --- /dev/null +++ b/app/harness/index.ts @@ -0,0 +1,5 @@ +export type { Driver, DriverContext, DriverSession, Report, SubmitTool } from "../runner/core/driver.js"; +export { InvalidInput } from "../runner/core/driver.js"; +export type { HarnessRun, HarnessRuntime } from "../runner/core/harness.js"; +export { type PiOptions, pi, type ThinkingLevel } from "./pi.js"; +export type { Harness } from "./types.js"; diff --git a/app/harness/pi.ts b/app/harness/pi.ts new file mode 100644 index 0000000..8e65c00 --- /dev/null +++ b/app/harness/pi.ts @@ -0,0 +1,42 @@ +import type { Api, Model, MutableModels } from "@earendil-works/pi-ai"; +import { piDriver, type ThinkingLevel } from "../runner/pi/driver.js"; +import type { Harness } from "./types.js"; + +export type { ThinkingLevel }; + +/** Each provider's key env var, for the providers the factory supports through Pi. */ +export const PI_PROVIDER_KEY_ENV: Readonly> = { + anthropic: "ANTHROPIC_API_KEY", + google: "GEMINI_API_KEY", + openai: "OPENAI_API_KEY", +}; + +export interface PiOptions { + /** A model from Pi's catalog, e.g. `getBuiltinModel("anthropic", "claude-opus-5")`. */ + model: Model; + /** Default "medium". */ + thinkingLevel?: ThinkingLevel; + /** + * The Pi model registry the model is resolved and streamed through: a custom or self-hosted provider registered + * on it. Default: Pi's built-in catalog. + */ + models?: () => MutableModels; + /** The env var holding the key. Default: the provider's, from PI_PROVIDER_KEY_ENV; required for other providers. */ + credential?: string; +} + +/** Runs an agent on Pi (`@earendil-works/pi-agent-core`) with the factory's tools. */ +export function pi(opts: PiOptions): Harness { + const { provider, id } = opts.model; + const credential = opts.credential ?? PI_PROVIDER_KEY_ENV[provider]; + if (!credential) { + throw new Error(`pi: no default key for provider "${provider}"; pass credential, or use one of ${Object.keys(PI_PROVIDER_KEY_ENV).join(", ")}`); + } + const thinkingLevel = opts.thinkingLevel ?? "medium"; + return { + name: "pi", + label: `${provider}/${id}`, + credential, + driver: piDriver({ model: opts.model, thinkingLevel, models: opts.models }), + }; +} diff --git a/app/harness/types.ts b/app/harness/types.ts new file mode 100644 index 0000000..bf77df5 --- /dev/null +++ b/app/harness/types.ts @@ -0,0 +1,15 @@ +import type { HarnessRuntime } from "../runner/core/harness.js"; + +/** + * What an adapter returns: the runtime the runner calls in the sandbox, plus what the host needs to know before a + * run. Build one with an adapter from this directory and give it to an agent definition. + */ +export interface Harness extends HarnessRuntime { + /** Names the library and model in setup, doctor, and logs before a run. The runner reports the exact model in `started`. */ + readonly label: string; + /** + * The env var on the Workflows service that holds this harness's key. The host sends its value into the sandbox + * for each run; setup and doctor manage it. + */ + readonly credential: string; +} diff --git a/app/runner/bundle.ts b/app/runner/bundle.ts index f1971ae..27efb00 100644 --- a/app/runner/bundle.ts +++ b/app/runner/bundle.ts @@ -1,19 +1,18 @@ +import { relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; const RUNNER_DIR = fileURLToPath(new URL(".", import.meta.url)); +const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url)); -/** - * Bundles app/runner/main.ts and every package it uses into one self-contained ESM file, so the sandbox - * needs only Node to run it: no npm, no node_modules. - */ -export async function bundleRunner(): Promise { +async function buildBundle(): Promise<{ text: string; inputs: string[] }> { const result = await build({ absWorkingDir: RUNNER_DIR, entryPoints: ["main.ts"], outfile: "runner.mjs", write: false, bundle: true, + metafile: true, platform: "node", format: "esm", target: "node22", @@ -23,5 +22,18 @@ export async function bundleRunner(): Promise { }); const [file] = result.outputFiles; if (!file) throw new Error("esbuild produced no output for the runner bundle"); - return file.text; + return { text: file.text, inputs: Object.keys(result.metafile.inputs) }; +} + +/** + * Bundles app/runner/main.ts, the agent registry with every harness it uses, and every package they need into one + * self-contained ESM file, so the sandbox needs only Node to run it: no npm, no node_modules. + */ +export async function bundleRunner(): Promise { + return (await buildBundle()).text; +} + +/** The source files in the bundle, relative to the repository root (tests check the sandbox's import boundary). */ +export async function bundleInputs(): Promise { + return (await buildBundle()).inputs.map((input) => relative(REPO_ROOT, resolve(RUNNER_DIR, input))); } diff --git a/app/runner/core/driver.ts b/app/runner/core/driver.ts new file mode 100644 index 0000000..e0831c9 --- /dev/null +++ b/app/runner/core/driver.ts @@ -0,0 +1,68 @@ +import type { FactoryTool } from "../tools/index.js"; +import type { Usage } from "./protocol.js"; + +/** + * Puts an agent library behind the kernel (`executeHarness`). A driver only translates: the kernel owns the + * run's lifecycle, its limits, the tools, the submission, and every protocol event. + */ +export interface Driver { + /** + * Prepares the library for this run and resolves what will run, before `started`. Throw InvalidInput for a + * configuration the library can't run. + */ + open(ctx: DriverContext): DriverSession | Promise; +} + +export interface DriverSession { + /** The library's version, reported in `started`. */ + readonly version: string; + /** The exact model, reported in `started`. */ + readonly model: string; + /** + * Converses with the model. Resolves when the model stops on its own and rejects with the reason when the + * library fails. Once the kernel has ended the run, how this settles no longer matters. + */ + run(): Promise; +} + +export interface DriverContext { + readonly systemPrompt: string; + readonly message: string; + readonly apiKey: string; + /** The library's working directory: the checkout, or `scratchDir` for an agent without tools. */ + readonly cwd: string; + /** The only writable location besides `cwd`. */ + readonly scratchDir: string; + /** Every tool the model may call besides `submit`. */ + readonly tools: readonly FactoryTool[]; + readonly submit: SubmitTool; + /** Aborted when the run ends: stop the model and any running tool. */ + readonly signal: AbortSignal; + /** Call immediately before each model request. False means the run is over: don't send it. */ + requestModel(): boolean; + readonly report: Report; +} + +/** The model's final answer. Every call returns the text for the model; the kernel records the outcome. */ +export interface SubmitTool { + readonly name: string; + readonly description: string; + /** JSON Schema of the arguments: the agent's output schema. */ + readonly parameters: Record; + /** Checks the model's arguments as it sent them, before any library validation or coercion. Never throws. */ + call(args: unknown): { text: string; isError: boolean }; +} + +/** What the library observed, for the event stream. */ +export interface Report { + text(text: string): void; + usage(usage: Usage): void; + toolCall(id: string, tool: string, args: unknown): void; + /** Ignored unless `toolCall` announced `id`; the kernel truncates `text`. */ + toolResult(id: string, tool: string, text: string, isError: boolean): void; +} + +/** A configuration the driver can't run, e.g. an unknown model: the run fails with `invalid_input`. */ +export class InvalidInput extends Error { + override readonly name = "InvalidInput"; +} diff --git a/app/runner/core/emit.ts b/app/runner/core/emit.ts new file mode 100644 index 0000000..f0e3ba1 --- /dev/null +++ b/app/runner/core/emit.ts @@ -0,0 +1,46 @@ +import type { RunnerEvent } from "./protocol.js"; + +type WithoutAt = E extends unknown ? Omit : never; +export type EventBody = WithoutAt; + +export interface Emitter { + (event: EventBody): void; + readonly terminated: boolean; + /** Replaces `secret` with `[REDACTED]` in every later line. */ + redact(secret: string): void; + /** Resolves once every line written so far has been flushed. */ + flush(): Promise; +} + +/** Writes one RunnerEvent per line. Drops everything after the first terminal (`result`/`error`) event. */ +export function createEmitter(write: (chunk: string) => Promise): Emitter { + const secrets: string[] = []; + let pending = Promise.resolve(); + let terminated = false; + const emit = (event: EventBody) => { + if (terminated) return; + terminated = event.type === "result" || event.type === "error"; + const { type, ...fields } = event; + let line = JSON.stringify({ type, at: new Date().toISOString(), ...fields }); + for (const secret of secrets) line = line.replaceAll(secret, "[REDACTED]"); + pending = pending.then(() => write(`${line}\n`)).catch(() => {}); + }; + // defineProperty, not Object.assign: assign would copy the getter's value once instead of the getter. + Object.defineProperty(emit, "terminated", { get: () => terminated }); + return Object.assign(emit as typeof emit & { readonly terminated: boolean }, { + redact(secret: string) { + if (secret) secrets.push(secret); + }, + flush: () => pending, + }); +} + +const PREVIEW_CHARS = 2000; +const TRUNCATED = "…[truncated]"; + +/** A tool result's text as it appears in `tool_result.output_preview`. */ +export function previewText(text: string): string { + return text.length > PREVIEW_CHARS ? text.slice(0, PREVIEW_CHARS - TRUNCATED.length) + TRUNCATED : text; +} + +export const errorMessage = (err: unknown): string => (err instanceof Error ? err.message : String(err)); diff --git a/app/runner/core/harness.ts b/app/runner/core/harness.ts new file mode 100644 index 0000000..f0cc1d4 --- /dev/null +++ b/app/runner/core/harness.ts @@ -0,0 +1,22 @@ +import type { Driver } from "./driver.js"; +import type { Emitter } from "./emit.js"; +import type { RunnerInput } from "./protocol.js"; + +/** One agent run, as the runner hands it to the kernel. */ +export interface HarnessRun { + /** Already validated with parseInput. */ + input: RunnerInput; + emit: Emitter; + /** The key delivered on stdin. */ + apiKey: string; + /** Epoch ms the `timeout_ms` budget is measured from. */ + startedAt: number; + /** Aborting stops the run with `agent_error`. */ + signal?: AbortSignal; +} + +/** What the runner runs inside the sandbox: the kernel (`executeHarness`) drives the agent library through `driver`. */ +export interface HarnessRuntime { + readonly name: string; + readonly driver: Driver; +} diff --git a/app/runner/core/input.ts b/app/runner/core/input.ts new file mode 100644 index 0000000..bbb256e --- /dev/null +++ b/app/runner/core/input.ts @@ -0,0 +1,32 @@ +import { statSync } from "node:fs"; +import { PROTOCOL_VERSION, RunnerInput } from "./protocol.js"; +import { outputSchemaProblem } from "./submit.js"; + +/** Validates a RunnerInput, including what Zod can't see: the protocol version, the output schema's root, and the directories. */ +export function parseInput(json: unknown): RunnerInput | string { + const version = (json as { protocol_version?: unknown } | null)?.protocol_version; + if (version !== PROTOCOL_VERSION) { + return `this runner speaks protocol ${PROTOCOL_VERSION}, but the input is protocol ${JSON.stringify(version ?? null)}: the host and the runner bundle are from different versions`; + } + const parsed = RunnerInput.safeParse(json); + if (!parsed.success) { + return `invalid RunnerInput: ${parsed.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ")}`; + } + const input = parsed.data; + const schemaProblem = outputSchemaProblem(input.output_schema); + if (schemaProblem) return schemaProblem; + if (input.tools.length > 0) { + if (!input.cwd) return "cwd is required when tools is non-empty"; + if (!input.scratch_dir) return "scratch_dir is required when tools is non-empty"; + if (!isDirectory(input.cwd)) return `cwd ${input.cwd} is not a directory`; + } + return input; +} + +function isDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} diff --git a/app/runner/core/kernel.ts b/app/runner/core/kernel.ts new file mode 100644 index 0000000..1e1051a --- /dev/null +++ b/app/runner/core/kernel.ts @@ -0,0 +1,159 @@ +import { tmpdir } from "node:os"; +import { setTimeout as sleep } from "node:timers/promises"; +import { createFactoryTools, type FactoryTool } from "../tools/index.js"; +import type { DriverContext, DriverSession, SubmitTool } from "./driver.js"; +import { InvalidInput } from "./driver.js"; +import { errorMessage, previewText } from "./emit.js"; +import type { HarnessRun, HarnessRuntime } from "./harness.js"; +import { RUNNER_EXIT, type RunnerErrorCode } from "./protocol.js"; +import { SUBMIT_ACCEPTED, SUBMIT_DUPLICATE, SUBMIT_TOOL, Submission, submitDescription, submitParameters } from "./submit.js"; + +/** Time kept back from `timeout_ms` so the runner reports its own timeout before the host kills it. */ +const SAFETY_MARGIN_MS = 10_000; +/** How long a driver may take to stop after the run ends before the outcome is reported anyway. */ +const STOP_GRACE_MS = 5_000; +const RUN_OVER = "The run is over."; + +type Outcome = { type: "result"; output: unknown } | { type: "error"; code: RunnerErrorCode; message: string }; + +/** + * Runs one agent on `harness.driver` and returns the exit code. The kernel emits `started`, then exactly one + * `result` or `error`: the first outcome reached (an accepted submission, a rejected last submission, the turn + * limit, the timeout, or cancellation) is final, and the driver is stopped once it is. + */ +export async function executeHarness(run: HarnessRun, harness: HarnessRuntime): Promise { + const { input, emit } = run; + const stop = new AbortController(); + let outcome: Outcome | undefined; + /** Records the outcome if it's the first; the driver may finish its current step (e.g. the submit result). */ + const end = (next: Outcome) => { + outcome ??= next; + }; + /** Records the outcome if it's the first, and stops the driver either way. */ + const halt = (next: Outcome) => { + end(next); + stop.abort(); + }; + + let requests = 0; + const requestModel = () => { + if (outcome) return false; + if (requests >= input.max_turns) { + halt({ type: "error", code: "max_turns", message: `turn limit (${input.max_turns}) reached without a submission` }); + return false; + } + requests++; + return true; + }; + + const submission = new Submission(input.output_schema); + const submit: SubmitTool = { + name: SUBMIT_TOOL, + description: submitDescription(input.output_schema), + parameters: submitParameters(input.output_schema), + call(args) { + if (submission.accepted) return { text: SUBMIT_DUPLICATE, isError: false }; + if (outcome) return { text: RUN_OVER, isError: true }; + const verdict = submission.check(args); + if (verdict.ok) { + submission.accept(verdict.checked); + end({ type: "result", output: verdict.checked }); + return { text: SUBMIT_ACCEPTED, isError: false }; + } + if (verdict.exhausted) halt({ type: "error", code: "invalid_submission", message: verdict.exhausted }); + return { text: verdict.message, isError: true }; + }, + }; + + const tools: FactoryTool[] = ( + input.tools.length > 0 && input.cwd && input.scratch_dir + ? createFactoryTools({ tools: input.tools, cwd: input.cwd, scratchDir: input.scratch_dir }) + : [] + ).map((tool) => ({ + ...tool, + async execute(args, signal) { + if (outcome) return { text: RUN_OVER, isError: true }; + return tool.execute(args, signal ? AbortSignal.any([signal, stop.signal]) : stop.signal); + }, + })); + + const announced = new Set(); + const scratchDir = input.scratch_dir ?? tmpdir(); + const ctx: DriverContext = { + systemPrompt: input.system_prompt, + message: input.message, + apiKey: run.apiKey, + cwd: input.cwd ?? scratchDir, + scratchDir, + tools, + submit, + signal: stop.signal, + requestModel, + report: { + text: (text) => emit({ type: "assistant_text", text }), + usage: (usage) => emit({ type: "usage", usage }), + toolCall(id, tool, args) { + announced.add(id); + emit({ type: "tool_call", tool_call_id: id, tool, args }); + }, + toolResult(id, tool, text, isError) { + if (!announced.delete(id)) return; + emit({ type: "tool_result", tool_call_id: id, tool, is_error: isError, output_preview: previewText(text) }); + }, + }, + }; + + const budgetMs = input.timeout_ms - Math.min(SAFETY_MARGIN_MS, Math.floor(input.timeout_ms / 10)); + const deadline = setTimeout( + () => halt({ type: "error", code: "timeout", message: `the run exceeded its ${budgetMs} ms budget` }), + Math.max(0, run.startedAt + budgetMs - Date.now()), + ); + const onCancel = () => halt({ type: "error", code: "agent_error", message: "the run was cancelled" }); + if (run.signal?.aborted) onCancel(); + run.signal?.addEventListener("abort", onCancel, { once: true }); + const grace = new AbortController(); + const stopped = new Promise((resolve) => + stop.signal.addEventListener("abort", () => void sleep(STOP_GRACE_MS, undefined, { signal: grace.signal }).then(resolve, resolve), { + once: true, + }), + ); + + let failure: { error: unknown } | undefined; + /** Runs a driver step until it settles, or until the run has ended and the driver had its grace to stop. */ + const step = async (work: () => T | Promise): Promise => { + const settled = Promise.resolve() + .then(work) + .catch((error: unknown) => { + failure ??= { error }; + return undefined; + }); + return Promise.race([settled, stopped.then(() => undefined)]); + }; + try { + const session: DriverSession | undefined = outcome ? undefined : await step(() => harness.driver.open(ctx)); + if (session && !outcome) { + emit({ type: "started", agent_id: input.agent_id, harness: harness.name, harness_version: session.version, model: session.model }); + await step(() => session.run()); + } else if (!outcome && failure) { + const { error } = failure; + end({ type: "error", code: error instanceof InvalidInput ? "invalid_input" : "agent_error", message: errorMessage(error) }); + } + } finally { + grace.abort(); + clearTimeout(deadline); + run.signal?.removeEventListener("abort", onCancel); + stop.abort(); + } + + const final: Outcome = outcome ?? + (failure + ? { type: "error", code: "agent_error", message: errorMessage(failure.error) } + : { type: "error", code: "no_submission", message: `the model stopped without calling ${SUBMIT_TOOL}` }); + if (final.type === "result") { + emit({ type: "result", output: final.output, turns: requests }); + return RUNNER_EXIT.ok; + } + emit({ type: "error", code: final.code, message: final.message }); + if (final.code === "timeout") return RUNNER_EXIT.timeout; + return final.code === "invalid_input" ? RUNNER_EXIT.invalid_input : RUNNER_EXIT.error; +} diff --git a/app/runner/core/keys.ts b/app/runner/core/keys.ts new file mode 100644 index 0000000..bc9f5b2 --- /dev/null +++ b/app/runner/core/keys.ts @@ -0,0 +1,25 @@ +/** + * Key env vars of every provider a built-in harness can call. Every one of them, plus each bundled harness's own + * key var, is removed before tools run: ANTHROPIC_OAUTH_TOKEN, for one, would take precedence inside Pi. + */ +export const KNOWN_KEY_ENV: readonly string[] = [ + "ANTHROPIC_API_KEY", + "ANTHROPIC_OAUTH_TOKEN", + "ANTHROPIC_AUTH_TOKEN", + "GEMINI_API_KEY", + "OPENAI_API_KEY", +]; + +/** + * Removes every key var in KNOWN_KEY_ENV and `extra` from `env`, so none reach the agent's tool subprocesses, and + * returns their values for redaction. The runner's only key is the one on stdin. + */ +export function scrubKeys(env: NodeJS.ProcessEnv, extra: readonly string[] = []): string[] { + const removed: string[] = []; + for (const name of new Set([...KNOWN_KEY_ENV, ...extra])) { + const value = env[name]; + if (value) removed.push(value); + delete env[name]; + } + return removed; +} diff --git a/app/runner/protocol.ts b/app/runner/core/protocol.ts similarity index 82% rename from app/runner/protocol.ts rename to app/runner/core/protocol.ts index f7339a7..fd6e401 100644 --- a/app/runner/protocol.ts +++ b/app/runner/core/protocol.ts @@ -1,27 +1,26 @@ /** * The contract between the host (`runAgent`) and the agent runner, which runs in a sandbox bundled into - * runner.mjs: input on stdin, events on stdout. + * runner.mjs: input on stdin, events on stdout. Every harness speaks it. */ import { z } from "zod"; -const TOOL_NAMES = ["read", "ls", "bash", "edit", "write"] as const; +/** Bumped on any incompatible change; a runner rejects input for another version. */ +export const PROTOCOL_VERSION = 2; + +export const TOOL_NAMES = ["read", "ls", "bash", "edit", "write"] as const; export const ToolName = z.enum(TOOL_NAMES); export type ToolName = z.infer; -const ThinkingLevel = z.enum(["off", "minimal", "low", "medium", "high"]); -export type ThinkingLevel = z.infer; - export const RunnerInput = z.object({ + protocol_version: z.literal(PROTOCOL_VERSION), /** Correlates host-side events; the sandbox runner validates but does not emit it. */ trace_id: z.string().min(1), + /** Selects the agent, and so its harness, from the runner bundle. */ agent_id: z.string().min(1), /** The agent's prompt file, verbatim. */ system_prompt: z.string().min(1), /** The task for this run. */ message: z.string().min(1), - model_provider: z.string().min(1), - model: z.string().min(1), - thinking_level: ThinkingLevel.default("medium"), tools: z.array(ToolName), /** JSON Schema (an object schema) of the agent's output; the runner's `submit` tool takes exactly this. */ output_schema: z.record(z.string(), z.unknown()), @@ -48,7 +47,14 @@ export type RunnerErrorCode = (typeof RUNNER_ERROR_CODES)[number]; /** One JSON object per line on the runner's stdout. Exactly one `result` or `error` ends the stream. */ export const RunnerEvent = z.discriminatedUnion("type", [ - z.object({ type: z.literal("started"), at: z.iso.datetime(), agent_id: z.string(), model: z.string(), pi_version: z.string() }), + z.object({ + type: z.literal("started"), + at: z.iso.datetime(), + agent_id: z.string(), + harness: z.string(), + harness_version: z.string(), + model: z.string(), + }), z.object({ type: z.literal("tool_call"), at: z.iso.datetime(), tool_call_id: z.string(), tool: z.string(), args: z.unknown() }), z.object({ type: z.literal("tool_result"), diff --git a/app/runner/core/submit.ts b/app/runner/core/submit.ts new file mode 100644 index 0000000..a054e4a --- /dev/null +++ b/app/runner/core/submit.ts @@ -0,0 +1,95 @@ +import type { TSchema } from "typebox"; +import { Value } from "typebox/value"; + +export const SUBMIT_TOOL = "submit"; +export const MAX_SUBMIT_ATTEMPTS = 3; + +type JsonOutputSchema = Record; + +/** + * Why `schema` can't be a submit tool's parameters, or undefined if it can. Pi's Anthropic adapter sends only + * the top-level `properties` and `required`, so the root must be a plain object schema, not a union or a ref. + */ +export function outputSchemaProblem(schema: JsonOutputSchema): string | undefined { + if (schema.type !== "object") return 'output_schema must have type "object" at its root'; + const props = schema.properties; + if (typeof props !== "object" || props === null || Array.isArray(props)) { + return "output_schema must have a properties object at its root"; + } + try { + Value.Check(schema as TSchema, {}); + } catch (err) { + return `output_schema is not a usable JSON Schema: ${err instanceof Error ? err.message : String(err)}`; + } + return undefined; +} + +/** The submit tool's parameters as the model sees them: the output schema without JSON Schema metadata. */ +export function submitParameters(schema: JsonOutputSchema): JsonOutputSchema { + const { $schema: _schema, $id: _id, ...rest } = schema; + return rest; +} + +export function submitDescription(schema: JsonOutputSchema): string { + const what = typeof schema.description === "string" && schema.description ? ` ${schema.description}` : ""; + return ( + `Submit your final output.${what} Call it exactly once, as your final action, on its own. The arguments are ` + + "checked against the schema; a rejected submission records nothing, so fix every listed problem and call it " + + `again (${MAX_SUBMIT_ATTEMPTS} attempts in total).` + ); +} + +export const SUBMIT_ACCEPTED = "Submission recorded. You are done."; +export const SUBMIT_DUPLICATE = "Already submitted; this call was ignored."; + +export type SubmitCheck = + | { ok: true; checked: unknown } + /** `exhausted` is set on the last allowed attempt: the run must stop with `invalid_submission` and this message. */ + | { ok: false; message: string; exhausted?: string }; + +/** + * One run's submission: checks each attempt against the output JSON Schema with TypeBox, which validates plain + * JSON Schema (including what `z.toJSONSchema` emits), and counts rejections. A usability check so the model can + * fix its submission: the host revalidates the accepted output with the agent's Zod schema, and that check is + * authoritative. Every harness's submit tool goes through this. + */ +export class Submission { + #accepted: { output: unknown } | undefined; + #rejected = 0; + + constructor(private readonly schema: JsonOutputSchema) {} + + get accepted(): { output: unknown } | undefined { + return this.#accepted; + } + + /** Checks `args` without recording them. The checked copy is what `accept` should record. */ + check(args: unknown): SubmitCheck { + if (Value.Check(this.schema as TSchema, args)) return { ok: true, checked: structuredClone(args) }; + const found = [...Value.Errors(this.schema as TSchema, args)].map((e) => `${e.instancePath || "(root)"}: ${e.message}`); + const problems = found.length > 0 ? [...new Set(found)] : ["(root): does not match the output schema"]; + this.#rejected++; + const remaining = MAX_SUBMIT_ATTEMPTS - this.#rejected; + const message = + `${SUBMIT_TOOL} rejected (attempt ${this.#rejected} of ${MAX_SUBMIT_ATTEMPTS}); nothing was recorded. ` + + (remaining > 0 ? "Fix every problem below and call it again:\n" : "No attempts remain:\n") + + problems.map((p) => `- ${p}`).join("\n"); + if (remaining > 0) return { ok: false, message }; + // Schema paths, rule names, and field names only: TypeBox's errors don't quote the submitted values. + const fields = args && typeof args === "object" && !Array.isArray(args) ? Object.keys(args) : []; + const names = + fields + .slice(0, 20) + .map((f) => f.slice(0, 40)) + .join(", ") || "none"; + const exhausted = `${SUBMIT_TOOL} was rejected ${this.#rejected} times; last: ${problems.slice(0, 5).join("; ")}; submitted fields: ${names}`; + return { ok: false, message, exhausted }; + } + + /** Records the first accepted output; false when one was already recorded. */ + accept(output: unknown): boolean { + if (this.#accepted) return false; + this.#accepted = { output }; + return true; + } +} diff --git a/app/runner/keys.ts b/app/runner/keys.ts deleted file mode 100644 index 56450f5..0000000 --- a/app/runner/keys.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** Each supported provider's key env vars, first one preferred. Every one of them is removed before tools run. */ -export const PROVIDER_KEY_ENV: Readonly> = { - anthropic: ["ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_AUTH_TOKEN"], - google: ["GEMINI_API_KEY"], - openai: ["OPENAI_API_KEY"], -}; - -const ALL_PROVIDER_KEY_ENV: readonly string[] = [...new Set(Object.values(PROVIDER_KEY_ENV).flat())]; - -/** The selected provider's own (first) key var, never another provider's. The host reads the key it sends with this. */ -export function providerApiKey(provider: string, env: Readonly>): string | undefined { - const name = PROVIDER_KEY_ENV[provider]?.[0]; - return (name && env[name]?.trim()) || undefined; -} - -/** - * Removes every provider's key vars from `env`, so none reach the agent's tool subprocesses, and returns their - * values for redaction. Every provider's vars go, not just the selected one: ANTHROPIC_OAUTH_TOKEN, for one, would - * take precedence inside Pi. The runner's only key is the one on stdin. - */ -export function scrubProviderKeys(env: NodeJS.ProcessEnv): string[] { - const removed: string[] = []; - for (const name of ALL_PROVIDER_KEY_ENV) { - const value = env[name]; - if (value) removed.push(value); - delete env[name]; - } - return removed; -} diff --git a/app/runner/main.ts b/app/runner/main.ts index b837f22..c091231 100644 --- a/app/runner/main.ts +++ b/app/runner/main.ts @@ -1,17 +1,20 @@ import { mkdirSync, realpathSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; -import { PROVIDER_KEY_ENV, scrubProviderKeys } from "./keys.js"; -import { RUNNER_EXIT } from "./protocol.js"; -import { createEmitter, createModels, errorMessage, PI_VERSION, parseInput, runSession } from "./session.js"; -import { defaultTestStub, type TestStub } from "./test-stub.js"; +import type { Agent } from "../agents/define.js"; +import { AGENTS } from "../agents/registry.js"; +import { createEmitter, errorMessage } from "./core/emit.js"; +import { parseInput } from "./core/input.js"; +import { scrubKeys } from "./core/keys.js"; +import { executeHarness } from "./core/kernel.js"; +import { PROTOCOL_VERSION, RUNNER_EXIT } from "./core/protocol.js"; interface MainIO { stdin: AsyncIterable; /** Resolves once `chunk` is flushed. */ write(chunk: string): Promise; - /** Replaces the default `--test-stub` responses (tests). */ - testStub?: TestStub; + /** Replaces the bundled agents (tests). */ + agents?: readonly Agent[]; } const processIO = (): MainIO => ({ @@ -26,11 +29,14 @@ function useScratch(scratchDir: string): void { } /** - * `runner.mjs [--key-stdin] [--test-stub] [--self-test]`: RunnerInput on stdin, RunnerEvent JSONL on stdout. With - * `--key-stdin`, the first line of stdin is the model key and the rest is the RunnerInput. Returns the exit code. + * `runner.mjs --key-stdin`: the model key on the first line of stdin, then a RunnerInput; RunnerEvent JSONL on + * stdout. The input's `agent_id` selects the agent, and so its harness, from the bundle. Returns the exit code. + * + * `runner.mjs --self-test` checks that the bundle and every agent definition in it load, and prints one JSON line. */ export async function main(argv: string[], io: MainIO = processIO()): Promise { const startedAt = Date.now(); + const agents = io.agents ?? AGENTS; const emit = createEmitter(io.write); const fail = async (code: "invalid_input" | "agent_error", message: string) => { emit({ type: "error", code, message }); @@ -38,21 +44,24 @@ export async function main(argv: string[], io: MainIO = processIO()): Promise a.harness.credential), + )) + emit.redact(secret); // stdout carries only RunnerEvent JSONL; route any library console output to stderr. console.log = console.info = console.debug = console.error; @@ -67,7 +76,7 @@ export async function main(argv: string[], io: MainIO = processIO()): Promise a.id === input.agent_id); + if (!agent) return fail("invalid_input", `unknown agent "${input.agent_id}": the runner bundle doesn't define it`); + if (!apiKey) return fail("invalid_input", "no model key: pass it as the first line of stdin with --key-stdin"); try { if (input.scratch_dir) useScratch(input.scratch_dir); - return await runSession({ - input, - emit, - apiKey, - testStub: args["test-stub"] ? (io.testStub ?? defaultTestStub) : undefined, - startedAt, - }); + return await executeHarness({ input, emit, apiKey, startedAt }, agent.harness); } catch (err) { console.error(err); if (!emit.terminated) return await fail("agent_error", errorMessage(err)); @@ -96,16 +102,16 @@ export async function main(argv: string[], io: MainIO = processIO()): Promise { - const models = createModels(); - const errors: string[] = []; - for (const p of Object.keys(PROVIDER_KEY_ENV)) { - if (!models.getModels(p).length) errors.push(`no models for provider "${p}"`); - } - const ok = errors.length === 0; - const report = { ok, pi_version: PI_VERSION, node: process.version, ...(ok ? {} : { errors }) }; +/** Reaching this proves the bundle and every agent definition in it load in this sandbox. Prints one JSON line. */ +async function selfTest(io: MainIO, agents: readonly Agent[]): Promise { + const report = { + ok: true, + protocol_version: PROTOCOL_VERSION, + node: process.version, + agents: agents.map((a) => ({ id: a.id, harness: a.harness.name, label: a.harness.label })), + }; await io.write(`${JSON.stringify(report)}\n`); - return ok ? RUNNER_EXIT.ok : RUNNER_EXIT.error; + return RUNNER_EXIT.ok; } async function readAll(stream: AsyncIterable): Promise { diff --git a/app/runner/pi/driver.ts b/app/runner/pi/driver.ts new file mode 100644 index 0000000..2294d47 --- /dev/null +++ b/app/runner/pi/driver.ts @@ -0,0 +1,186 @@ +import { setTimeout as sleep } from "node:timers/promises"; +import { type AfterToolCallContext, Agent, type AgentEvent } from "@earendil-works/pi-agent-core"; +import agentCorePackage from "@earendil-works/pi-agent-core/package.json" with { type: "json" }; +import { + type Api, + type AssistantMessage, + type AssistantMessageEventStream, + calculateCost, + createAssistantMessageEventStream, + InMemoryCredentialStore, + isRetryableAssistantError, + type Model, + type MutableModels, + type Usage as PiUsage, +} from "@earendil-works/pi-ai"; +import { builtinModels } from "@earendil-works/pi-ai/providers/all"; +import { type Driver, type DriverContext, InvalidInput, type Report } from "../core/driver.js"; +import type { Usage } from "../core/protocol.js"; +import { contentText } from "../tools/index.js"; +import { piSubmitTool, piTool } from "./tools.js"; + +export const PI_VERSION: string = agentCorePackage.version; + +export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high"; + +export interface PiDriverOptions { + model: Model; + thinkingLevel: ThinkingLevel; + /** Default: the built-in catalog (createModels). */ + models?: () => MutableModels; +} + +const MAX_OUTPUT_TOKENS = 16_384; +/** Passed to the agent so the max_tokens cap below stays in step with Pi's thinking budget. */ +const THINKING_BUDGETS = { minimal: 1024, low: 2048, medium: 8192, high: 16_384 } as const; + +/** Retries of a model request that failed transiently (overloaded, rate limited, dropped stream). */ +const MAX_MODEL_RETRIES = 2; +const RETRY_BASE_DELAY_MS = 2_000; + +/** The built-in provider catalog with in-memory credentials: the key is passed per request, nothing is stored. */ +export function createModels(): MutableModels { + return builtinModels({ credentials: new InMemoryCredentialStore() }); +} + +/** Runs agents on Pi (`@earendil-works/pi-agent-core`). */ +export function piDriver(opts: PiDriverOptions): Driver { + return { + open(ctx) { + const models = (opts.models ?? createModels)(); + const { provider, id } = opts.model; + const requested = models.getModel(provider, id); + if (!requested) throw new InvalidInput(`unknown model "${provider}/${id}" (not in the Pi ${PI_VERSION} registry)`); + const model = capMaxTokens(requested, opts.thinkingLevel); + return { + version: PI_VERSION, + model: `${model.provider}/${model.id}`, + run: () => converse(ctx, models, model, opts.thinkingLevel), + }; + }, + }; +} + +async function converse(ctx: DriverContext, models: MutableModels, model: Model, thinkingLevel: ThinkingLevel): Promise { + const agent = new Agent({ + initialState: { + systemPrompt: ctx.systemPrompt, + model, + thinkingLevel, + tools: [...ctx.tools.map(piTool), piSubmitTool(ctx.submit)], + }, + // Every provider request, retries included, passes the kernel's turn limit first. + streamFn: (m, context, options) => (ctx.requestModel() ? models.streamSimple(m, context, options) : refused(m)), + getApiKey: () => ctx.apiKey, + thinkingBudgets: THINKING_BUDGETS, + afterToolCall: async (call) => (endsBatch(call, ctx.submit.name) ? { terminate: true } : undefined), + }); + const unsubscribe = agent.subscribe((event) => report(event, ctx.report, model)); + const abort = () => agent.abort(); + ctx.signal.addEventListener("abort", abort, { once: true }); + try { + if (ctx.signal.aborted) return; + await agent.prompt(ctx.message); + // Restarts the last request while it fails with a transient model error. + for (let attempt = 1; attempt <= MAX_MODEL_RETRIES && !ctx.signal.aborted; attempt++) { + const last = lastAssistant(agent); + if (last?.stopReason !== "error" || !isRetryableAssistantError(last)) break; + agent.state.messages = agent.state.messages.slice(0, -1); + await sleep(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), undefined, { signal: ctx.signal }).catch(() => {}); + if (ctx.signal.aborted) return; + await agent.continue(); + } + const last = lastAssistant(agent); + if (last?.stopReason === "error" || last?.stopReason === "aborted") { + throw new Error(last.errorMessage ?? agent.state.errorMessage ?? `model request ${last.stopReason}`); + } + } finally { + unsubscribe(); + ctx.signal.removeEventListener("abort", abort); + } +} + +/** Applies the Anthropic thinking token budget only for anthropic-messages models; others use effort levels. */ +function capMaxTokens(model: Model, level: ThinkingLevel): Model { + if (model.api !== "anthropic-messages") return model; + const thinking = level === "off" ? 0 : THINKING_BUDGETS[level]; + return { ...model, maxTokens: Math.min(model.maxTokens, MAX_OUTPUT_TOKENS + thinking) }; +} + +/** A request the kernel refused, ended the way Pi ends an aborted one. */ +function refused(model: Model): AssistantMessageEventStream { + const message: AssistantMessage = { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "aborted", + errorMessage: "the run is over", + timestamp: Date.now(), + }; + const stream = createAssistantMessageEventStream(); + stream.push({ type: "error", reason: "aborted", error: message }); + stream.end(message); + return stream; +} + +function report(event: AgentEvent, out: Report, model: Model) { + switch (event.type) { + case "tool_execution_start": + out.toolCall(event.toolCallId, event.toolName, event.args); + break; + case "tool_execution_end": + out.toolResult(event.toolCallId, event.toolName, contentText((event.result as { content?: unknown } | undefined)?.content), event.isError); + break; + case "message_end": { + const message = event.message; + if (message.role !== "assistant") break; + const text = message.content + .flatMap((c) => (c.type === "text" ? [c.text] : [])) + .join("\n") + .trim(); + if (text) out.text(text); + const usage = toUsage(message.usage, model); + if (usage) out.usage(usage); + break; + } + } +} + +/** + * Undefined for all-zero usage (e.g. a request aborted before it was sent). When the provider reports no cost + * (the test stub never does), the cost is computed from the model's prices. + */ +function toUsage(u: PiUsage, model: Model): Usage | undefined { + const count = (n: number) => (Number.isFinite(n) && n > 0 ? Math.round(n) : 0); + const usage: Usage = { + input_tokens: count(u.input), + output_tokens: count(u.output), + cache_read_tokens: count(u.cacheRead), + cache_write_tokens: count(u.cacheWrite), + }; + if (usage.input_tokens + usage.output_tokens + usage.cache_read_tokens + usage.cache_write_tokens === 0) return undefined; + let cost = u.cost?.total; + if (!cost) cost = calculateCost(model, { ...u, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }).total; + return typeof cost === "number" && Number.isFinite(cost) && cost >= 0 ? { ...usage, cost_usd: cost } : usage; +} + +/** + * The loop ends only when every tool result in a batch sets `terminate`, so a submission batched with other + * calls (e.g. `ls` + submit) would cost another LLM request. The submit tool terminates on acceptance; + * this marks its siblings too. A rejected submission carries no `terminate`, so the model still gets to retry. + */ +function endsBatch({ toolCall, assistantMessage }: AfterToolCallContext, submit: string): boolean { + return toolCall.name !== submit && assistantMessage.content.some((c) => c.type === "toolCall" && c.name === submit); +} + +function lastAssistant(agent: Agent): AssistantMessage | undefined { + const messages = agent.state.messages; + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i]; + if (m?.role === "assistant") return m; + } + return undefined; +} diff --git a/app/runner/pi/tools.ts b/app/runner/pi/tools.ts new file mode 100644 index 0000000..9a5d359 --- /dev/null +++ b/app/runner/pi/tools.ts @@ -0,0 +1,45 @@ +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import type { TSchema } from "typebox"; +import type { SubmitTool } from "../core/driver.js"; +import type { FactoryTool } from "../tools/index.js"; + +/** A factory tool as a Pi tool. Pi validates the arguments against the same schema before `execute`. */ +export function piTool(tool: FactoryTool): AgentTool { + return { + name: tool.name, + label: tool.name, + description: tool.description, + parameters: tool.parameters as TSchema, + async execute(_toolCallId, params, signal) { + const out = await tool.execute(params, signal); + // Pi marks a tool call failed only when execute throws. + if (out.isError) throw new Error(out.text); + return { content: [{ type: "text", text: out.text }], details: undefined }; + }, + }; +} + +/** + * The submit tool as a Pi tool. Submissions are checked in `prepareArguments`, which runs before Pi's own argument + * validation: Pi would otherwise reject a call without it counting as an attempt, and coerce values the host's Zod + * schema would refuse. Sequential, so each `execute` follows its own `prepareArguments`. + */ +export function piSubmitTool(submit: SubmitTool): AgentTool { + let text = ""; + return { + name: submit.name, + label: "Submit", + description: submit.description, + parameters: submit.parameters as TSchema, + executionMode: "sequential", + prepareArguments(args) { + const out = submit.call(args); + if (out.isError) throw new Error(out.text); + text = out.text; + return args as never; + }, + async execute() { + return { content: [{ type: "text", text }], details: undefined, terminate: true }; + }, + }; +} diff --git a/app/runner/session.ts b/app/runner/session.ts deleted file mode 100644 index c1a0228..0000000 --- a/app/runner/session.ts +++ /dev/null @@ -1,362 +0,0 @@ -import { statSync } from "node:fs"; -import { setTimeout as sleep } from "node:timers/promises"; -import { type AfterToolCallContext, Agent, type AgentEvent } from "@earendil-works/pi-agent-core"; -import agentCorePackage from "@earendil-works/pi-agent-core/package.json" with { type: "json" }; -import { - type Api, - type AssistantMessage, - calculateCost, - InMemoryCredentialStore, - isRetryableAssistantError, - type Model, - type MutableModels, - type Usage as PiUsage, -} from "@earendil-works/pi-ai"; -import { builtinModels } from "@earendil-works/pi-ai/providers/all"; -import { RUNNER_EXIT, type RunnerErrorCode, type RunnerEvent, RunnerInput, type ThinkingLevel, type Usage } from "./protocol.js"; -import { createSubmitTool, MAX_SUBMIT_ATTEMPTS, outputSchemaProblem, SUBMIT_TOOL } from "./submit.js"; -import { registerTestStubModel, type TestStub } from "./test-stub.js"; -import { createTools } from "./tools.js"; - -/** Reported in `started` and `--self-test`. */ -export const PI_VERSION: string = agentCorePackage.version; - -const MAX_OUTPUT_TOKENS = 16_384; -/** Passed to the agent so the max_tokens cap below stays in step with Pi's thinking budget. */ -const THINKING_BUDGETS = { minimal: 1024, low: 2048, medium: 8192, high: 16_384 } as const; - -/** Time kept back from `timeout_ms` so the runner reports its own timeout before the host kills it. */ -const SAFETY_MARGIN_MS = 10_000; -/** How long an abort may take to settle before the runner reports the stop anyway. */ -const ABORT_GRACE_MS = 5_000; -/** Retries of a model request that failed transiently (overloaded, rate limited, dropped stream). */ -const MAX_MODEL_RETRIES = 2; -const RETRY_BASE_DELAY_MS = 2_000; - -const SUBMIT_REMINDER = `You stopped without calling ${SUBMIT_TOOL}. Call it now, exactly once, with your final output built from what you have found so far.`; - -export const errorMessage = (err: unknown): string => (err instanceof Error ? err.message : String(err)); - -// ── events ────────────────────────────────────────────────────────────── - -type WithoutAt = E extends unknown ? Omit : never; -type EventBody = WithoutAt; - -interface Emitter { - (event: EventBody): void; - readonly terminated: boolean; - /** Replaces `secret` with `[REDACTED]` in every later line. */ - redact(secret: string): void; - /** Resolves once every line written so far has been flushed. */ - flush(): Promise; -} - -/** Writes one RunnerEvent per line. Drops everything after the first terminal (`result`/`error`) event. */ -export function createEmitter(write: (chunk: string) => Promise): Emitter { - const secrets: string[] = []; - let pending = Promise.resolve(); - let terminated = false; - const emit = (event: EventBody) => { - if (terminated) return; - terminated = event.type === "result" || event.type === "error"; - const { type, ...fields } = event; - let line = JSON.stringify({ type, at: new Date().toISOString(), ...fields }); - for (const secret of secrets) line = line.replaceAll(secret, "[REDACTED]"); - pending = pending.then(() => write(`${line}\n`)).catch(() => {}); - }; - return Object.assign(emit, { - get terminated() { - return terminated; - }, - redact(secret: string) { - if (secret) secrets.push(secret); - }, - flush: () => pending, - }); -} - -const PREVIEW_CHARS = 2000; -const TRUNCATED = "…[truncated]"; - -function outputPreview(result: unknown): string { - const content = (result as { content?: unknown } | undefined)?.content; - if (!Array.isArray(content)) return ""; - const text = content.map((c: { type?: string; text?: string }) => (c?.type === "text" ? (c.text ?? "") : `[${c?.type ?? "unknown"}]`)).join("\n"); - return text.length > PREVIEW_CHARS ? text.slice(0, PREVIEW_CHARS - TRUNCATED.length) + TRUNCATED : text; -} - -/** - * Undefined for all-zero usage (e.g. a request aborted before it was sent). When the provider reports no cost - * (the test stub never does), the cost is computed from the model's prices. - */ -function toUsage(u: PiUsage, model: Model): Usage | undefined { - const count = (n: number) => (Number.isFinite(n) && n > 0 ? Math.round(n) : 0); - const usage: Usage = { - input_tokens: count(u.input), - output_tokens: count(u.output), - cache_read_tokens: count(u.cacheRead), - cache_write_tokens: count(u.cacheWrite), - }; - if (usage.input_tokens + usage.output_tokens + usage.cache_read_tokens + usage.cache_write_tokens === 0) return undefined; - let cost = u.cost?.total; - if (!cost) cost = calculateCost(model, { ...u, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }).total; - return typeof cost === "number" && Number.isFinite(cost) && cost >= 0 ? { ...usage, cost_usd: cost } : usage; -} - -// ── models ────────────────────────────────────────────────────────────── - -/** The built-in provider catalog with in-memory credentials: the key is passed per request, nothing is stored. */ -export function createModels(): MutableModels { - return builtinModels({ credentials: new InMemoryCredentialStore() }); -} - -/** Applies the Anthropic thinking token budget only for anthropic-messages models; others use effort levels. */ -function capMaxTokens(model: Model, level: ThinkingLevel): Model { - if (model.api !== "anthropic-messages") return model; - const thinking = level === "off" ? 0 : THINKING_BUDGETS[level]; - return { ...model, maxTokens: Math.min(model.maxTokens, MAX_OUTPUT_TOKENS + thinking) }; -} - -// ── input ─────────────────────────────────────────────────────────────── - -/** Validates a RunnerInput, including what Zod can't see: the output schema's root and the directories. */ -export function parseInput(json: unknown): RunnerInput | string { - const parsed = RunnerInput.safeParse(json); - if (!parsed.success) { - return `invalid RunnerInput: ${parsed.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ")}`; - } - const input = parsed.data; - const schemaProblem = outputSchemaProblem(input.output_schema); - if (schemaProblem) return schemaProblem; - if (input.tools.length > 0) { - if (!input.cwd) return "cwd is required when tools is non-empty"; - if (!input.scratch_dir) return "scratch_dir is required when tools is non-empty"; - if (!isDirectory(input.cwd)) return `cwd ${input.cwd} is not a directory`; - } - return input; -} - -function isDirectory(path: string): boolean { - try { - return statSync(path).isDirectory(); - } catch { - return false; - } -} - -// ── session ───────────────────────────────────────────────────────────── - -type Failure = { code: RunnerErrorCode; message: string }; - -interface SessionOptions { - /** Already validated with parseInput. */ - input: RunnerInput; - emit: Emitter; - /** Passed with every request; unused with `testStub`. */ - apiKey?: string; - /** Replaces the model with deterministic test responses (tests, snapshot self-checks). */ - testStub?: TestStub; - /** Epoch ms the `timeout_ms` budget is measured from (default: now). */ - startedAt?: number; - /** Aborting stops the run with `agent_error`. */ - signal?: AbortSignal; -} - -/** Runs one agent and emits `started` … then exactly one `result` or `error`. Returns the exit code. */ -export async function runSession(opts: SessionOptions): Promise { - const { input, emit } = opts; - const startedAt = opts.startedAt ?? Date.now(); - const fail = (code: RunnerErrorCode, message: string) => { - emit({ type: "error", code, message }); - return code === "invalid_input" ? RUNNER_EXIT.invalid_input : RUNNER_EXIT.error; - }; - - const models = createModels(); - const requested = models.getModel(input.model_provider, input.model); - if (!requested) { - return fail("invalid_input", `unknown model "${input.model_provider}/${input.model}" (not in the Pi ${PI_VERSION} registry)`); - } - if (!opts.testStub && !opts.apiKey) { - return fail("agent_error", `no API key for provider "${input.model_provider}": pass it with --key-stdin`); - } - const model = capMaxTokens(opts.testStub ? registerTestStubModel(models, opts.testStub(input), requested) : requested, input.thinking_level); - - let submission: { output: unknown } | undefined; - let failure: Failure | undefined; - const stopped = new AbortController(); - const tools = - input.tools.length > 0 && input.cwd && input.scratch_dir - ? createTools({ tools: input.tools, cwd: input.cwd, scratchDir: input.scratch_dir }) - : []; - - const agent = new Agent({ - initialState: { - systemPrompt: input.system_prompt, - model, - thinkingLevel: input.thinking_level, - tools: [ - ...tools, - createSubmitTool(input.output_schema, { - onAccepted: (output) => { - submission = { output }; - }, - onRejected: (attempts, problems, fields) => { - if (attempts < MAX_SUBMIT_ATTEMPTS) return; - // Schema paths, rule names, and field names only: TypeBox's errors don't quote the submitted values. - const last = problems.slice(0, 5).join("; "); - const names = - fields - .slice(0, 20) - .map((f) => f.slice(0, 40)) - .join(", ") || "none"; - stop({ code: "invalid_submission", message: `${SUBMIT_TOOL} was rejected ${attempts} times; last: ${last}; submitted fields: ${names}` }); - }, - }), - ], - }, - streamFn: (m, context, options) => models.streamSimple(m, context, options), - getApiKey: () => opts.apiKey, - thinkingBudgets: THINKING_BUDGETS, - afterToolCall: async (ctx) => (endsBatch(ctx) || failure !== undefined ? { terminate: true } : undefined), - }); - - emit({ type: "started", agent_id: input.agent_id, model: `${model.provider}/${model.id}`, pi_version: PI_VERSION }); - - function stop(reason?: Failure) { - if (reason) failure ??= reason; - stopped.abort(); - if (agent.state.isStreaming) agent.abort(); - } - /** Sends `text`, then restarts the last turn while it fails with a transient model error. */ - const prompt = async (text: string) => { - await agent.prompt(text); - for (let attempt = 1; attempt <= MAX_MODEL_RETRIES && !stopped.signal.aborted; attempt++) { - const last = lastAssistant(agent); - if (last?.stopReason !== "error" || !isRetryableAssistantError(last)) return; - agent.state.messages = agent.state.messages.slice(0, -1); - await sleep(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), undefined, { signal: stopped.signal }).catch(() => {}); - if (stopped.signal.aborted) return; - await agent.continue(); - } - }; - let turns = 0; - const unsubscribe = agent.subscribe((event) => { - forward(event, emit, model); - if (event.type !== "turn_end") return; - turns++; - // Backstop for batches that still continue after a submission (e.g. a sibling call failed validation). - if (submission || failure) stop(); - else if (turns >= input.max_turns) stop({ code: "max_turns", message: `turn limit (${input.max_turns}) reached without a submission` }); - }); - - const onAbort = () => stop({ code: "agent_error", message: "the run was cancelled" }); - if (opts.signal?.aborted) onAbort(); - opts.signal?.addEventListener("abort", onAbort, { once: true }); - - const budgetMs = input.timeout_ms - Math.min(SAFETY_MARGIN_MS, Math.floor(input.timeout_ms / 10)); - let deadline: NodeJS.Timeout | undefined; - let grace: NodeJS.Timeout | undefined; - const settled = new Promise((resolve) => { - const giveUp = () => { - grace ??= setTimeout(resolve, ABORT_GRACE_MS); - }; - stopped.signal.addEventListener("abort", giveUp, { once: true }); - deadline = setTimeout( - () => stop({ code: "timeout", message: `the run exceeded its ${budgetMs} ms budget` }), - Math.max(0, startedAt + budgetMs - Date.now()), - ); - }); - - const conversation = (async () => { - if (stopped.signal.aborted) return; - await prompt(input.message); - if (!submission && !failure && stoppedOnItsOwn(agent)) await prompt(SUBMIT_REMINDER); - })(); - let thrown: unknown; - await Promise.race([ - conversation.catch((err) => { - thrown = err; - }), - settled, - ]); - clearTimeout(deadline); - clearTimeout(grace); - unsubscribe(); - opts.signal?.removeEventListener("abort", onAbort); - - if (submission) { - emit({ type: "result", output: submission.output, turns }); - return RUNNER_EXIT.ok; - } - failure ??= diagnose(agent, thrown); - emit({ type: "error", ...failure }); - return failure.code === "timeout" ? RUNNER_EXIT.timeout : RUNNER_EXIT.error; -} - -function forward(event: AgentEvent, emit: Emitter, model: Model) { - switch (event.type) { - case "tool_execution_start": - emit({ type: "tool_call", tool_call_id: event.toolCallId, tool: event.toolName, args: event.args }); - break; - case "tool_execution_end": - emit({ - type: "tool_result", - tool_call_id: event.toolCallId, - tool: event.toolName, - is_error: event.isError, - output_preview: outputPreview(event.result), - }); - break; - case "message_end": { - const message = event.message; - if (message.role !== "assistant") break; - const text = message.content - .flatMap((c) => (c.type === "text" ? [c.text] : [])) - .join("\n") - .trim(); - if (text) emit({ type: "assistant_text", text }); - const usage = toUsage(message.usage, model); - if (usage) emit({ type: "usage", usage }); - break; - } - } -} - -function callsSubmit(message: { role: string; content?: unknown }): boolean { - return ( - message.role === "assistant" && Array.isArray(message.content) && message.content.some((c) => c?.type === "toolCall" && c.name === SUBMIT_TOOL) - ); -} - -/** - * The loop ends only when every tool result in a batch sets `terminate`, so a submission batched with other - * calls (e.g. `ls` + submit) would cost another LLM request. The submit tool terminates on acceptance; - * this marks its siblings too. A rejected submission carries no `terminate`, so the model still gets to retry. - */ -function endsBatch({ toolCall, assistantMessage }: AfterToolCallContext): boolean { - return toolCall.name !== SUBMIT_TOOL && callsSubmit(assistantMessage); -} - -function lastAssistant(agent: Agent): AssistantMessage | undefined { - const messages = agent.state.messages; - for (let i = messages.length - 1; i >= 0; i--) { - const m = messages[i]; - if (m?.role === "assistant") return m; - } - return undefined; -} - -/** The model ended its turn without being stopped or failing (so a reminder may help). */ -function stoppedOnItsOwn(agent: Agent): boolean { - const last = lastAssistant(agent); - return last?.stopReason === "stop" || last?.stopReason === "length"; -} - -function diagnose(agent: Agent, thrown: unknown): Failure { - if (thrown !== undefined) return { code: "agent_error", message: errorMessage(thrown) }; - const last = lastAssistant(agent); - if (last?.stopReason === "error" || last?.stopReason === "aborted") { - return { code: "agent_error", message: last.errorMessage ?? agent.state.errorMessage ?? `model request ${last.stopReason}` }; - } - return { code: "no_submission", message: `the model stopped without calling ${SUBMIT_TOOL}, even after a reminder` }; -} diff --git a/app/runner/submit.ts b/app/runner/submit.ts deleted file mode 100644 index 352000f..0000000 --- a/app/runner/submit.ts +++ /dev/null @@ -1,100 +0,0 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; -import type { TSchema } from "typebox"; -import { Value } from "typebox/value"; - -export const SUBMIT_TOOL = "submit"; -export const MAX_SUBMIT_ATTEMPTS = 3; - -type JsonOutputSchema = Record; - -/** - * Why `schema` can't be a submit tool's parameters, or undefined if it can. Pi's Anthropic adapter sends only - * the top-level `properties` and `required`, so the root must be a plain object schema, not a union or a ref. - */ -export function outputSchemaProblem(schema: JsonOutputSchema): string | undefined { - if (schema.type !== "object") return 'output_schema must have type "object" at its root'; - const props = schema.properties; - if (typeof props !== "object" || props === null || Array.isArray(props)) { - return "output_schema must have a properties object at its root"; - } - try { - Value.Check(schema as TSchema, {}); - } catch (err) { - return `output_schema is not a usable JSON Schema: ${err instanceof Error ? err.message : String(err)}`; - } - return undefined; -} - -type Verdict = { ok: true } | { ok: false; problems: string[] }; - -/** - * Checks a submission against the output JSON Schema with TypeBox, which validates plain JSON Schema (including - * what `z.toJSONSchema` emits). A usability check so the model can fix its submission: the host revalidates - * the accepted output with the agent's Zod schema, and that check is authoritative. - */ -function checkSubmission(schema: JsonOutputSchema, candidate: unknown): Verdict { - if (Value.Check(schema as TSchema, candidate)) return { ok: true }; - const problems = [...Value.Errors(schema as TSchema, candidate)].map((e) => `${e.instancePath || "(root)"}: ${e.message}`); - return { ok: false, problems: problems.length > 0 ? [...new Set(problems)] : ["(root): does not match the output schema"] }; -} - -interface SubmitHandlers { - onAccepted(output: unknown): void; - /** Called with the number of rejected attempts so far, what was wrong with the latest one, and its top-level field names. */ - onRejected(attempts: number, problems: readonly string[], fields: readonly string[]): void; -} - -/** The tool's parameters as the model sees them: the output schema without JSON Schema metadata. */ -function toolParameters(schema: JsonOutputSchema): TSchema { - const { $schema: _schema, $id: _id, ...rest } = schema; - return rest as TSchema; -} - -/** - * The generic submit tool: its parameters are the agent's output schema. Submissions are checked in - * `prepareArguments`, which runs before Pi's own argument validation (which would otherwise reject a call - * without counting it, and coerce values the host's Zod schema would refuse). - */ -export function createSubmitTool(schema: JsonOutputSchema, handlers: SubmitHandlers): AgentTool { - let accepted: { output: unknown } | undefined; - /** The arguments as checked, before Pi's validation coerces them; calls run one at a time (sequential). */ - let checked: unknown; - let rejected = 0; - const what = typeof schema.description === "string" && schema.description ? ` ${schema.description}` : ""; - return { - name: SUBMIT_TOOL, - label: "Submit", - description: - `Submit your final output.${what} Call it exactly once, as your final action, on its own. The arguments are ` + - "checked against the schema; a rejected submission records nothing, so fix every listed problem and call it " + - `again (${MAX_SUBMIT_ATTEMPTS} attempts in total).`, - parameters: toolParameters(schema), - executionMode: "sequential", - prepareArguments(args) { - if (accepted) return args; - const verdict = checkSubmission(schema, args); - if (verdict.ok) { - checked = structuredClone(args); - return args; - } - rejected++; - const remaining = MAX_SUBMIT_ATTEMPTS - rejected; - const fields = args && typeof args === "object" && !Array.isArray(args) ? Object.keys(args) : []; - handlers.onRejected(rejected, verdict.problems, fields); - throw new Error( - `${SUBMIT_TOOL} rejected (attempt ${rejected} of ${MAX_SUBMIT_ATTEMPTS}); nothing was recorded. ` + - (remaining > 0 ? "Fix every problem below and call it again:\n" : "No attempts remain:\n") + - verdict.problems.map((p) => `- ${p}`).join("\n"), - ); - }, - async execute(_toolCallId, params) { - if (accepted) { - return { content: [{ type: "text", text: "Already submitted; this call was ignored." }], details: accepted.output, terminate: true }; - } - const output = checked ?? params; - accepted = { output }; - handlers.onAccepted(output); - return { content: [{ type: "text", text: "Submission recorded. You are done." }], details: output, terminate: true }; - }, - }; -} diff --git a/app/runner/test-stub.ts b/app/runner/test-stub.ts deleted file mode 100644 index 4ab2eaa..0000000 --- a/app/runner/test-stub.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { - type Api, - type FauxResponseStep as TestStubResponse, - fauxAssistantMessage as testStubAssistantMessage, - fauxProvider as testStubProvider, - fauxText as testStubText, - fauxToolCall as testStubToolCall, - type Model, - type MutableModels, -} from "@earendil-works/pi-ai"; -import type { RunnerInput } from "./protocol.js"; -import { SUBMIT_TOOL } from "./submit.js"; - -const TEST_STUB_MODEL = "test-stub"; -export type TestStub = (input: RunnerInput) => TestStubResponse[]; - -/** - * Adds a provider that returns deterministic test responses instead of calling a model. It carries the requested - * model's prices, so the runner can cost the stub's token estimates. - */ -export function registerTestStubModel(models: MutableModels, responses: TestStubResponse[], priced: Model): Model { - const { input, output, cacheRead, cacheWrite } = priced.cost; - const stub = testStubProvider({ - provider: TEST_STUB_MODEL, - models: [{ id: TEST_STUB_MODEL, reasoning: false, cost: { input, output, cacheRead, cacheWrite } }], - }); - models.setProvider(stub.provider); - stub.setResponses(responses); - return stub.getModel() as Model; -} - -const toolUse = { stopReason: "toolUse" } as const; - -/** - * Deterministic `--test-stub` run: lists the working directory if the agent has `ls`, then submits a minimal value - * built from the output schema. Exercises the runner end to end without a model (snapshot self-checks, tests). - */ -export const defaultTestStub: TestStub = (input) => { - const responses: TestStubResponse[] = []; - if (input.tools.includes("ls")) { - responses.push(testStubAssistantMessage([testStubText("[test-stub] Listing the working directory."), testStubToolCall("ls", {})], toolUse)); - } - const output = sampleFromSchema(input.output_schema) as Record; - responses.push(testStubAssistantMessage(testStubToolCall(SUBMIT_TOOL, output), toolUse)); - return responses; -}; - -/** A small value that satisfies common JSON Schema keywords (enough for test-stub runs, not a general generator). */ -function sampleFromSchema(schema: unknown): unknown { - if (typeof schema !== "object" || schema === null) return null; - const s = schema as Record; - if ("const" in s) return s.const; - if (Array.isArray(s.enum) && s.enum.length > 0) return s.enum[0]; - if (s.default !== undefined) return s.default; - const variants = (s.anyOf ?? s.oneOf) as unknown[] | undefined; - if (Array.isArray(variants) && variants.length > 0) { - const nonNull = variants.find((v) => (v as { type?: unknown })?.type !== "null"); - return sampleFromSchema(nonNull ?? variants[0]); - } - const types = Array.isArray(s.type) ? s.type : [s.type]; - const type = types.find((t) => t !== "null") ?? types[0]; - switch (type) { - case "object": { - const props = (s.properties ?? {}) as Record; - const required = Array.isArray(s.required) ? (s.required as string[]) : []; - return Object.fromEntries(required.map((key) => [key, sampleFromSchema(props[key])])); - } - case "array": - return Array.from({ length: typeof s.minItems === "number" ? s.minItems : 0 }, () => sampleFromSchema(s.items)); - case "string": { - const min = typeof s.minLength === "number" ? s.minLength : 0; - return "[test-stub]".padEnd(min, "."); - } - case "integer": - case "number": - return typeof s.minimum === "number" && s.minimum > 0 ? s.minimum : 0; - case "boolean": - return false; - case "null": - return null; - default: - return null; - } -} diff --git a/app/runner/tools.ts b/app/runner/tools/index.ts similarity index 75% rename from app/runner/tools.ts rename to app/runner/tools/index.ts index 3228126..da246a8 100644 --- a/app/runner/tools.ts +++ b/app/runner/tools/index.ts @@ -13,10 +13,13 @@ import { type ReadToolDetails, } from "@earendil-works/pi-agent-core"; import { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node"; -import { Type } from "typebox"; -import type { ToolName } from "./protocol.js"; +import { type TSchema, Type } from "typebox"; +import { Value } from "typebox/value"; +import type { ToolName } from "../core/protocol.js"; import { rejectReason } from "./shell-policy.js"; +export { rejectReason } from "./shell-policy.js"; + const DEFAULT_TIMEOUT_S = 120; const MAX_TIMEOUT_S = 1800; const LS_DEFAULT_LIMIT = 500; @@ -71,11 +74,53 @@ export function createTools(opts: ToolOptions): AgentTool[] { return [...new Set(opts.tools)].map((name) => factories[name]()); } -type HarnessTool = ReturnType | ReturnType; +/** A factory tool as any harness registers it: JSON Schema parameters and a text result, no library types. */ +export interface FactoryTool { + name: ToolName; + description: string; + /** JSON Schema of the arguments. */ + parameters: Record; + /** Validates `args`, runs the tool, and never throws: a failure comes back with `isError`. */ + execute(args: unknown, signal?: AbortSignal): Promise<{ text: string; isError: boolean }>; +} + +let nextCallId = 0; + +/** + * The tools named in `opts.tools` as FactoryTools: the same implementations, environment, policy, and limits as + * `createTools`, so every harness gives an agent identical tool behavior. + */ +export function createFactoryTools(opts: ToolOptions): FactoryTool[] { + return createTools(opts).map((tool) => ({ + name: tool.name as ToolName, + description: tool.description, + parameters: tool.parameters as Record, + async execute(args, signal) { + if (!Value.Check(tool.parameters as TSchema, args)) { + const problems = [...Value.Errors(tool.parameters as TSchema, args)].map((e) => `${e.instancePath || "(root)"}: ${e.message}`); + return { text: `Invalid arguments for ${tool.name}:\n${problems.map((p) => `- ${p}`).join("\n")}`, isError: true }; + } + try { + const result = await tool.execute(`factory-${++nextCallId}`, args, signal); + return { text: contentText(result.content), isError: false }; + } catch (err) { + return { text: err instanceof Error ? err.message : String(err), isError: true }; + } + }, + })); +} + +/** A tool result's text content; other content types become a placeholder. */ +export function contentText(content: unknown): string { + if (!Array.isArray(content)) return ""; + return content.map((c: { type?: string; text?: string }) => (c?.type === "text" ? (c.text ?? "") : `[${c?.type ?? "unknown"}]`)).join("\n"); +} + +type ContextTool = ReturnType | ReturnType; /** Binds one of Pi's harness tools (which take the execution context per call) to `context`. */ // biome-ignore lint/suspicious/noExplicitAny: see createTools. -function withContext(inner: HarnessTool, context: ExecutionToolContext): AgentTool { +function withContext(inner: ContextTool, context: ExecutionToolContext): AgentTool { return { ...inner, // biome-ignore lint/suspicious/noExplicitAny: the params match inner.parameters, validated by Pi. diff --git a/app/runner/shell-policy.ts b/app/runner/tools/shell-policy.ts similarity index 100% rename from app/runner/shell-policy.ts rename to app/runner/tools/shell-policy.ts diff --git a/app/sandbox/sandbox.ts b/app/sandbox/sandbox.ts index 3d951f0..e5f208c 100644 --- a/app/sandbox/sandbox.ts +++ b/app/sandbox/sandbox.ts @@ -11,6 +11,7 @@ import { computeLockfileHash, credentialFile, credentialHelperArgs, + ensureRunnerPayload, readToken, resolveBaseHead, runInstall, @@ -68,7 +69,8 @@ export async function openSandbox(repo: RepoContext, ref: SandboxRef, deps: Open const id = await createFromSnapshot(client, repo, deps); try { await client.waitRunning(id); - const info = await verifySandbox(client, id, { waitReady: true }); + await ensureRunnerPayload(client, id); + const info = await verifySandbox(client, id); // verify.sh can only probe /proc as root; a sandbox where it couldn't check never gets a model key. if (!info.runnerNonDumpable) throw new Error(`could not confirm the runner is non-dumpable in ${id}`); const manifest = SnapshotManifestSchema.parse(JSON.parse((await client.download(id, `${APP_DIR}/manifest.json`)).toString("utf8"))); diff --git a/app/sandbox/snapshot.ts b/app/sandbox/snapshot.ts index c058f48..725267d 100644 --- a/app/sandbox/snapshot.ts +++ b/app/sandbox/snapshot.ts @@ -1,7 +1,8 @@ import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; import { cp, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { ClientError, SandboxSnapshotNotFoundError } from "@renderinc/sdk"; @@ -79,7 +80,7 @@ export function credentialFile(token: string): string { // // Each step is an exec or upload in an allow-all builder sandbox, ordered by trust: // -// 1. Upload the scripts and runner bundle; install tools; create the runner user; self-test the runner +// 1. Upload the scripts and runner payload; install tools; create the runner user; self-test the runner // 2. Upload a read-only token, clone at `sha`, scrub the token and verify none remains // 3. Hand REPO_DIR to the runner user and run the repository's install as that user (no secret present) // 4. Manifest, lockdown, verify.sh, named snapshot, prune @@ -104,13 +105,14 @@ export async function buildSnapshot(client: SandboxClient, repo: RepoContext, sh await client.check(id, `mkdir -p -m 700 ${STAGING_DIR}`, { ...boot, waitReady: true }); // Uploaded as an opaque .tgz and extracted in the sandbox: the upload endpoint rejects an // uncompressed tar of the bundled runner's JavaScript, and doesn't accept application/gzip. - await client.upload(id, `${STAGING_DIR}/stage.tgz`, await stagingArchive()); + const payload = await runnerPayload(); + await client.upload(id, `${STAGING_DIR}/stage.tgz`, await stagingArchive(payload)); await client.check(id, `tar -xzf ${STAGING_DIR}/stage.tgz -C ${STAGING_DIR} && rm ${STAGING_DIR}/stage.tgz`, boot); await client.check( id, `mkdir -p ${APP_DIR}/runner ${APP_DIR}/build ${REPO_DIR} ${SCRATCH_DIR}` + ` && cp ${STAGING_DIR}/*.sh ${APP_DIR}/ && cp ${STAGING_DIR}/build/*.sh ${APP_DIR}/build/` + - ` && cp ${STAGING_DIR}/runner/runner.mjs ${APP_DIR}/runner/`, + ` && cp -R ${STAGING_DIR}/runner/. ${APP_DIR}/runner/`, boot, ); await client.check(id, `bash ${APP_DIR}/install-tools.sh`, prepare); @@ -265,14 +267,110 @@ async function createSnapshot(client: SandboxClient, sandboxId: string, name: st return snap; } -/** A gzipped tar of the runtime scripts and a freshly built, self-contained runner bundle. */ -async function stagingArchive(): Promise { - const bundle = await bundleRunner(); +// ── Runner payload ────────────────────────────────────────────────────── + +const RUNNER_DIR = `${APP_DIR}/runner`; + +export interface RunnerPayload { + /** Paths relative to APP_DIR/runner/. */ + files: ReadonlyMap; + sha256: string; + runnerSha256: string; +} + +let built: Promise | undefined; + +/** The runner bundle, built once per process. */ +export function runnerPayload(): Promise { + built ??= buildPayload().catch((err: unknown) => { + built = undefined; + throw err; + }); + return built; +} + +const sha256 = (data: string | Buffer) => createHash("sha256").update(data).digest("hex"); + +async function buildPayload(): Promise { + const runner = Buffer.from(await bundleRunner()); + const files = new Map([["runner.mjs", runner]]); + const digest = createHash("sha256"); + for (const path of [...files.keys()].sort()) { + const data = files.get(path) as Buffer; + digest.update(`${path}\0${data.length}\0`).update(data); + } + return { files, sha256: digest.digest("hex"), runnerSha256: sha256(runner) }; +} + +/** The payload as a gzipped tar rooted at APP_DIR/runner/. */ +async function payloadArchive(payload: RunnerPayload): Promise { + const dir = await mkdtemp(join(tmpdir(), "factory-payload-")); + try { + for (const [path, data] of payload.files) { + await mkdir(dirname(join(dir, path)), { recursive: true }); + await writeFile(join(dir, path), data); + } + await writeFile(join(dir, "payload.sha256"), payload.sha256); + const chunks: Buffer[] = []; + for await (const chunk of createTar({ cwd: dir, portable: true, gzip: true }, await readdir(dir))) chunks.push(chunk as Buffer); + return Buffer.concat(chunks); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** + * Replaces a restored sandbox's runner directory with this host's payload when they differ, so harness, model, and + * agent changes reach existing snapshots without a rebuild. It runs before verify.sh, which self-tests the runner. + * Returns whether it replaced the payload. + */ +export async function ensureRunnerPayload(client: SandboxClient, id: string): Promise { + const payload = await runnerPayload(); + const quick = { timeoutMs: client.t.verifyTimeoutMs }; + const current = (await client.exec(id, `cat ${RUNNER_DIR}/payload.sha256 2>/dev/null`, { ...quick, waitReady: true })).stdout.trim(); + let replaced = false; + if (current !== payload.sha256) { + const stage = `/tmp/factory-runner-${randomUUID()}`; + const next = `${APP_DIR}/runner.next`; + await client.check(id, `mkdir -m 700 ${stage}`, quick); + try { + // Gzipped, as the staging archive: the upload endpoint rejects the bundle's raw JavaScript. + await client.upload(id, `${stage}/payload.tgz`, await payloadArchive(payload)); + await client.check( + id, + [ + `rm -rf ${next} ${APP_DIR}/runner.prev`, + `mkdir -m 755 ${next}`, + `tar -xzf ${stage}/payload.tgz -C ${next}`, + `[ "$(sha256sum ${next}/runner.mjs | cut -d' ' -f1)" = ${payload.runnerSha256} ]`, + `chown -R root:root ${next}`, + `chmod -R a+rX,go-w ${next}`, + `mv ${RUNNER_DIR} ${APP_DIR}/runner.prev`, + `mv ${next} ${RUNNER_DIR}`, + `rm -rf ${APP_DIR}/runner.prev`, + ].join(" && "), + quick, + ); + } finally { + await client.exec(id, `rm -rf ${stage}`, quick).catch(() => {}); + } + log("info", "runner payload replaced", { sandbox: id, from: current || null, to: payload.sha256 }); + replaced = true; + } + return replaced; +} + +/** A gzipped tar of the runtime scripts and the runner payload under runner/. */ +async function stagingArchive(payload: RunnerPayload): Promise { const dir = await mkdtemp(join(tmpdir(), "factory-stage-")); try { await mkdir(join(dir, "runner")); await mkdir(join(dir, "build")); - await writeFile(join(dir, "runner/runner.mjs"), bundle); + for (const [path, data] of payload.files) { + await mkdir(dirname(join(dir, "runner", path)), { recursive: true }); + await writeFile(join(dir, "runner", path), data); + } + await writeFile(join(dir, "runner/payload.sha256"), payload.sha256); for (const file of RUNTIME_SCRIPTS) await cp(join(SANDBOX_SCRIPTS_DIR, file), join(dir, file)); const chunks: Buffer[] = []; for await (const chunk of createTar({ cwd: dir, portable: true, gzip: true }, await readdir(dir))) chunks.push(chunk as Buffer); diff --git a/docs/architecture.md b/docs/architecture.md index 2dab38f..63112b7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -117,12 +117,14 @@ A base-branch push builds a repository snapshot with the runner, a base checkout, and installed dependencies. A GitHub task then: 1. restores the newest snapshot; -2. verifies runtime permissions and process hardening; -3. fetches the exact requested ref with a temporary read-only token; -4. removes the token and verifies its removal; -5. reinstalls dependencies when the lockfiles or install command changed; -6. runs agents and repository commands as an unprivileged user; and -7. closes the Sandbox in `finally`. +2. replaces the snapshot's runner bundle with this deployment's when their + hashes differ, root-owned and read-only to the runner user; +3. verifies runtime permissions and process hardening; +4. fetches the exact requested ref with a temporary read-only token; +5. removes the token and verifies its removal; +6. reinstalls dependencies when the lockfiles or install command changed; +7. runs agents and repository commands as an unprivileged user; and +8. closes the Sandbox in `finally`. Render terminates a cancelled task without running `finally`. For that case, each GitHub task records its Sandbox ID in the status comment. A replacement @@ -138,7 +140,7 @@ prepared state, not trusted current state. An agent definition fixes: - its prompt; -- model tier and thinking level; +- its harness: the agent library and model that run it; - available tools; - output schema; - turn limit; and @@ -147,6 +149,17 @@ An agent definition fixes: The model loop and every model-directed tool run in the Sandbox beside the checkout. Agent tools never run on the Workflows host. +A harness adapter in `app/harness/` wraps an agent library's own +configuration. Agent definitions and their harnesses are bundled into the +Sandbox runner, which runs the agent named in its input on that agent's +harness. The host and the runner speak a versioned contract +(`app/runner/core/protocol.ts`): JSON on stdin, with the key on the first line, +and JSON Lines events on stdout. Every harness gives the model the factory's +own tools and `submit`, so tool authority and the read-only shell policy don't +depend on the library. In the runner, a kernel owns each run: its events, turn +and time limits, tools, and submission. A harness's driver only translates its +library. + Every agent finishes through a typed `submit` tool. The runner validates the submission against JSON Schema. The host validates it again with the agent's Zod schema. Repositories, refs, issue numbers, pull requests, branches, and @@ -251,7 +264,8 @@ Start with: 5. [`app/workflows/autopilot/index.ts`](../app/workflows/autopilot/index.ts) — parent/subtask orchestration. 6. [`app/agents/run.ts`](../app/agents/run.ts) — host-side agent execution. -7. [`app/runner/main.ts`](../app/runner/main.ts) — Sandbox runner entry point. +7. [`app/runner/main.ts`](../app/runner/main.ts) — Sandbox runner entry point; + [`app/harness/`](../app/harness/) — agent library adapters. 8. [`app/sandbox/sandbox.ts`](../app/sandbox/sandbox.ts) — Sandbox lifecycle. 9. [`app/changeset/index.ts`](../app/changeset/index.ts) — validated publishing. diff --git a/docs/configuration.md b/docs/configuration.md index af615bb..5e41d51 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,10 +31,6 @@ export default defineFactoryConfig({ }, commandPrefix: "/factory", minimumPermission: "write", - models: { - medium: "anthropic/claude-sonnet-5", - large: "anthropic/claude-opus-5", - }, limits: { repairRounds: 2, maxChangedFiles: 200, @@ -79,8 +75,6 @@ After adding a repository: | `labels.autopilot` | `factory:autopilot` | Runs all three stages through a reviewed draft PR | | `commandPrefix` | `/factory` | Prefix for commands such as `/factory cancel` | | `minimumPermission` | `write` | Live repository permission required to start work | -| `models.medium` | `anthropic/claude-sonnet-5` | Triage and PR Reviewer model | -| `models.large` | `anthropic/claude-opus-5` | Spec writer and implementer model | | `limits.repairRounds` | `2` | Repair attempts after failed verification | | `limits.maxChangedFiles` | `200` | Maximum files in a published change | | `limits.maxFileBytes` | `1000000` | Maximum size of one changed file | @@ -99,22 +93,41 @@ remove old ones. Keep `sandbox.timeoutSeconds` at least as long as the longest Workflows task. -## Models +## Agent harnesses and models + +Each agent runs on a harness: an agent library and its configuration, built by +an adapter in [`app/harness/`](../app/harness/). The built-in agents share two +harnesses from [`app/agents/models.ts`](../app/agents/models.ts): + +```typescript +export const piSonnet = pi({ model: getBuiltinModel("anthropic", "claude-sonnet-5") }); +export const piOpus = pi({ model: getBuiltinModel("anthropic", "claude-opus-5") }); +``` + +Triage and PR Reviewer use `piSonnet`; the spec writer and implementer use +`piOpus`. Each value is named for its library and model. To change a model, +add or edit a value and rename it to match, or give one agent its own harness +in its definition. Model IDs are type-checked against the library's +catalog, so a typo fails `npm run typecheck`. The `pi` adapter also takes +`thinkingLevel` (`off`, `minimal`, `low`, `medium`, or `high`; default +`medium`). + +Each harness declares the env var of its key, its `credential`. Setup and +`doctor` derive the keys they manage from the agents' harnesses, so a harness that needs a new key +needs no other configuration change. The `pi` adapter supports Anthropic +(`ANTHROPIC_API_KEY`), Google (`GEMINI_API_KEY`), and OpenAI +(`OPENAI_API_KEY`). After a harness change that needs a new key: -Agent definitions select the `medium` or `large` tier. -`factory.config.ts` maps each tier to a `provider/model` supported by -`@earendil-works/pi-ai`, the model client used by the Sandbox runner. +1. add the key to local `.env`; +2. run `npm run setup:render` to add it to the `software-factory-workflows` + env group; +3. deploy `factory-workflows`; and +4. run `npm run doctor -- --service workflows`. -Changing a model mapping requires a Workflows redeploy. To add another -provider: +### Tool support -1. choose a registered `provider/model`; -2. find its key name in [`app/runner/keys.ts`](../app/runner/keys.ts); -3. add the key to `factory-workflows` in [`render.yaml`](../render.yaml); -4. add it to [`.env.example`](../.env.example) and - `SERVICE_ENV` in `scripts/lib/requirements.ts`; -5. set the key in Render; and -6. run `npm run doctor -- --service workflows`. +Every harness gives the model the factory's own tools, so an agent's tools and +the read-only shell policy behave the same on any harness. Use a key dedicated to this deployment with a provider-enforced spend limit. The factory has no hard dollar budget. @@ -124,11 +137,12 @@ The factory has no hard dollar budget. | Change | Required action | |---|---| | Repository, label, permission, or publishing limit | Deploy both services | -| Model mapping | Deploy `factory-workflows` | -| Prompt or Workflows service code | Deploy `factory-workflows` | +| Agent harness or model | Deploy `factory-workflows` | +| Prompt, agent definition, or Workflows service code | Deploy `factory-workflows` | | Gateway routing or validation | Deploy `factory-gateway` | | Label, App permission, or subscribed event | Run `npm run setup:github`; update the App when prompted | -| `app/runner/` or `sandbox/` | Deploy Workflows, then refresh each repository snapshot | +| `app/runner/` or `app/harness/` sources | Deploy `factory-workflows`; each Sandbox gets the new runner payload when it opens | +| `sandbox/` | Deploy `factory-workflows`, then refresh each repository snapshot | | Repository `install` command or lockfiles | Refresh the snapshot, or let the next task reinstall | ## Snapshots @@ -261,7 +275,7 @@ service. Every other deployed value comes from an | `RENDER_WORKSPACE_ID` | Workspace that owns Sandboxes and snapshots | | `GITHUB_APP_ID` | Authenticate the GitHub App | | `GITHUB_APP_PRIVATE_KEY` | Mint installation tokens | -| `ANTHROPIC_API_KEY` | Call the default models | +| `ANTHROPIC_API_KEY` | Call the default models; each agent's harness declares the key it needs | | `GITHUB_API_URL` | Optional GitHub API base URL | `GITHUB_API_URL` is not in the Blueprint. GitHub Enterprise Server users must diff --git a/docs/runbooks/extend.md b/docs/runbooks/extend.md index 082b84c..6ad0fca 100644 --- a/docs/runbooks/extend.md +++ b/docs/runbooks/extend.md @@ -1,7 +1,7 @@ # Extension runbook -Use this runbook when adding an agent, a Render Workflows task, or a GitHub -trigger. It is written for coding agents working in a fork of +Use this runbook when adding an agent, an agent harness, a Render Workflows +task, or a GitHub trigger. It is written for coding agents working in a fork of software-factory. Read [Architecture](../architecture.md) first. Do not change the authority @@ -15,6 +15,7 @@ Inspect: - `factory.config.ts`; - `app/agents/`; +- `app/harness/`; - `app/workflows/`; - `app/schemas/`; - `app/gateway/routes.ts`; @@ -29,6 +30,8 @@ Decide which change you need: - **Agent only:** a new prompt, tool set, and typed result used by an existing task. +- **Agent harness:** another agent library or model client behind the runner + contract. - **Claiming task:** work attached to one issue or pull request that must stop when another run takes over. - **Plain task:** maintenance work without an issue or pull request claim, @@ -73,10 +76,12 @@ app/agents// Define the agent: ```typescript +import { piSonnet } from "../models.js"; + export const finder = defineAgent({ id: "finder", prompt: "finder/prompt.md", - model: "medium", + harness: piSonnet, tools: ["read", "ls", "bash"], output: Finding, maxTurns: 40, @@ -87,21 +92,24 @@ The definition fixes the agent's authority: - `id` must be unique and stable; - `prompt` is relative to `app/agents/`; -- `model` is `medium` or `large`; +- `harness` is a value from `app/agents/models.ts` or one built with an + adapter from `app/harness/`; - `tools` may include `read`, `ls`, `bash`, `edit`, and `write`; -- `output` is a bounded strict Zod object; -- `maxTurns` is a positive model-call limit; and -- `thinking` is optional. +- `output` is a bounded strict Zod object; and +- `maxTurns` is a positive model-call limit. -Grant `edit` or `write` only when the agent must change the checkout. The -read-only shell policy in `app/runner/tools.ts` guides read-only agents; it is -not a security boundary. +Grant `edit` or `write` only when the agent must change the checkout. +The read-only shell policy in `app/runner/tools/shell-policy.ts` guides +read-only agents; it is not a security boundary. The prompt should explain the job, evidence requirements, limits, and when to call `submit`. Put repository-, issue-, and run-specific data in the message passed to `runAgent`, not in the prompt. -Export the agent from `app/agents/index.ts`. +Export the agent from `app/agents/index.ts` and add it to `AGENTS` in +`app/agents/registry.ts`. The Sandbox runner is bundled from that list and +runs only the agents in it. Agent modules are bundled into the runner, so they +must not import host-only code. ### 3. Call the agent @@ -122,6 +130,45 @@ list only if write tools are intentional. Add task-level tests for the user-visible result and meaningful failures. Use fakes; tests must not call GitHub, Render, or a model provider. +## Add an agent harness + +A harness puts an agent library behind the runner. Inside the Sandbox, the +kernel ([`app/runner/core/kernel.ts`](../../app/runner/core/kernel.ts)) runs +the agent: it emits every protocol event, enforces `max_turns` and +`timeout_ms`, and owns the tools and the submission. A harness supplies a +driver that only translates the library. + +1. Add an adapter in `app/harness/.ts` that takes the library's own + configuration and returns a `Harness` (`app/harness/types.ts`): + - `name` and `label` name the library and model before a run; + - `credential` is the env var on the Workflows service that holds its key; + the host sends its value into the Sandbox; and + - `driver` is a `Driver` (`app/runner/core/driver.ts`). +2. Write the driver in `app/runner//`. `open(ctx)` resolves the model + and reports the library version; the session's `run()` converses with the + model: + - offer the model exactly `ctx.tools` and `ctx.submit`, bound to the + library's tool type. Never offer the library's own shell, file, network, + or subagent tools: their authority isn't the factory's; + - call `ctx.requestModel()` before every model request, retries included, + and don't send the request when it returns false; + - report text, usage, and tool calls and results through `ctx.report`; and + - stop when `ctx.signal` aborts. + `app/runner/pi/` is the reference. +3. Add the harness to `HARNESSES` in `tests/helpers/runner.ts` so it runs the + conformance suite. Each entry builds the harness around a scripted model + that plays the test's turns, using the library's own fake model (for + example, Pi's fake provider). Production code has no test mode; if the + library has no fake model, test it live. +4. The runner bundle is one self-contained file and the Sandbox has only + Node, so the library must bundle with esbuild. If it holds the key in a + process other than the runner, make that process non-dumpable before it + reads the key, so tool subprocesses can't read its memory. +5. Export the adapter from `app/harness/index.ts` and use it in an agent + definition or `app/agents/models.ts`. + +`tests/agents-catalog.test.ts` fails if the bundle imports host-only code. + ## Add a GitHub task Use `defineGitHubTask()` for work attached to one issue or pull request. It @@ -287,7 +334,8 @@ Before handing off, confirm: - the new agent has the minimum tools it needs; - model output cannot select an external write target; - the gateway still imports no GitHub auth, Sandbox, agent, or workflow code; -- `app/runner/` still imports nothing from the rest of `app/`; +- the runner bundle (`app/runner/`, `app/harness/`, agent definitions, and + schemas) still imports no host-only code; - GitHub tasks record their Sandbox before running an agent; - tests cover the user-visible success and important failures; and - documentation and setup requirements changed when the App contract changed. @@ -301,7 +349,12 @@ Before handing off, confirm: `scripts/lib/requirements.ts`. Add it to `.env`, rerun `npm run setup:render`, and redeploy the services that link its group. See [Environment groups](../configuration.md#environment-groups). -- Changes under `app/runner/` or `sandbox/` also require a snapshot refresh for +- A harness whose key is new needs the key in `.env`, `npm run setup:render`, + and a Workflows deploy. Setup and `doctor` derive the key list from the + agents' harnesses. +- Changes under `app/runner/`, `app/harness/`, or agent definitions reach + Sandboxes with the Workflows deploy: each Sandbox gets the new runner bundle + when it opens. Changes under `sandbox/` also require a snapshot refresh for every configured repository. State the required deploy and refresh actions in the handoff. diff --git a/factory.config.ts b/factory.config.ts index 5594ecc..2444fbf 100644 --- a/factory.config.ts +++ b/factory.config.ts @@ -1,5 +1,3 @@ -export type ModelTier = "medium" | "large"; - export interface RepoConfig { /** Branch that specs are implemented against and snapshots are built from. */ baseBranch: string; @@ -17,8 +15,6 @@ export interface FactoryConfig { commandPrefix: string; /** GitHub permission the triggering actor needs on the repository. */ minimumPermission: "triage" | "write" | "maintain" | "admin"; - /** "provider/model" per tier, from Pi's model registry. */ - models: Record; limits: { /** Fix rounds after a failed verify in Spec-to-PR. */ repairRounds: number; @@ -53,10 +49,6 @@ export default defineFactoryConfig({ }, commandPrefix: "/factory", minimumPermission: "write", - models: { - medium: "anthropic/claude-sonnet-5", - large: "anthropic/claude-opus-5", - }, limits: { repairRounds: 2, maxChangedFiles: 200, diff --git a/scripts/doctor.ts b/scripts/doctor.ts index ac51ef8..5605f83 100644 --- a/scripts/doctor.ts +++ b/scripts/doctor.ts @@ -26,7 +26,7 @@ if (!services) { const out = new Reporter(); try { - process.exitCode = await runDoctor({ env: process.env, services, repos: Object.keys(config.repos), models: config.models }, out); + process.exitCode = await runDoctor({ env: process.env, services, repos: Object.keys(config.repos) }, out); } catch (error) { out.fail(`doctor crashed: ${errorMessage(error)}`); process.exitCode = 1; diff --git a/scripts/lib/doctor.ts b/scripts/lib/doctor.ts index 9862312..3c52ccb 100644 --- a/scripts/lib/doctor.ts +++ b/scripts/lib/doctor.ts @@ -1,35 +1,28 @@ /** doctor: read-only checks of one or both services' environment, the GitHub App, and Workflows tasks. */ +import type { Agent } from "../../app/agents/define.js"; +import { AGENTS } from "../../app/agents/registry.js"; import { TASKS } from "../../app/config/index.js"; -import { PROVIDER_KEY_ENV } from "../../app/runner/keys.js"; import { appCredentialsFrom, type Env, GitHubApp, type ScriptDeps, splitRepo } from "./github.js"; import { PLACEHOLDER_WEBHOOK_URL } from "./manifest.js"; import { errorMessage, type Reporter } from "./output.js"; -import { - envGroupFor, - LABELS, - missingEvents, - missingPermissions, - SECRET_ENV, - SERVICE_ENV, - type Service, - WEBHOOK_PATH, -} from "./requirements.js"; +import { envGroupFor, LABELS, missingEvents, missingPermissions, SECRET_ENV, SERVICE_ENV, type Service, WEBHOOK_PATH } from "./requirements.js"; import { appSettingsUrl } from "./setup-github.js"; export interface DoctorOptions extends ScriptDeps { services: readonly Service[]; repos: readonly string[]; - /** config.models: "provider/model" per tier. */ - models: Readonly>; + /** The agents whose harnesses to check; defaults to every agent the factory runs. */ + agents?: readonly Agent[]; renderApiUrl?: string; } export async function runDoctor(opts: DoctorOptions, out: Reporter): Promise { const { env } = opts; - out.addSecrets(...SECRET_ENV.map((name) => env[name]), ...modelKeys(opts.models).map((name) => env[name])); + const agents = opts.agents ?? AGENTS; + out.addSecrets(...SECRET_ENV.map((name) => env[name]), ...agents.map((agent) => env[agent.harness.credential])); checkEnv(opts, out); - if (opts.services.includes("workflows")) checkModels(opts, out); + if (opts.services.includes("workflows")) checkHarnesses(env, agents, out); await checkGitHub(opts, out); await checkTasks(opts, out); return out.summary(); @@ -55,41 +48,20 @@ function setFix(service: Service, name: string): string { return `Set it in the ${group.name} env group: add it to .env and run \`npm run setup:render\``; } -/** The env var each configured provider's key is read from, in config order. */ -function modelKeys(models: Readonly>): string[] { - const names = Object.values(models) - .map((id) => PROVIDER_KEY_ENV[id.split("/")[0] ?? ""]?.[0]) - .filter((name): name is string => !!name); - return [...new Set(names)]; -} - -function checkModels({ env, models }: DoctorOptions, out: Reporter): void { - out.heading("Model keys"); - const tiersByProvider = new Map(); - for (const [tier, id] of Object.entries(models)) { - const slash = id.indexOf("/"); - if (slash <= 0) { - out.fail(`models.${tier} is not "provider/model": ${id}`); - continue; - } - const provider = id.slice(0, slash); - tiersByProvider.set(provider, [...(tiersByProvider.get(provider) ?? []), tier]); +/** Each harness the agents use, the agents on it, and whether its key is set. */ +function checkHarnesses(env: Env, agents: readonly Agent[], out: Reporter): void { + out.heading("Agent harnesses"); + const groups = new Map(); + for (const agent of agents) { + const key = `${agent.harness.name} ${agent.harness.label}`; + const group = groups.get(key) ?? { agent, ids: [] }; + group.ids.push(agent.id); + groups.set(key, group); } - for (const [provider, tiers] of tiersByProvider) { - const keyName = PROVIDER_KEY_ENV[provider]?.[0]; - const label = `${provider} (${tiers.join(", ")})`; - if (!keyName) { - out.fail(`${label}: unknown provider`, "Use a provider from app/runner/keys.ts"); - continue; - } - if (env[keyName]?.trim()) out.pass(`${label}: ${keyName} is set`); - else out.fail(`${label}: ${keyName} is not set`, `${setFix("workflows", keyName)}; use a spend-capped key`); - if (!SERVICE_ENV.workflows.includes(keyName)) { - out.warn( - `setup:render doesn't manage ${keyName}`, - "Add it to the software-factory-workflows group in ENV_GROUPS and SERVICE_ENV (scripts/lib/requirements.ts)", - ); - } + for (const [label, { agent, ids }] of groups) { + const { credential } = agent.harness; + if (env[credential]?.trim()) out.pass(`${label} (${ids.join(", ")}): ${credential} is set`); + else out.fail(`${label} (${ids.join(", ")}): ${credential} is not set`, `${setFix("workflows", credential)}; use a spend-capped key`); } } diff --git a/scripts/lib/requirements.ts b/scripts/lib/requirements.ts index 6a6f9b7..1765e93 100644 --- a/scripts/lib/requirements.ts +++ b/scripts/lib/requirements.ts @@ -1,4 +1,5 @@ /** What the factory needs from GitHub and from each service's environment, in one place for setup and doctor. */ +import { AGENTS } from "../../app/agents/registry.js"; import { config } from "../../app/config/index.js"; import { WORKING_LABEL } from "../../app/claims/status.js"; @@ -60,10 +61,13 @@ export const LABELS: readonly LabelSpec[] = [ export type Service = "gateway" | "workflows"; +/** The credential each agent's harness declares (app/agents/), once each, in agent order. Workflows sends them to its sandboxes. */ +export const HARNESS_KEY_ENV: readonly string[] = [...new Set(AGENTS.map((agent) => agent.harness.credential))]; + /** Each service's env vars, exactly as render.yaml declares them (through its env groups or directly). */ export const SERVICE_ENV: Readonly> = { gateway: ["GITHUB_WEBHOOK_SECRET", "RENDER_API_KEY", "RENDER_WORKFLOW_SLUG"], - workflows: ["RENDER_API_KEY", "RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", "ANTHROPIC_API_KEY"], + workflows: ["RENDER_API_KEY", "RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", ...HARNESS_KEY_ENV], }; export interface EnvGroupSpec { @@ -84,7 +88,7 @@ export const ENV_GROUPS: readonly EnvGroupSpec[] = [ { name: "software-factory-workflows", services: ["workflows"], - keys: ["RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", "ANTHROPIC_API_KEY"], + keys: ["RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", ...HARNESS_KEY_ENV], }, ]; diff --git a/tests/agents-catalog.test.ts b/tests/agents-catalog.test.ts index b3f8989..0cbc203 100644 --- a/tests/agents-catalog.test.ts +++ b/tests/agents-catalog.test.ts @@ -1,8 +1,25 @@ import { describe, expect, it } from "vitest"; -import { implementer, prReviewer, specWriter, triage } from "../app/agents/index.js"; +import { AGENTS, implementer, prReviewer, specWriter, triage } from "../app/agents/index.js"; +import { bundleInputs } from "../app/runner/bundle.js"; const agents = [triage, specWriter, implementer, prReviewer]; +describe("agent registry", () => { + it("bundles every built-in agent, each with a non-empty prompt", () => { + expect(AGENTS.map((agent) => agent.id)).toEqual(agents.map((agent) => agent.id)); + for (const agent of AGENTS) expect(agent.prompt.trim(), agent.id).not.toBe(""); + }); + + it("keeps host-only code out of the sandbox runner bundle", { timeout: 60_000 }, async () => { + const app = (await bundleInputs()).filter((path) => !path.includes("node_modules/")); + const hostOnly = app.filter( + (path) => /^app\/(github|sandbox|workflows|claims|gateway|events|config|entrypoints)\//.test(path) || path === "app/agents/run.ts", + ); + expect(hostOnly).toEqual([]); + expect(app).toContain("app/agents/registry.ts"); + }); +}); + describe("agent authority", () => { it("gives every agent a unique id and prompt file", () => { const ids = agents.map((agent) => agent.id); diff --git a/tests/agents.test.ts b/tests/agents.test.ts index 08bbf82..9d393cb 100644 --- a/tests/agents.test.ts +++ b/tests/agents.test.ts @@ -1,8 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { z } from "zod"; -import { defineAgent, runAgent } from "../app/agents/index.js"; +import { defineAgent, piOpus, runAgent } from "../app/agents/index.js"; import { emit } from "../app/events/index.js"; -import { RunnerInput } from "../app/runner/protocol.js"; +import { PROTOCOL_VERSION, RunnerInput } from "../app/runner/core/protocol.js"; import { REPO_DIR } from "../app/sandbox/types.js"; import { eventLine, FakeSandbox } from "./helpers/fake-sandbox.js"; @@ -20,11 +20,10 @@ const Report = z.object({ const reviewer = defineAgent({ id: "reviewer", prompt: PROMPT, - model: "large", + harness: piOpus, tools: ["read", "ls", "bash"], output: Report, maxTurns: 40, - thinking: "high", }); const emitted = () => vi.mocked(emit).mock.calls.map(([e]) => e); @@ -48,7 +47,7 @@ describe("defineAgent", () => { const cannedRun = (output: unknown) => [ - eventLine({ type: "started", agent_id: "reviewer", model: "anthropic/claude-opus-5", pi_version: "0.83.0" }), + eventLine({ type: "started", agent_id: "reviewer", harness: "pi", harness_version: "0.83.0", model: "anthropic/claude-opus-5" }), eventLine({ type: "usage", usage: { input_tokens: 1000, output_tokens: 200, cache_read_tokens: 50, cache_write_tokens: 0, cost_usd: 0.01 } }), eventLine({ type: "tool_call", tool_call_id: "c1", tool: "ls", args: {} }), eventLine({ type: "usage", usage: { input_tokens: 1500, output_tokens: 100, cache_read_tokens: 0, cache_write_tokens: 10, cost_usd: 0.02 } }), @@ -76,6 +75,7 @@ describe("runAgent in a sandbox", () => { expect(sandbox.uploads.get(`${runDir}/key`)).toBe(HOST_KEY); const input = RunnerInput.parse(JSON.parse(sandbox.uploads.get(`${runDir}/input.json`) as string)); expect(input).toMatchObject({ + protocol_version: PROTOCOL_VERSION, agent_id: "reviewer", system_prompt: reviewer.prompt, message: "Review it.", @@ -89,7 +89,15 @@ describe("runAgent in a sandbox", () => { expect(emitted()[0]).toMatchObject({ type: "agent.finished", traceId: "trace-2", - payload: { outcome: "ok", turns: 2, model: "anthropic/claude-opus-5", costUsd: run.costUsd, usage: run.usage }, + payload: { + outcome: "ok", + turns: 2, + harness: "pi", + harnessVersion: "0.83.0", + model: "anthropic/claude-opus-5", + costUsd: run.costUsd, + usage: run.usage, + }, }); }); }); diff --git a/tests/harness-pi.test.ts b/tests/harness-pi.test.ts new file mode 100644 index 0000000..a54cbde --- /dev/null +++ b/tests/harness-pi.test.ts @@ -0,0 +1,57 @@ +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; +import { describe, expect, it } from "vitest"; +import { fauxPi } from "./helpers/pi-scripted.js"; +import { GOOD, makeInput, makeWorkspace, parseEvents, runContract, SUBMIT, terminal, toolResults } from "./helpers/runner.js"; + +const calls = (...list: [string, Record][]) => + fauxAssistantMessage( + list.map(([name, args]) => fauxToolCall(name, args)), + { stopReason: "toolUse" }, + ); +const overloaded = () => fauxAssistantMessage([], { stopReason: "error", errorMessage: "overloaded_error: Overloaded" }); + +describe("pi driver", { timeout: 30_000 }, () => { + it("ends with no_submission when the model stops without submitting, without asking it again", async () => { + const { harness, stub } = fauxPi([fauxAssistantMessage([fauxText("All done.")]), calls([SUBMIT, GOOD])]); + const { code, events } = await runContract(harness, makeInput(makeWorkspace())); + expect(terminal(events)).toMatchObject({ type: "error", code: "no_submission", message: "the model stopped without calling submit" }); + expect(stub.getPendingResponseCount()).toBe(1); + expect(code).toBe(1); + }); + + it("ends a batch that submits without another model request", async () => { + const { harness, stub } = fauxPi([calls(["ls", {}], [SUBMIT, GOOD]), calls(["ls", {}])]); + const { code, events } = await runContract(harness, makeInput(makeWorkspace())); + expect(toolResults(events).map((r) => [r.tool, r.is_error])).toEqual([ + ["ls", false], + [SUBMIT, false], + ]); + expect(terminal(events)).toMatchObject({ type: "result", output: GOOD, turns: 1 }); + expect(stub.getPendingResponseCount()).toBe(1); + expect(code).toBe(0); + }); + + it("retries a transient model error, and the retry counts as a turn", async () => { + const { harness } = fauxPi([overloaded(), calls([SUBMIT, GOOD])]); + const { code, events } = await runContract(harness, makeInput(makeWorkspace(), { max_turns: 2 })); + expect(terminal(events)).toMatchObject({ type: "result", output: GOOD, turns: 2 }); + expect(code).toBe(0); + }); + + it("doesn't send a retry past max_turns", async () => { + const { harness, stub } = fauxPi([overloaded(), calls([SUBMIT, GOOD])]); + const { code, events } = await runContract(harness, makeInput(makeWorkspace(), { max_turns: 1 })); + expect(terminal(events)).toMatchObject({ type: "error", code: "max_turns", message: expect.stringContaining("turn limit (1)") }); + expect(stub.getPendingResponseCount()).toBe(1); + expect(code).toBe(1); + }); + + it("rejects a model its registry doesn't have, before starting", async () => { + const { harness } = fauxPi([], "missing"); + const { code, stdout } = await runContract(harness, makeInput(makeWorkspace())); + expect(parseEvents(stdout)).toEqual([ + expect.objectContaining({ type: "error", code: "invalid_input", message: expect.stringContaining('unknown model "scripted/missing"') }), + ]); + expect(code).toBe(2); + }); +}); diff --git a/tests/helpers/fake-sandbox.ts b/tests/helpers/fake-sandbox.ts index 74f1ff9..c6f986d 100644 --- a/tests/helpers/fake-sandbox.ts +++ b/tests/helpers/fake-sandbox.ts @@ -1,4 +1,4 @@ -import type { RunnerEvent } from "../../app/runner/protocol.js"; +import type { RunnerEvent } from "../../app/runner/core/protocol.js"; import type { ExecOptions, ExecResult, Sandbox } from "../../app/sandbox/types.js"; type WithoutAt = E extends unknown ? Omit : never; diff --git a/tests/helpers/pi-scripted.ts b/tests/helpers/pi-scripted.ts new file mode 100644 index 0000000..dce38ad --- /dev/null +++ b/tests/helpers/pi-scripted.ts @@ -0,0 +1,29 @@ +import { type FauxProviderHandle, type FauxResponseStep, fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; +import type { Harness } from "../../app/harness/index.js"; +import { pi } from "../../app/harness/index.js"; +import { createModels } from "../../app/runner/pi/driver.js"; +import type { TestTurn } from "./scripted-harness.js"; + +/** The pi harness with Pi's fake provider as its model, playing `turns`: a model like any other, to the harness. */ +export function scriptedPi(turns: readonly TestTurn[]): Harness { + return fauxPi( + turns.map((turn) => + fauxAssistantMessage([...(turn.text ? [fauxText(turn.text)] : []), ...turn.calls.map((c) => fauxToolCall(c.name, c.args))], { + stopReason: turn.calls.length > 0 ? "toolUse" : "stop", + }), + ), + ).harness; +} + +/** The pi harness on Pi's fake provider, answering each model request with the next of `responses`. */ +export function fauxPi(responses: FauxResponseStep[], modelId = "scripted"): { harness: Harness; stub: FauxProviderHandle } { + const stub = fauxProvider({ provider: "scripted", models: [{ id: "scripted", reasoning: false }] }); + stub.setResponses(responses); + const models = () => { + const registry = createModels(); + registry.setProvider(stub.provider); + return registry; + }; + const model = { ...stub.getModel(), id: modelId }; + return { harness: pi({ model, models, thinkingLevel: "off", credential: "SCRIPTED_API_KEY" }), stub }; +} diff --git a/tests/helpers/runner.ts b/tests/helpers/runner.ts index 00116c0..894d05c 100644 --- a/tests/helpers/runner.ts +++ b/tests/helpers/runner.ts @@ -4,13 +4,19 @@ import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Readable } from "node:stream"; -import { fauxAssistantMessage as testStubAssistantMessage, fauxToolCall as testStubToolCall } from "@earendil-works/pi-ai"; +import { z } from "zod"; +import { type Agent, defineAgent } from "../../app/agents/define.js"; +import type { Harness } from "../../app/harness/types.js"; +import { PROTOCOL_VERSION, RunnerEvent, type RunnerInput, type ToolName } from "../../app/runner/core/protocol.js"; import { bundleRunner } from "../../app/runner/bundle.js"; import { main } from "../../app/runner/main.js"; -import { RunnerEvent, type RunnerInput, type ToolName } from "../../app/runner/protocol.js"; -import type { TestStub } from "../../app/runner/test-stub.js"; +import { scriptedPi } from "./pi-scripted.js"; +import { scripted, type TestTurn } from "./scripted-harness.js"; + +export type { TestTurn }; export const SUBMIT = "submit"; +export const TEST_AGENT_ID = "test-agent"; /** The output schema most runner tests use: what `z.toJSONSchema` emits for a small object. */ const OUTPUT_SCHEMA = { @@ -26,6 +32,30 @@ const OUTPUT_SCHEMA = { } as const; export const GOOD = { summary: "All good.", files: ["README.md"], confidence: "high" }; +/** A test agent on `harness`; the runner takes its tools and schema from the input, so these are placeholders. */ +export function testAgent(harness: Harness): Agent { + return defineAgent({ + id: TEST_AGENT_ID, + prompt: "../../tests/fixtures/agents/echo.md", + harness, + tools: ["read", "ls", "bash", "edit", "write"], + output: z.object({ summary: z.string() }), + maxTurns: 20, + }); +} + +/** + * Every harness the conformance suite runs against, each built around a scripted model that plays the test's turns: + * to the harness, a model like any other. + */ +export const HARNESSES: ReadonlyArray<{ name: string; make: (turns: readonly TestTurn[]) => Harness }> = [ + { name: "pi", make: scriptedPi }, + { name: "scripted", make: scripted }, +]; + +/** The key conformance runs deliver on stdin; scripted models ignore it. */ +export const TEST_KEY = "sk-test-STDINKEY-0123456789"; + export interface Workspace { cwd: string; scratchDir: string; @@ -55,13 +85,11 @@ export function makeWorkspace(): Workspace { export function makeInput(ws: Workspace | undefined, overrides: Partial = {}): RunnerInput { const tools: ToolName[] = overrides.tools ?? (ws ? ["read", "ls", "bash"] : []); return { + protocol_version: PROTOCOL_VERSION, trace_id: randomUUID(), - agent_id: "test-agent", + agent_id: TEST_AGENT_ID, system_prompt: "You are a test agent.", message: "Do the thing.", - model_provider: "anthropic", - model: "claude-sonnet-4-5", - thinking_level: "off", tools, output_schema: OUTPUT_SCHEMA, max_turns: 20, @@ -89,22 +117,25 @@ export function terminal(events: RunnerEvent[]) { export const toolResults = (events: RunnerEvent[]) => events.flatMap((e) => (e.type === "tool_result" ? [e] : [])); -/** Runs main() in-process with deterministic model responses. */ -export async function runWithTestStub(input: unknown, testStub: TestStub, extraArgs: string[] = []) { +/** One scripted model turn that makes one tool call. */ +export const call = (name: string, args: Record): TestTurn => ({ calls: [{ name, args }] }); + +/** + * Runs main() in-process through the runner contract as the sandbox does: `key` on the first line of stdin, then + * the input; RunnerEvent JSONL out. `harness` runs behind the test agent. + */ +export async function runContract(harness: Harness, input: RunnerInput, key = TEST_KEY) { let stdout = ""; - const code = await main(["--test-stub", ...extraArgs], { - stdin: Readable.from([typeof input === "string" ? input : JSON.stringify(input)]), + const code = await main(["--key-stdin"], { + stdin: Readable.from([`${key}\n${JSON.stringify(input)}`]), write: async (chunk) => { stdout += chunk; }, - testStub, + agents: [testAgent(harness)], }); return { code, stdout, events: parseEvents(stdout) }; } -const toolUse = { stopReason: "toolUse" } as const; -export const call = (name: string, args: Record) => testStubAssistantMessage(testStubToolCall(name, args), toolUse); - let runnerPath: string | undefined; /** Bundles the runner into a fresh temp dir, outside the repo, so no node_modules is reachable from it. */ diff --git a/tests/helpers/scripted-harness.ts b/tests/helpers/scripted-harness.ts new file mode 100644 index 0000000..4a0bd3f --- /dev/null +++ b/tests/helpers/scripted-harness.ts @@ -0,0 +1,43 @@ +import type { DriverContext } from "../../app/runner/core/driver.js"; +import type { Harness } from "../../app/harness/types.js"; + +/** One scripted model turn: optional text, then tool calls in order. A turn without calls ends the conversation. */ +export interface TestTurn { + text?: string; + calls: { name: string; args: Record }[]; +} + +/** + * A driver with no agent library: each model request plays the next of `turns`. It stands in for a third-party + * library in the conformance suite, proving a harness can be swapped without host or runner changes. + */ +export function scripted(turns: readonly TestTurn[]): Harness { + return { + name: "scripted", + label: "scripted", + credential: "SCRIPTED_API_KEY", + driver: { open: (ctx) => ({ version: "1.0.0", model: "scripted", run: () => play(ctx, turns) }) }, + }; +} + +async function play(ctx: DriverContext, turns: readonly TestTurn[]): Promise { + const tools = new Map(ctx.tools.map((t) => [t.name as string, t])); + let callId = 0; + for (const turn of turns) { + if (!ctx.requestModel()) return; + if (turn.text) ctx.report.text(turn.text); + if (turn.calls.length === 0) return; + for (const call of turn.calls) { + const id = `call-${++callId}`; + ctx.report.toolCall(id, call.name, call.args); + const tool = tools.get(call.name); + const out = + call.name === ctx.submit.name + ? ctx.submit.call(call.args) + : tool + ? await tool.execute(call.args, ctx.signal) + : { text: `Tool ${call.name} not found`, isError: true }; + ctx.report.toolResult(id, call.name, out.text, out.isError); + } + } +} diff --git a/tests/kernel.test.ts b/tests/kernel.test.ts new file mode 100644 index 0000000..b862a4c --- /dev/null +++ b/tests/kernel.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it } from "vitest"; +import { type Driver, type DriverContext, InvalidInput } from "../app/runner/core/driver.js"; +import { createEmitter } from "../app/runner/core/emit.js"; +import { executeHarness } from "../app/runner/core/kernel.js"; +import type { RunnerInput } from "../app/runner/core/protocol.js"; +import { GOOD, makeInput, makeWorkspace, parseEvents, terminal, toolResults } from "./helpers/runner.js"; + +const BAD = { summary: "x", files: "README.md" }; + +/** A driver whose conversation is `converse`, run with the kernel's context. */ +const driver = (converse: (ctx: DriverContext) => Promise): Driver => ({ + open: (ctx) => ({ version: "1.2.3", model: "fake-model", run: () => converse(ctx) }), +}); + +const aborted = (ctx: DriverContext) => new Promise((resolve) => ctx.signal.addEventListener("abort", () => resolve(), { once: true })); + +async function execute(harness: Driver, input: RunnerInput = makeInput(undefined), signal?: AbortSignal) { + let stdout = ""; + const emit = createEmitter(async (chunk) => { + stdout += chunk; + }); + const code = await executeHarness({ input, emit, apiKey: "sk-test", startedAt: Date.now(), signal }, { name: "fake", driver: harness }); + const events = async () => { + await emit.flush(); + return parseEvents(stdout); + }; + return { code, events: await events(), later: events }; +} + +describe("kernel", { timeout: 15_000 }, () => { + // Assertions run outside the driver: once the outcome is decided, the kernel ignores how the driver settles. + it("starts with the driver's version and model, and ends with the accepted submission and the model requests made", async () => { + const seen: unknown[] = []; + const { code, events } = await execute( + driver(async (ctx) => { + seen.push(ctx.requestModel()); + ctx.report.toolCall("c1", ctx.submit.name, BAD); + ctx.report.toolResult("c1", ctx.submit.name, ctx.submit.call(BAD).text, true); + seen.push(ctx.requestModel()); + ctx.report.toolCall("c2", ctx.submit.name, GOOD); + const out = ctx.submit.call(GOOD); + ctx.report.toolResult("c2", ctx.submit.name, out.text, out.isError); + seen.push(ctx.submit.call({ ...GOOD, summary: "Another." }), ctx.requestModel()); + }), + ); + expect(seen).toEqual([true, true, { isError: false, text: expect.stringContaining("Already submitted") }, false]); + expect(events[0]).toMatchObject({ type: "started", agent_id: "test-agent", harness: "fake", harness_version: "1.2.3", model: "fake-model" }); + const [rejected, accepted] = toolResults(events); + expect(rejected).toMatchObject({ is_error: true, output_preview: expect.stringContaining("attempt 1 of 3") }); + expect(accepted).toMatchObject({ is_error: false }); + expect(terminal(events)).toEqual(expect.objectContaining({ type: "result", output: GOOD, turns: 2 })); + expect(code).toBe(0); + }); + + it("fails with invalid_input, without starting, when the driver can't run the configuration", async () => { + const invalid = await execute({ + open: () => { + throw new InvalidInput("unknown model"); + }, + }); + expect(invalid.events).toEqual([expect.objectContaining({ type: "error", code: "invalid_input", message: "unknown model" })]); + expect(invalid.code).toBe(2); + + const broken = await execute({ open: () => Promise.reject(new Error("no library")) }); + expect(broken.events).toEqual([expect.objectContaining({ type: "error", code: "agent_error", message: "no library" })]); + expect(broken.code).toBe(1); + }); + + it("ends with invalid_submission and stops the driver after the last rejected submission", async () => { + const seen: unknown[] = []; + const { code, events } = await execute( + driver(async (ctx) => { + for (let i = 0; i < 3; i++) ctx.submit.call(BAD); + seen.push(ctx.signal.aborted, ctx.requestModel(), ctx.submit.call(GOOD)); + }), + ); + expect(seen).toEqual([true, false, { text: "The run is over.", isError: true }]); + expect(terminal(events)).toMatchObject({ type: "error", code: "invalid_submission", message: expect.stringContaining("rejected 3 times") }); + expect(code).toBe(1); + }); + + it("refuses the model request past max_turns and stops the driver", async () => { + const seen: unknown[] = []; + const { code, events } = await execute( + driver(async (ctx) => { + seen.push(ctx.requestModel(), ctx.requestModel(), ctx.requestModel(), ctx.signal.aborted); + }), + makeInput(undefined, { max_turns: 2 }), + ); + expect(seen).toEqual([true, true, false, true]); + expect(terminal(events)).toMatchObject({ type: "error", code: "max_turns", message: expect.stringContaining("turn limit (2)") }); + expect(code).toBe(1); + }); + + it("reports no_submission when the model stops, and the driver's failure when the library fails", async () => { + const stopped = await execute(driver(async () => {})); + expect(terminal(stopped.events)).toMatchObject({ type: "error", code: "no_submission", message: "the model stopped without calling submit" }); + + const failed = await execute(driver(() => Promise.reject(new Error("boom")))); + expect(terminal(failed.events)).toMatchObject({ type: "error", code: "agent_error", message: "boom" }); + expect(failed.code).toBe(1); + }); + + it("keeps the first outcome: a failure after an accepted submission still ends with the result", async () => { + const { code, events } = await execute( + driver(async (ctx) => { + ctx.submit.call(GOOD); + throw new Error("late failure"); + }), + ); + expect(terminal(events)).toMatchObject({ type: "result", output: GOOD }); + expect(code).toBe(0); + }); + + it("times out with the budget left after the safety margin, and refuses a later submission", async () => { + let late: unknown; + const { code, events } = await execute( + driver(async (ctx) => { + await aborted(ctx); + late = ctx.submit.call(GOOD); + }), + makeInput(undefined, { timeout_ms: 1000 }), + ); + expect(late).toEqual({ text: "The run is over.", isError: true }); + expect(terminal(events)).toMatchObject({ type: "error", code: "timeout", message: "the run exceeded its 900 ms budget" }); + expect(code).toBe(3); + }); + + it("keeps an accepted submission when the timeout stops a driver that kept going", async () => { + const { code, events } = await execute( + driver(async (ctx) => { + ctx.submit.call(GOOD); + await aborted(ctx); + }), + makeInput(undefined, { timeout_ms: 1000 }), + ); + expect(terminal(events)).toMatchObject({ type: "result", output: GOOD }); + expect(code).toBe(0); + }); + + it("times out a driver that is still opening, without starting", async () => { + const { code, events } = await execute({ open: (ctx) => aborted(ctx).then(() => ({ version: "1", model: "m", run: async () => {} })) }, makeInput(undefined, { timeout_ms: 1000 })); + expect(events).toEqual([expect.objectContaining({ type: "error", code: "timeout" })]); + expect(code).toBe(3); + }); + + it("cancels on the run's signal, mid-run or before it starts", async () => { + const controller = new AbortController(); + const mid = await execute( + driver(async (ctx) => { + controller.abort(); + await aborted(ctx); + }), + makeInput(undefined), + controller.signal, + ); + expect(terminal(mid.events)).toMatchObject({ type: "error", code: "agent_error", message: "the run was cancelled" }); + + const before = await execute(driver(async () => {}), makeInput(undefined), AbortSignal.abort()); + expect(before.events).toEqual([expect.objectContaining({ type: "error", code: "agent_error", message: "the run was cancelled" })]); + }); + + it("reports the outcome after a grace period when the driver ignores the stop", async () => { + const started = Date.now(); + const { events } = await execute( + driver(async (ctx) => { + ctx.requestModel(); + await new Promise(() => {}); + }), + makeInput(undefined, { max_turns: 1, timeout_ms: 1000 }), + ); + expect(terminal(events)).toMatchObject({ type: "error", code: "timeout" }); + expect(Date.now() - started).toBeGreaterThanOrEqual(5_000); + }); + + it("gives the driver exactly the input's tools, and refuses them once the run is over", async () => { + let names: string[] = []; + let afterEnd: unknown; + await execute( + driver(async (ctx) => { + names = ctx.tools.map((t) => t.name); + ctx.submit.call(GOOD); + afterEnd = await ctx.tools[0]?.execute({}); + }), + makeInput(makeWorkspace(), { tools: ["ls", "read"] }), + ); + expect(names).toEqual(["ls", "read"]); + expect(afterEnd).toEqual({ text: "The run is over.", isError: true }); + + let none: number | undefined; + await execute( + driver(async (ctx) => { + none = ctx.tools.length; + }), + ); + expect(none).toBe(0); + }); + + it("emits a tool result only for an announced call, truncated, and nothing after the outcome", async () => { + let captured: DriverContext | undefined; + const run = await execute( + driver(async (ctx) => { + captured = ctx; + ctx.report.toolResult("never-announced", "ls", "text", false); + ctx.report.toolCall("c1", "ls", {}); + ctx.report.toolResult("c1", "ls", "x".repeat(5000), false); + ctx.report.toolResult("c1", "ls", "duplicate", false); + ctx.submit.call(GOOD); + }), + ); + captured?.report.text("after the outcome"); + const events = await run.later(); + const results = toolResults(events); + expect(results).toHaveLength(1); + expect(results[0]?.tool_call_id).toBe("c1"); + expect(results[0]?.output_preview).toHaveLength(2000); + expect(results[0]?.output_preview.endsWith("…[truncated]")).toBe(true); + expect(events.some((e) => e.type === "assistant_text")).toBe(false); + expect(terminal(events)).toMatchObject({ type: "result" }); + }); +}); diff --git a/tests/runner.test.ts b/tests/runner.test.ts index b59cc80..e97608d 100644 --- a/tests/runner.test.ts +++ b/tests/runner.test.ts @@ -2,17 +2,20 @@ import { existsSync, readFileSync } from "node:fs"; import { rm } from "node:fs/promises"; import { dirname, join } from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { rejectReason } from "../app/runner/shell-policy.js"; +import { PROTOCOL_VERSION } from "../app/runner/core/protocol.js"; +import { rejectReason } from "../app/runner/tools/shell-policy.js"; import { buildRunner, call, GOOD, + HARNESSES, makeInput, makeWorkspace, parseEvents, runCli, - runWithTestStub, + runContract, SUBMIT, + TEST_KEY, terminal, toolResults, } from "./helpers/runner.js"; @@ -23,59 +26,57 @@ beforeAll(async () => { runnerDir = dirname(await buildRunner()); }, 120_000); -afterAll(() => rm(runnerDir, { recursive: true, force: true })); +afterAll(async () => { + await rm(runnerDir, { recursive: true, force: true }); +}); const ENV_KEYS = ["ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "OPENAI_API_KEY", "RENDER_API_KEY"]; afterEach(() => { for (const name of ENV_KEYS) delete process.env[name]; }); +// The bundled runner runs only the agents bundled into it; triage is one, on Pi. describe("bundled runner (runner.mjs)", { timeout: 60_000 }, () => { - it("--test-stub lists the working directory and submits a value built from the output schema", () => { - const ws = makeWorkspace(); - const { code, stdout, stderr } = runCli(["--test-stub"], JSON.stringify(makeInput(ws))); - const events = parseEvents(stdout); - expect(events[0]).toEqual({ - type: "started", - at: expect.any(String), - agent_id: "test-agent", - model: "test-stub/test-stub", - pi_version: "0.83.0", - }); - expect(toolResults(events).map((r) => [r.tool, r.is_error])).toEqual([ - ["ls", false], - [SUBMIT, false], + it("--self-test loads the bundle and every agent definition, and reports each agent's harness", () => { + const { code, stdout } = runCli(["--self-test"], ""); + const report = JSON.parse(stdout.trim()); + expect(report).toMatchObject({ ok: true, protocol_version: PROTOCOL_VERSION }); + expect(report.agents).toEqual([ + { id: "triage", harness: "pi", label: "anthropic/claude-sonnet-5" }, + { id: "spec-writer", harness: "pi", label: "anthropic/claude-opus-5" }, + { id: "implementer", harness: "pi", label: "anthropic/claude-opus-5" }, + { id: "pr-reviewer", harness: "pi", label: "anthropic/claude-sonnet-5" }, ]); - expect(toolResults(events)[0]?.output_preview.split("\n")).toEqual([".git/", "README.md", "src/"]); - expect(terminal(events)).toMatchObject({ type: "result", output: { summary: "[test-stub]", files: [] }, turns: 2 }); - expect(code, stderr).toBe(0); - }); - - it("runs an agent with no tools and no cwd: the model gets only submit", () => { - const { code, stdout } = runCli(["--test-stub"], JSON.stringify(makeInput(undefined))); - const events = parseEvents(stdout); - expect(toolResults(events).map((r) => r.tool)).toEqual([SUBMIT]); - expect(terminal(events).type).toBe("result"); expect(code).toBe(0); }); - it("--key-stdin takes the key from the first line of stdin and the input from the rest, and never echoes the key", () => { - const ws = makeWorkspace(); - const key = "sk-ant-test-stdin-0123456789abcdef"; - const { code, stdout, stderr } = runCli(["--test-stub", "--key-stdin"], ` ${key} \r\n${JSON.stringify(makeInput(ws))}`); - expect(terminal(parseEvents(stdout))).toMatchObject({ type: "result", output: { summary: "[test-stub]" } }); - expect(stdout + stderr).not.toContain(key); - expect(code).toBe(0); + it("rejects an agent the bundle doesn't define, input from another protocol version, and a run without a key", () => { + const keyed = (input: unknown) => `${TEST_KEY}\n${JSON.stringify(input)}`; + const unknown = runCli(["--key-stdin"], keyed(makeInput(undefined, { agent_id: "nope" }))); + expect(terminal(parseEvents(unknown.stdout))).toMatchObject({ code: "invalid_input", message: expect.stringContaining('unknown agent "nope"') }); + expect(unknown.code).toBe(2); + + const old = runCli(["--key-stdin"], keyed({ ...makeInput(undefined, { agent_id: "triage" }), protocol_version: 1 })); + expect(terminal(parseEvents(old.stdout))).toMatchObject({ + code: "invalid_input", + message: expect.stringContaining(`speaks protocol ${PROTOCOL_VERSION}, but the input is protocol 1`), + }); + + const keyless = runCli([], JSON.stringify(makeInput(undefined, { agent_id: "triage" }))); + expect(terminal(parseEvents(keyless.stdout))).toMatchObject({ code: "invalid_input", message: expect.stringContaining("no model key") }); + + const empty = runCli(["--key-stdin"], `\n${JSON.stringify(makeInput(undefined, { agent_id: "triage" }))}`); + expect(terminal(parseEvents(empty.stdout))).toMatchObject({ code: "invalid_input", message: "--key-stdin: the first line of stdin is empty" }); }); }); -describe("submit", { timeout: 30_000 }, () => { - it("returns every problem to the model, and accepts the corrected submission", async () => { +// The runner contract every harness must honor. A new harness joins HARNESSES in tests/helpers/runner.ts. +describe.each(HARNESSES)("$name harness conformance", { timeout: 30_000 }, ({ make }) => { + it("returns every submit problem to the model, and accepts the corrected submission", async () => { const ws = makeWorkspace(); - const { code, events } = await runWithTestStub(makeInput(ws), () => [ - call(SUBMIT, { summary: "x", files: "README.md", confidence: "maybe", extra: 1 }), - call(SUBMIT, GOOD), - ]); + const harness = make([call(SUBMIT, { summary: "x", files: "README.md", confidence: "maybe", extra: 1 }), call(SUBMIT, GOOD)]); + const { code, events } = await runContract(harness, makeInput(ws)); + expect(events[0]).toMatchObject({ type: "started", harness: harness.name, harness_version: expect.stringMatching(/\S/) }); const [first, second] = toolResults(events); expect(first).toMatchObject({ tool: SUBMIT, is_error: true }); expect(first?.output_preview).toContain("attempt 1 of 3"); @@ -85,10 +86,10 @@ describe("submit", { timeout: 30_000 }, () => { expect(code).toBe(0); }); - it("stops after the last rejected attempt and says what was wrong with it", async () => { + it("stops after the last rejected submission and says what was wrong with it", async () => { const ws = makeWorkspace(); const bad = call(SUBMIT, { summary: "ok summary", files: "README.md" }); - const { code, events } = await runWithTestStub(makeInput(ws), () => [bad, bad, bad, call(SUBMIT, GOOD)]); + const { code, events } = await runContract(make([bad, bad, bad, call(SUBMIT, GOOD)]), makeInput(ws)); const end = terminal(events); expect(end).toMatchObject({ type: "error", code: "invalid_submission" }); const message = end.type === "error" ? end.message : ""; @@ -97,25 +98,21 @@ describe("submit", { timeout: 30_000 }, () => { expect(message).not.toContain("ok summary"); expect(code).toBe(1); }); -}); -describe("run limits", { timeout: 30_000 }, () => { it("stops after max_turns without a submission", async () => { const ws = makeWorkspace(); - const { code, events } = await runWithTestStub(makeInput(ws, { max_turns: 3 }), () => [ - ...Array.from({ length: 5 }, () => call("ls", {})), - call(SUBMIT, GOOD), - ]); + const { code, events } = await runContract( + make([...Array.from({ length: 5 }, () => call("ls", {})), call(SUBMIT, GOOD)]), + makeInput(ws, { max_turns: 3 }), + ); expect(terminal(events)).toMatchObject({ type: "error", code: "max_turns", message: expect.stringContaining("turn limit (3)") }); expect(toolResults(events)).toHaveLength(3); expect(code).toBe(1); }); -}); -// Tests that inspect a command's environment have it write to a file: Pi stops reading a command's stdout -// 100 ms after the process exits, so under load the tool result can come back empty. -describe("credentials", { timeout: 30_000 }, () => { - it("takes the key from stdin and keeps every secret out of the tool env", async () => { + // Tests that inspect a command's environment have it write to a file: Pi stops reading a command's stdout + // 100 ms after the process exits, so under load the tool result can come back empty. + it("takes the key from stdin and keeps every secret out of the tool env and the output", async () => { const ws = makeWorkspace(); const stdinKey = "sk-ant-test-STDINKEY-0123456789"; process.env.ANTHROPIC_API_KEY = "sk-ant-test-ENVKEY-9876543210"; @@ -123,11 +120,8 @@ describe("credentials", { timeout: 30_000 }, () => { process.env.OPENAI_API_KEY = "sk-openai-not-for-the-model"; process.env.RENDER_API_KEY = "rnd-not-for-the-model"; - const { code, stdout, events } = await runWithTestStub( - `${stdinKey}\n${JSON.stringify(makeInput(ws, { tools: ["bash", "write"] }))}`, - () => [call("bash", { command: "env > $HOME/env.txt" }), call(SUBMIT, GOOD)], - ["--key-stdin"], - ); + const harness = make([call("bash", { command: "env > $HOME/env.txt" }), call(SUBMIT, GOOD)]); + const { code, stdout, events } = await runContract(harness, makeInput(ws, { tools: ["bash", "write"] }), stdinKey); expect(toolResults(events)[0]?.is_error).toBe(false); const env = readFileSync(join(ws.scratchDir, "env.txt"), "utf8"); @@ -138,29 +132,59 @@ describe("credentials", { timeout: 30_000 }, () => { expect(process.env.ANTHROPIC_API_KEY).toBeUndefined(); expect(code).toBe(0); }); -}); -describe("tools", { timeout: 30_000 }, () => { - it("read-only agents: bash rejects git push and rm, and runs read-only commands", async () => { + it("read-only agents: bash rejects git push, rm, and redirects into the checkout, and runs read-only commands", async () => { const ws = makeWorkspace(); - const { events } = await runWithTestStub(makeInput(ws), () => [ - call("bash", { command: "git push origin main" }), - call("bash", { command: "rm README.md" }), - call("bash", { command: "echo hi > notes.txt" }), - call("bash", { command: "git log -1 --format=%H > $HOME/head.txt" }), - call(SUBMIT, GOOD), - ]); - const [push, rmCall, redirect, log] = toolResults(events); + const { events } = await runContract( + make([ + call("bash", { command: "git push origin main" }), + call("bash", { command: "rm README.md" }), + call("bash", { command: "echo hi > notes.txt" }), + call("bash", { command: "git log -1 --format=%H > $HOME/head.txt" }), + call("write", { path: "notes.txt", content: "not allowed" }), + call(SUBMIT, GOOD), + ]), + makeInput(ws), + ); + const [push, rmCall, redirect, log, write] = toolResults(events); expect(push).toMatchObject({ is_error: true, output_preview: expect.stringContaining("git push is not allowed") }); expect(rmCall).toMatchObject({ is_error: true, output_preview: expect.stringContaining("rm is only allowed on files in the scratch directory") }); expect(redirect).toMatchObject({ is_error: true, output_preview: expect.stringContaining("redirecting output to notes.txt") }); expect(log).toMatchObject({ is_error: false }); + expect(write).toMatchObject({ tool: "write", is_error: true }); expect(readFileSync(join(ws.scratchDir, "head.txt"), "utf8").trim()).toBe(ws.commit); expect(existsSync(join(ws.cwd, "README.md"))).toBe(true); expect(existsSync(join(ws.cwd, "notes.txt"))).toBe(false); }); - it("the read-only shell policy lets inspection and test commands through and steers away from obvious writes", () => { + it("agents with edit and write create and edit files in cwd, and bash is unrestricted", async () => { + const ws = makeWorkspace(); + const { events } = await runContract( + make([ + call("write", { path: "src/new.ts", content: "export const added = true;\n" }), + call("edit", { path: "src/app.ts", edits: [{ oldText: "41", newText: "42" }] }), + call("bash", { command: "rm README.md && git status --porcelain > $HOME/status.txt" }), + call("read", { path: "src/app.ts" }), + call(SUBMIT, GOOD), + ]), + makeInput(ws, { tools: ["read", "bash", "edit", "write"] }), + ); + expect(toolResults(events).map((r) => [r.tool, r.is_error])).toEqual([ + ["write", false], + ["edit", false], + ["bash", false], + ["read", false], + [SUBMIT, false], + ]); + expect(readFileSync(join(ws.cwd, "src/new.ts"), "utf8")).toBe("export const added = true;\n"); + expect(readFileSync(join(ws.cwd, "src/app.ts"), "utf8")).toBe("export const answer = 42;\n"); + expect(existsSync(join(ws.cwd, "README.md"))).toBe(false); + expect(readFileSync(join(ws.scratchDir, "status.txt"), "utf8")).toContain("D README.md"); + }); +}); + +describe("read-only shell policy", () => { + it("lets inspection and test commands through and steers away from obvious writes", () => { const scratch = "/factory/scratch"; const allowed = [ "rg -n 'a|b; c' src", @@ -185,26 +209,4 @@ describe("tools", { timeout: 30_000 }, () => { for (const command of allowed) expect(rejectReason(command, scratch), command).toBeUndefined(); for (const command of rejected) expect(rejectReason(command, scratch), command).toBeDefined(); }); - - it("agents with edit and write create and edit files in cwd, and bash is unrestricted", async () => { - const ws = makeWorkspace(); - const { events } = await runWithTestStub(makeInput(ws, { tools: ["read", "bash", "edit", "write"] }), () => [ - call("write", { path: "src/new.ts", content: "export const added = true;\n" }), - call("edit", { path: "src/app.ts", edits: [{ oldText: "41", newText: "42" }] }), - call("bash", { command: "rm README.md && git status --porcelain > $HOME/status.txt" }), - call("read", { path: "src/app.ts" }), - call(SUBMIT, GOOD), - ]); - expect(toolResults(events).map((r) => [r.tool, r.is_error])).toEqual([ - ["write", false], - ["edit", false], - ["bash", false], - ["read", false], - [SUBMIT, false], - ]); - expect(readFileSync(join(ws.cwd, "src/new.ts"), "utf8")).toBe("export const added = true;\n"); - expect(readFileSync(join(ws.cwd, "src/app.ts"), "utf8")).toBe("export const answer = 42;\n"); - expect(existsSync(join(ws.cwd, "README.md"))).toBe(false); - expect(readFileSync(join(ws.scratchDir, "status.txt"), "utf8")).toContain("D README.md"); - }); }); diff --git a/tests/sandbox-render.test.ts b/tests/sandbox-render.test.ts index 2b67b07..bc405a4 100644 --- a/tests/sandbox-render.test.ts +++ b/tests/sandbox-render.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { SandboxSnapshotNotFoundError } from "@renderinc/sdk"; @@ -6,8 +7,27 @@ import { list as listTar } from "tar"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { config, type RepoContext } from "../app/config/index.js"; import { buildSnapshot, type OpenSandboxDeps, openSandbox, type SandboxApi, SandboxClient, type SnapshotManifest } from "../app/sandbox/index.js"; +import { runnerPayload } from "../app/sandbox/snapshot.js"; + +/** The files in a gzipped tar, by path. */ +async function untar(data: Buffer): Promise> { + const files = new Map(); + await pipeline( + Readable.from(data), + listTar({ + onReadEntry: (e) => { + const chunks: Buffer[] = []; + e.on("data", (c: Buffer) => chunks.push(c)); + e.on("end", () => files.set(e.path.replace(/^\.\//, ""), Buffer.concat(chunks).toString("utf8"))); + }, + }), + ); + return files; +} +const FAKE_BUNDLE = "// fake runner bundle\n"; vi.mock("../app/runner/bundle.js", () => ({ bundleRunner: async () => "// fake runner bundle\n" })); +const sha256 = (text: string) => createHash("sha256").update(text).digest("hex"); const TOKEN = "ghs_factoryReadToken0000000000000000000"; const BASE_SHA = "1".repeat(40); @@ -178,6 +198,7 @@ describe("buildSnapshot", () => { const tokenUpload = fake.index((c) => c.op === "upload" && c.path === "/tmp/factory-stage/creds/github.com"); const order = [ fake.execIndex("install-tools.sh"), + fake.index((c) => c.op === "upload" && c.path === "/opt/factory/app/env"), fake.execIndex("--self-test"), tokenUpload, fake.execIndex(" clone "), @@ -231,17 +252,7 @@ describe("buildSnapshot", () => { contentType?: string; }; expect(stage.contentType).toBeUndefined(); - const files = new Map(); - await pipeline( - Readable.from(stage.data), - listTar({ - onReadEntry: (e) => { - const chunks: Buffer[] = []; - e.on("data", (c: Buffer) => chunks.push(c)); - e.on("end", () => files.set(e.path, Buffer.concat(chunks).toString("utf8"))); - }, - }), - ); + const files = await untar(stage.data); expect([...files.keys()].filter((p) => !p.endsWith("/")).sort()).toEqual([ "as-runner.sh", "build/git-credential-staged.sh", @@ -250,10 +261,12 @@ describe("buildSnapshot", () => { "install-tools.sh", "lockfile-hash.sh", "run-agent.sh", + "runner/payload.sha256", "runner/runner.mjs", "verify.sh", ]); expect(files.get("runner/runner.mjs")).toBe("// fake runner bundle\n"); + expect(files.get("runner/payload.sha256")).toBe((await runnerPayload()).sha256); const env = fake.calls.find((c) => c.op === "upload" && c.path === "/opt/factory/app/env") as { data: string }; expect(env.data).toContain("FACTORY_RUN_AS=factory\n"); @@ -346,6 +359,29 @@ describe("openSandbox", () => { expect(fake.calls.at(-1)).toEqual({ op: "terminate", id: "sbx-1" }); }); + it("replaces a stale runner payload, root-owned, before verifying the sandbox", async () => { + await openSandbox(repo(), { kind: "branch", name: "main" }, deps()); + const upload = fake.calls.find((c) => c.op === "upload" && /^\/tmp\/factory-runner-[0-9a-f-]{36}\/payload\.tgz$/.test(c.path)) as { + path: string; + data: Buffer; + }; + const files = await untar(upload.data); + expect(files.get("runner.mjs")).toBe(FAKE_BUNDLE); + const swap = fake.commands().find((c) => c.includes("mv /opt/factory/app/runner.next /opt/factory/app/runner")) as string; + expect(swap).toContain(`= ${sha256(FAKE_BUNDLE)} ]`); + expect(swap).toContain("chown -R root:root /opt/factory/app/runner.next"); + expect(swap).toContain("chmod -R a+rX,go-w /opt/factory/app/runner.next"); + expect(fake.execIndex("mv /opt/factory/app/runner.next")).toBeGreaterThanOrEqual(0); + expect(fake.execIndex("mv /opt/factory/app/runner.next")).toBeLessThan(fake.execIndex("verify.sh")); + }); + + it("leaves a matching runner payload alone", async () => { + const { sha256: current } = await runnerPayload(); + fake.overrides = (_id, command) => (command.startsWith("cat /opt/factory/app/runner/payload.sha256") ? [out(current), exit(0)] : []); + await openSandbox(repo(), { kind: "branch", name: "main" }, deps()); + expect(fake.calls.some((c) => c.op === "upload" && c.path.includes("factory-runner-"))).toBe(false); + }); + it("refuses a sandbox where the runner could not be confirmed non-dumpable", async () => { fake.overrides = (_id, command) => (command.includes("verify.sh") ? [out(info({ runnerNonDumpable: false })), exit(0)] : []); await expect(openSandbox(repo(), { kind: "branch", name: "main" }, deps())).rejects.toThrow("non-dumpable"); diff --git a/tests/scripts-doctor.test.ts b/tests/scripts-doctor.test.ts index 7d3a6ae..1766e30 100644 --- a/tests/scripts-doctor.test.ts +++ b/tests/scripts-doctor.test.ts @@ -105,7 +105,6 @@ async function doctor(overrides: Partial & { world?: World } = {} sleep: async () => {}, services: ["gateway", "workflows"], repos: ["acme/web"], - models: { medium: "anthropic/sonnet", large: "anthropic/opus" }, ...rest, }, out,