diff --git a/AGENTS.md b/AGENTS.md index 25cd84f..8575657 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,9 @@ This file applies to the whole repository. ## Use the right guide +- Agent libraries, models, or harness adapters: + [docs/configuration.md#agent-harnesses-and-models](docs/configuration.md#agent-harnesses-and-models) + and [docs/runbooks/extend.md](docs/runbooks/extend.md#add-an-agent-harness) - First deployment: [docs/runbooks/setup.md](docs/runbooks/setup.md) - Repositories, labels, models, permissions, limits, or snapshots: @@ -31,8 +34,10 @@ database. Two dependency rules are mandatory: -1. `app/runner/` imports nothing from the rest of `app/`. It is bundled into - a self-contained Sandbox program. +1. The Sandbox runner bundle (`app/runner/`, `app/harness/`, the agent + definitions in `app/agents/registry.ts`, and `app/schemas/`) imports no + host-only code: GitHub, Sandboxes, Workflows, claims, events, or config. + `tests/agents-catalog.test.ts` enforces it. 2. `app/gateway/` does not import GitHub authentication, Sandboxes, agents, or workflows. The gateway holds no GitHub credential and runs no model. @@ -93,8 +98,13 @@ strict output schemas. credentials or a model key to the gateway. - App permission, event, or label: update `scripts/lib/requirements.ts`, setup behavior, and doctor tests. -- `app/runner/` or `sandbox/`: refresh every repository snapshot after - deployment. +- Agent harness: declare the env var of its key as the adapter's + `credential`; setup and `doctor` derive the key list. It must give the + model only the factory's tools; add it to `HARNESSES` in + `tests/helpers/runner.ts` if it can run the conformance suite. +- `sandbox/`: refresh every repository snapshot after deployment. Changes to + `app/runner/`, `app/harness/`, or agents reach Sandboxes with the Workflows + deploy, because each Sandbox replaces a stale runner bundle when it opens. ## Verify diff --git a/README.md b/README.md index 8bf87f0..7cd4241 100644 --- a/README.md +++ b/README.md @@ -58,15 +58,16 @@ questions, the workflow stops and continues when a person replies. The built-in agents and tasks are examples of the template's extension model: -- Add or replace an agent with a Markdown prompt, a model tier, a bounded Zod - output schema, and the minimum tools it needs. +- Add or replace an agent with a Markdown prompt, a harness (agent library and + model), a bounded Zod output schema, and the minimum tools it needs. +- Run agents on another agent library by adding a harness adapter. - Define a GitHub task for work attached to an issue or pull request. - Add a plain Workflows task for maintenance or other work without a GitHub claim. - Route new GitHub events, labels, or commands through the authenticated gateway. -- Change repositories, verification commands, models, permissions, and limits - in `factory.config.ts`. +- Change repositories, verification commands, permissions, and limits in + `factory.config.ts`, and agent models in `app/agents/models.ts`. The Sandbox runner, claim lifecycle, GitHub client, formatting helpers, and publishing boundary are reusable building blocks. Start with the diff --git a/app/agents/define.ts b/app/agents/define.ts index 64d10c4..aa56637 100644 --- a/app/agents/define.ts +++ b/app/agents/define.ts @@ -1,9 +1,9 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { z } from "zod"; -import type { ModelTier } from "../config/index.js"; -import type { ThinkingLevel, ToolName } from "../runner/protocol.js"; -import { outputSchemaProblem } from "../runner/submit.js"; +import type { Harness } from "../harness/types.js"; +import type { ToolName } from "../runner/core/protocol.js"; +import { outputSchemaProblem } from "../runner/core/submit.js"; export type OutputSchema = z.ZodObject; @@ -11,36 +11,51 @@ export interface AgentSpec extends Omit, "prompt"> { /** Absolute path of the prompt file. */ promptPath: string; - /** The prompt file's contents, verbatim: the agent's system prompt. */ - prompt: string; + /** The prompt file's contents, verbatim: the agent's system prompt. Read on first use, on the host. */ + readonly prompt: string; /** `output` as JSON Schema (input side, so fields with defaults are optional): the runner's submit tool takes it. */ outputJsonSchema: Record; } const AGENTS_DIR = new URL("./", import.meta.url); -/** Reads the prompt file once, as a string (no frontmatter, no parsing), and converts the output schema. */ +/** + * Converts the output schema. Agent modules are also bundled into the sandbox runner, where prompt files don't + * exist, so the prompt is read lazily. + */ export function defineAgent( spec: AgentSpec, ): Agent { if (!Number.isInteger(spec.maxTurns) || spec.maxTurns <= 0) throw new Error(`agent ${spec.id}: maxTurns must be a positive integer`); const promptPath = fileURLToPath(new URL(spec.prompt, AGENTS_DIR)); - const prompt = readFileSync(promptPath, "utf8"); - if (!prompt.trim()) throw new Error(`agent ${spec.id}: prompt file ${promptPath} is empty`); const outputJsonSchema = z.toJSONSchema(spec.output, { io: "input" }) as Record; const problem = outputSchemaProblem(outputJsonSchema); if (problem) throw new Error(`agent ${spec.id}: ${problem}`); - return { ...spec, promptPath, prompt, outputJsonSchema }; + let prompt: string | undefined; + const { prompt: _relative, ...rest } = spec; + return { + ...rest, + promptPath, + outputJsonSchema, + get prompt() { + if (prompt === undefined) { + const text = readFileSync(promptPath, "utf8"); + if (!text.trim()) throw new Error(`agent ${spec.id}: prompt file ${promptPath} is empty`); + prompt = text; + } + return prompt; + }, + }; } diff --git a/app/agents/implementer/index.ts b/app/agents/implementer/index.ts index d72df1f..f45be73 100644 --- a/app/agents/implementer/index.ts +++ b/app/agents/implementer/index.ts @@ -1,5 +1,6 @@ import { Implementation } from "../../schemas/index.js"; import { defineAgent } from "../define.js"; +import { large } from "../models.js"; /** * Implements the approved spec in the sandbox's working tree and runs the verify commands itself. A repair run @@ -8,7 +9,7 @@ import { defineAgent } from "../define.js"; export const implementer = defineAgent({ id: "implementer", prompt: "implementer/prompt.md", - model: "large", + harness: large, tools: ["read", "ls", "bash", "edit", "write"], output: Implementation, maxTurns: 150, diff --git a/app/agents/index.ts b/app/agents/index.ts index 1175ef9..17b1af2 100644 --- a/app/agents/index.ts +++ b/app/agents/index.ts @@ -1,6 +1,8 @@ export { type Agent, type AgentSpec, defineAgent, type OutputSchema } from "./define.js"; export { implementer } from "./implementer/index.js"; +export { large, medium } from "./models.js"; export { prReviewer } from "./pr-reviewer/index.js"; +export { AGENTS } from "./registry.js"; export { AgentError, type AgentErrorCode, type AgentRun, type RunAgentOptions, runAgent } from "./run.js"; export { specWriter } from "./spec-writer/index.js"; export { triage } from "./triage/index.js"; diff --git a/app/agents/models.ts b/app/agents/models.ts new file mode 100644 index 0000000..2104767 --- /dev/null +++ b/app/agents/models.ts @@ -0,0 +1,10 @@ +import { getBuiltinModel } from "@earendil-works/pi-ai/providers/all"; +import { pi } from "../harness/index.js"; + +/** + * The harnesses the built-in agents use. Change an agent's library or model here, or give an agent its own + * harness in its definition. Each value carries its library's own configuration, so a typo in a model ID is a + * type error. + */ +export const medium = pi({ model: getBuiltinModel("anthropic", "claude-sonnet-5") }); +export const large = pi({ model: getBuiltinModel("anthropic", "claude-opus-5") }); diff --git a/app/agents/pr-reviewer/index.ts b/app/agents/pr-reviewer/index.ts index e342c15..8ddaa72 100644 --- a/app/agents/pr-reviewer/index.ts +++ b/app/agents/pr-reviewer/index.ts @@ -1,11 +1,12 @@ import { Review } from "../../schemas/index.js"; import { defineAgent } from "../define.js"; +import { medium } from "../models.js"; /** Reviews a pull request across correctness, security, performance, and tests. Read-only, in the PR Reviewer's sandbox. */ export const prReviewer = defineAgent({ id: "pr-reviewer", prompt: "pr-reviewer/prompt.md", - model: "medium", + harness: medium, tools: ["read", "ls", "bash"], output: Review, maxTurns: 100, diff --git a/app/agents/registry.ts b/app/agents/registry.ts new file mode 100644 index 0000000..675b092 --- /dev/null +++ b/app/agents/registry.ts @@ -0,0 +1,11 @@ +import type { Agent } from "./define.js"; +import { implementer } from "./implementer/index.js"; +import { prReviewer } from "./pr-reviewer/index.js"; +import { specWriter } from "./spec-writer/index.js"; +import { triage } from "./triage/index.js"; + +/** + * Every agent the factory runs. The sandbox runner is bundled from this list and runs an agent only if it is here. + * Bundled modules must not import host-only code (GitHub, Sandbox, Workflows, claims, events, config). + */ +export const AGENTS: readonly Agent[] = [triage, specWriter, implementer, prReviewer]; diff --git a/app/agents/run.ts b/app/agents/run.ts index 801f7af..c36a3c2 100644 --- a/app/agents/run.ts +++ b/app/agents/run.ts @@ -1,10 +1,8 @@ import { randomUUID } from "node:crypto"; import type { z } from "zod"; -import { modelFor } from "../config/index.js"; import { emit } from "../events/index.js"; -import { providerApiKey } from "../runner/keys.js"; -import { type RunnerErrorCode, RunnerEvent, type RunnerInput, type ToolName, type Usage } from "../runner/protocol.js"; -import { errorMessage } from "../runner/session.js"; +import { errorMessage } from "../runner/core/emit.js"; +import { PROTOCOL_VERSION, type RunnerErrorCode, RunnerEvent, type RunnerInput, type ToolName, type Usage } from "../runner/core/protocol.js"; import { APP_DIR, REPO_DIR, SCRATCH_DIR, type Sandbox } from "../sandbox/types.js"; import type { Agent, OutputSchema } from "./define.js"; @@ -55,7 +53,7 @@ export async function runAgent> { const { sandbox, ...base } = opts; - const { provider, model } = modelFor(agent.model); + const { harness } = agent; const timeoutMs = base.timeoutMs ?? DEFAULT_TIMEOUT_MS; const finish = (outcome: string, summary: Summary, message?: string) => emit({ @@ -66,7 +64,9 @@ export async function runAgent new AgentError(agent.id, code, text); try { const runnerInput: RunnerInput = { + protocol_version: PROTOCOL_VERSION, trace_id: base.traceId, agent_id: agent.id, system_prompt: agent.prompt, message, - model_provider: provider, - model, - thinking_level: agent.thinking ?? "medium", tools: [...agent.tools], output_schema: agent.outputJsonSchema, max_turns: agent.maxTurns, @@ -94,8 +92,8 @@ export async function runAgent; + /** From the runner's `started` event: what actually ran, which the host can't always know. */ + harnessVersion?: string; + model?: string; /** Lines that aren't valid RunnerEvents. */ invalidLines: number; } @@ -197,7 +198,10 @@ function summarize(jsonl: string): Summary { continue; } if (summary.terminal) continue; - if (event.type === "usage") { + if (event.type === "started") { + summary.harnessVersion = event.harness_version; + summary.model = event.model; + } else if (event.type === "usage") { const u = summary.usage; u.input_tokens += event.usage.input_tokens; u.output_tokens += event.usage.output_tokens; diff --git a/app/agents/spec-writer/index.ts b/app/agents/spec-writer/index.ts index b86ce81..db33a72 100644 --- a/app/agents/spec-writer/index.ts +++ b/app/agents/spec-writer/index.ts @@ -1,11 +1,12 @@ import { SpecDraft } from "../../schemas/index.js"; import { defineAgent } from "../define.js"; +import { large } from "../models.js"; /** Researches the repository (read-only) and drafts or revises the spec. Runs in triage's sandbox after a "draft" decision. */ export const specWriter = defineAgent({ id: "spec-writer", prompt: "spec-writer/prompt.md", - model: "large", + harness: large, tools: ["read", "ls", "bash"], output: SpecDraft, maxTurns: 60, diff --git a/app/agents/triage/index.ts b/app/agents/triage/index.ts index cea6321..e36a67b 100644 --- a/app/agents/triage/index.ts +++ b/app/agents/triage/index.ts @@ -1,11 +1,12 @@ import { TriageDecision } from "../../schemas/index.js"; import { defineAgent } from "../define.js"; +import { medium } from "../models.js"; /** Reads the repository (read-only) and decides whether the issue is ready for a spec. Runs in a sandbox at the base branch. */ export const triage = defineAgent({ id: "triage", prompt: "triage/prompt.md", - model: "medium", + harness: medium, tools: ["read", "ls", "bash"], output: TriageDecision, maxTurns: 40, diff --git a/app/config/index.ts b/app/config/index.ts index 7bef46c..0d6a3b4 100644 --- a/app/config/index.ts +++ b/app/config/index.ts @@ -1,6 +1,6 @@ -import config, { type ModelTier, type RepoConfig } from "../../factory.config.js"; +import config, { type RepoConfig } from "../../factory.config.js"; -export { config, type ModelTier, type RepoConfig }; +export { config, type RepoConfig }; /** One repository a run acts on, resolved from the webhook that started it. */ export interface RepoContext { @@ -29,13 +29,6 @@ export function requireRepo(fullName: string, installationId: number): RepoConte return repo; } -export function modelFor(tier: ModelTier): { provider: string; model: string } { - const id = config.models[tier]; - const slash = id.indexOf("/"); - if (slash <= 0) throw new Error(`models.${tier} must be "provider/model", got "${id}"`); - return { provider: id.slice(0, slash), model: id.slice(slash + 1) }; -} - export function requireEnv(name: string): string { const value = process.env[name]?.trim(); if (!value) throw new Error(`${name} is not set`); diff --git a/app/harness/index.ts b/app/harness/index.ts new file mode 100644 index 0000000..0f8fd74 --- /dev/null +++ b/app/harness/index.ts @@ -0,0 +1,3 @@ +export type { HarnessRun, HarnessRuntime, HarnessSelfTest } from "../runner/core/harness.js"; +export { type PiOptions, pi, type ThinkingLevel } from "./pi.js"; +export type { Harness } from "./types.js"; diff --git a/app/harness/pi.ts b/app/harness/pi.ts new file mode 100644 index 0000000..84f9570 --- /dev/null +++ b/app/harness/pi.ts @@ -0,0 +1,37 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; +import { createModels, PI_VERSION, runPiSession, type ThinkingLevel } from "../runner/pi/session.js"; +import type { Harness } from "./types.js"; + +export type { ThinkingLevel }; + +/** Each provider's key env var, for the providers the factory supports through Pi. */ +export const PI_PROVIDER_KEY_ENV: Readonly> = { + anthropic: "ANTHROPIC_API_KEY", + google: "GEMINI_API_KEY", + openai: "OPENAI_API_KEY", +}; + +export interface PiOptions { + /** A model from Pi's catalog, e.g. `getBuiltinModel("anthropic", "claude-opus-5")`. */ + model: Model; + /** Default "medium". */ + thinkingLevel?: ThinkingLevel; +} + +/** Runs an agent on Pi (`@earendil-works/pi-agent-core`) with the factory's tools. */ +export function pi(opts: PiOptions): Harness { + const { provider, id } = opts.model; + const credential = PI_PROVIDER_KEY_ENV[provider]; + if (!credential) throw new Error(`pi: provider "${provider}" is not supported; use one of ${Object.keys(PI_PROVIDER_KEY_ENV).join(", ")}`); + const thinkingLevel = opts.thinkingLevel ?? "medium"; + return { + name: "pi", + label: `${provider}/${id}`, + credential, + run: (run) => runPiSession(run, { model: opts.model, thinkingLevel }), + async selfTest() { + const known = createModels().getModel(provider, id); + return { version: PI_VERSION, problems: known ? [] : [`pi: model ${provider}/${id} is not in the Pi ${PI_VERSION} registry`] }; + }, + }; +} diff --git a/app/harness/types.ts b/app/harness/types.ts new file mode 100644 index 0000000..bf77df5 --- /dev/null +++ b/app/harness/types.ts @@ -0,0 +1,15 @@ +import type { HarnessRuntime } from "../runner/core/harness.js"; + +/** + * What an adapter returns: the runtime the runner calls in the sandbox, plus what the host needs to know before a + * run. Build one with an adapter from this directory and give it to an agent definition. + */ +export interface Harness extends HarnessRuntime { + /** Names the library and model in setup, doctor, and logs before a run. The runner reports the exact model in `started`. */ + readonly label: string; + /** + * The env var on the Workflows service that holds this harness's key. The host sends its value into the sandbox + * for each run; setup and doctor manage it. + */ + readonly credential: string; +} diff --git a/app/runner/bundle.ts b/app/runner/bundle.ts index f1971ae..27efb00 100644 --- a/app/runner/bundle.ts +++ b/app/runner/bundle.ts @@ -1,19 +1,18 @@ +import { relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; const RUNNER_DIR = fileURLToPath(new URL(".", import.meta.url)); +const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url)); -/** - * Bundles app/runner/main.ts and every package it uses into one self-contained ESM file, so the sandbox - * needs only Node to run it: no npm, no node_modules. - */ -export async function bundleRunner(): Promise { +async function buildBundle(): Promise<{ text: string; inputs: string[] }> { const result = await build({ absWorkingDir: RUNNER_DIR, entryPoints: ["main.ts"], outfile: "runner.mjs", write: false, bundle: true, + metafile: true, platform: "node", format: "esm", target: "node22", @@ -23,5 +22,18 @@ export async function bundleRunner(): Promise { }); const [file] = result.outputFiles; if (!file) throw new Error("esbuild produced no output for the runner bundle"); - return file.text; + return { text: file.text, inputs: Object.keys(result.metafile.inputs) }; +} + +/** + * Bundles app/runner/main.ts, the agent registry with every harness it uses, and every package they need into one + * self-contained ESM file, so the sandbox needs only Node to run it: no npm, no node_modules. + */ +export async function bundleRunner(): Promise { + return (await buildBundle()).text; +} + +/** The source files in the bundle, relative to the repository root (tests check the sandbox's import boundary). */ +export async function bundleInputs(): Promise { + return (await buildBundle()).inputs.map((input) => relative(REPO_ROOT, resolve(RUNNER_DIR, input))); } diff --git a/app/runner/core/emit.ts b/app/runner/core/emit.ts new file mode 100644 index 0000000..6cbdbcc --- /dev/null +++ b/app/runner/core/emit.ts @@ -0,0 +1,47 @@ +import type { RunnerEvent } from "./protocol.js"; + +type WithoutAt = E extends unknown ? Omit : never; +export type EventBody = WithoutAt; + +export interface Emitter { + (event: EventBody): void; + readonly terminated: boolean; + /** Replaces `secret` with `[REDACTED]` in every later line. */ + redact(secret: string): void; + /** Resolves once every line written so far has been flushed. */ + flush(): Promise; +} + +/** Writes one RunnerEvent per line. Drops everything after the first terminal (`result`/`error`) event. */ +export function createEmitter(write: (chunk: string) => Promise): Emitter { + const secrets: string[] = []; + let pending = Promise.resolve(); + let terminated = false; + const emit = (event: EventBody) => { + if (terminated) return; + terminated = event.type === "result" || event.type === "error"; + const { type, ...fields } = event; + let line = JSON.stringify({ type, at: new Date().toISOString(), ...fields }); + for (const secret of secrets) line = line.replaceAll(secret, "[REDACTED]"); + pending = pending.then(() => write(`${line}\n`)).catch(() => {}); + }; + return Object.assign(emit, { + get terminated() { + return terminated; + }, + redact(secret: string) { + if (secret) secrets.push(secret); + }, + flush: () => pending, + }); +} + +const PREVIEW_CHARS = 2000; +const TRUNCATED = "…[truncated]"; + +/** A tool result's text as it appears in `tool_result.output_preview`. */ +export function previewText(text: string): string { + return text.length > PREVIEW_CHARS ? text.slice(0, PREVIEW_CHARS - TRUNCATED.length) + TRUNCATED : text; +} + +export const errorMessage = (err: unknown): string => (err instanceof Error ? err.message : String(err)); diff --git a/app/runner/core/harness.ts b/app/runner/core/harness.ts new file mode 100644 index 0000000..40cdbcd --- /dev/null +++ b/app/runner/core/harness.ts @@ -0,0 +1,37 @@ +import type { Emitter } from "./emit.js"; +import type { RunnerInput, TestTurn } from "./protocol.js"; + +/** One agent run, as the runner hands it to the agent's harness. */ +export interface HarnessRun { + /** Already validated with parseInput. */ + input: RunnerInput; + emit: Emitter; + /** The key delivered on stdin; unused with `testScript`. */ + apiKey?: string; + /** Set by `--test-stub`: these turns replace the model. */ + testScript?: TestTurn[]; + /** Epoch ms the `timeout_ms` budget is measured from. */ + startedAt: number; + /** Aborting stops the run with `agent_error`. */ + signal?: AbortSignal; +} + +export interface HarnessSelfTest { + /** The agent library's version, as installed in this sandbox. */ + version: string; + /** Problems that would stop it from running here; empty when it is ready. */ + problems: string[]; +} + +/** + * The part of a harness the runner calls, inside the sandbox. What a harness must do, beyond these signatures, + * is pinned by the conformance suite (tests/runner.test.ts): give the model only the factory's tools for + * `input.tools` (app/runner/tools/) and a submit tool checked by `Submission`, emit `started` with the library + * version and exact model, then exactly one `result` or `error`, and stop at `max_turns` and `timeout_ms`. + */ +export interface HarnessRuntime { + readonly name: string; + /** Runs one agent and returns the exit code. */ + run(run: HarnessRun): Promise; + selfTest(): Promise; +} diff --git a/app/runner/core/input.ts b/app/runner/core/input.ts new file mode 100644 index 0000000..bbb256e --- /dev/null +++ b/app/runner/core/input.ts @@ -0,0 +1,32 @@ +import { statSync } from "node:fs"; +import { PROTOCOL_VERSION, RunnerInput } from "./protocol.js"; +import { outputSchemaProblem } from "./submit.js"; + +/** Validates a RunnerInput, including what Zod can't see: the protocol version, the output schema's root, and the directories. */ +export function parseInput(json: unknown): RunnerInput | string { + const version = (json as { protocol_version?: unknown } | null)?.protocol_version; + if (version !== PROTOCOL_VERSION) { + return `this runner speaks protocol ${PROTOCOL_VERSION}, but the input is protocol ${JSON.stringify(version ?? null)}: the host and the runner bundle are from different versions`; + } + const parsed = RunnerInput.safeParse(json); + if (!parsed.success) { + return `invalid RunnerInput: ${parsed.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ")}`; + } + const input = parsed.data; + const schemaProblem = outputSchemaProblem(input.output_schema); + if (schemaProblem) return schemaProblem; + if (input.tools.length > 0) { + if (!input.cwd) return "cwd is required when tools is non-empty"; + if (!input.scratch_dir) return "scratch_dir is required when tools is non-empty"; + if (!isDirectory(input.cwd)) return `cwd ${input.cwd} is not a directory`; + } + return input; +} + +function isDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} diff --git a/app/runner/core/keys.ts b/app/runner/core/keys.ts new file mode 100644 index 0000000..9aac5d5 --- /dev/null +++ b/app/runner/core/keys.ts @@ -0,0 +1,26 @@ +/** + * Key env vars of every provider a built-in harness can call. Every one of them, plus each bundled harness's own + * key var, is removed before tools run: ANTHROPIC_OAUTH_TOKEN, for one, would take precedence inside Pi. + */ +export const KNOWN_KEY_ENV: readonly string[] = [ + "ANTHROPIC_API_KEY", + "ANTHROPIC_OAUTH_TOKEN", + "ANTHROPIC_AUTH_TOKEN", + "GEMINI_API_KEY", + "OPENAI_API_KEY", + "CURSOR_API_KEY", +]; + +/** + * Removes every key var in KNOWN_KEY_ENV and `extra` from `env`, so none reach the agent's tool subprocesses, and + * returns their values for redaction. The runner's only key is the one on stdin. + */ +export function scrubKeys(env: NodeJS.ProcessEnv, extra: readonly string[] = []): string[] { + const removed: string[] = []; + for (const name of new Set([...KNOWN_KEY_ENV, ...extra])) { + const value = env[name]; + if (value) removed.push(value); + delete env[name]; + } + return removed; +} diff --git a/app/runner/protocol.ts b/app/runner/core/protocol.ts similarity index 72% rename from app/runner/protocol.ts rename to app/runner/core/protocol.ts index f7339a7..dcdf594 100644 --- a/app/runner/protocol.ts +++ b/app/runner/core/protocol.ts @@ -1,27 +1,33 @@ /** * The contract between the host (`runAgent`) and the agent runner, which runs in a sandbox bundled into - * runner.mjs: input on stdin, events on stdout. + * runner.mjs: input on stdin, events on stdout. Every harness speaks it. */ import { z } from "zod"; -const TOOL_NAMES = ["read", "ls", "bash", "edit", "write"] as const; +/** Bumped on any incompatible change; a runner rejects input for another version. */ +export const PROTOCOL_VERSION = 2; + +export const TOOL_NAMES = ["read", "ls", "bash", "edit", "write"] as const; export const ToolName = z.enum(TOOL_NAMES); export type ToolName = z.infer; -const ThinkingLevel = z.enum(["off", "minimal", "low", "medium", "high"]); -export type ThinkingLevel = z.infer; +/** One scripted model turn for `--test-stub` runs: optional text, then tool calls in order. */ +export const TestTurn = z.strictObject({ + text: z.string().optional(), + calls: z.array(z.strictObject({ name: z.string().min(1), args: z.record(z.string(), z.unknown()) })), +}); +export type TestTurn = z.infer; export const RunnerInput = z.object({ + protocol_version: z.literal(PROTOCOL_VERSION), /** Correlates host-side events; the sandbox runner validates but does not emit it. */ trace_id: z.string().min(1), + /** Selects the agent, and so its harness, from the runner bundle. */ agent_id: z.string().min(1), /** The agent's prompt file, verbatim. */ system_prompt: z.string().min(1), /** The task for this run. */ message: z.string().min(1), - model_provider: z.string().min(1), - model: z.string().min(1), - thinking_level: ThinkingLevel.default("medium"), tools: z.array(ToolName), /** JSON Schema (an object schema) of the agent's output; the runner's `submit` tool takes exactly this. */ output_schema: z.record(z.string(), z.unknown()), @@ -31,6 +37,8 @@ export const RunnerInput = z.object({ cwd: z.string().min(1).optional(), /** The only writable location besides `cwd` (HOME and TMPDIR for tools). */ scratch_dir: z.string().min(1).optional(), + /** Scripted model turns, honored only with `--test-stub`; absent means the default test script. */ + test_script: z.array(TestTurn).optional(), }); export type RunnerInput = z.infer; @@ -48,7 +56,14 @@ export type RunnerErrorCode = (typeof RUNNER_ERROR_CODES)[number]; /** One JSON object per line on the runner's stdout. Exactly one `result` or `error` ends the stream. */ export const RunnerEvent = z.discriminatedUnion("type", [ - z.object({ type: z.literal("started"), at: z.iso.datetime(), agent_id: z.string(), model: z.string(), pi_version: z.string() }), + z.object({ + type: z.literal("started"), + at: z.iso.datetime(), + agent_id: z.string(), + harness: z.string(), + harness_version: z.string(), + model: z.string(), + }), z.object({ type: z.literal("tool_call"), at: z.iso.datetime(), tool_call_id: z.string(), tool: z.string(), args: z.unknown() }), z.object({ type: z.literal("tool_result"), diff --git a/app/runner/core/submit.ts b/app/runner/core/submit.ts new file mode 100644 index 0000000..a054e4a --- /dev/null +++ b/app/runner/core/submit.ts @@ -0,0 +1,95 @@ +import type { TSchema } from "typebox"; +import { Value } from "typebox/value"; + +export const SUBMIT_TOOL = "submit"; +export const MAX_SUBMIT_ATTEMPTS = 3; + +type JsonOutputSchema = Record; + +/** + * Why `schema` can't be a submit tool's parameters, or undefined if it can. Pi's Anthropic adapter sends only + * the top-level `properties` and `required`, so the root must be a plain object schema, not a union or a ref. + */ +export function outputSchemaProblem(schema: JsonOutputSchema): string | undefined { + if (schema.type !== "object") return 'output_schema must have type "object" at its root'; + const props = schema.properties; + if (typeof props !== "object" || props === null || Array.isArray(props)) { + return "output_schema must have a properties object at its root"; + } + try { + Value.Check(schema as TSchema, {}); + } catch (err) { + return `output_schema is not a usable JSON Schema: ${err instanceof Error ? err.message : String(err)}`; + } + return undefined; +} + +/** The submit tool's parameters as the model sees them: the output schema without JSON Schema metadata. */ +export function submitParameters(schema: JsonOutputSchema): JsonOutputSchema { + const { $schema: _schema, $id: _id, ...rest } = schema; + return rest; +} + +export function submitDescription(schema: JsonOutputSchema): string { + const what = typeof schema.description === "string" && schema.description ? ` ${schema.description}` : ""; + return ( + `Submit your final output.${what} Call it exactly once, as your final action, on its own. The arguments are ` + + "checked against the schema; a rejected submission records nothing, so fix every listed problem and call it " + + `again (${MAX_SUBMIT_ATTEMPTS} attempts in total).` + ); +} + +export const SUBMIT_ACCEPTED = "Submission recorded. You are done."; +export const SUBMIT_DUPLICATE = "Already submitted; this call was ignored."; + +export type SubmitCheck = + | { ok: true; checked: unknown } + /** `exhausted` is set on the last allowed attempt: the run must stop with `invalid_submission` and this message. */ + | { ok: false; message: string; exhausted?: string }; + +/** + * One run's submission: checks each attempt against the output JSON Schema with TypeBox, which validates plain + * JSON Schema (including what `z.toJSONSchema` emits), and counts rejections. A usability check so the model can + * fix its submission: the host revalidates the accepted output with the agent's Zod schema, and that check is + * authoritative. Every harness's submit tool goes through this. + */ +export class Submission { + #accepted: { output: unknown } | undefined; + #rejected = 0; + + constructor(private readonly schema: JsonOutputSchema) {} + + get accepted(): { output: unknown } | undefined { + return this.#accepted; + } + + /** Checks `args` without recording them. The checked copy is what `accept` should record. */ + check(args: unknown): SubmitCheck { + if (Value.Check(this.schema as TSchema, args)) return { ok: true, checked: structuredClone(args) }; + const found = [...Value.Errors(this.schema as TSchema, args)].map((e) => `${e.instancePath || "(root)"}: ${e.message}`); + const problems = found.length > 0 ? [...new Set(found)] : ["(root): does not match the output schema"]; + this.#rejected++; + const remaining = MAX_SUBMIT_ATTEMPTS - this.#rejected; + const message = + `${SUBMIT_TOOL} rejected (attempt ${this.#rejected} of ${MAX_SUBMIT_ATTEMPTS}); nothing was recorded. ` + + (remaining > 0 ? "Fix every problem below and call it again:\n" : "No attempts remain:\n") + + problems.map((p) => `- ${p}`).join("\n"); + if (remaining > 0) return { ok: false, message }; + // Schema paths, rule names, and field names only: TypeBox's errors don't quote the submitted values. + const fields = args && typeof args === "object" && !Array.isArray(args) ? Object.keys(args) : []; + const names = + fields + .slice(0, 20) + .map((f) => f.slice(0, 40)) + .join(", ") || "none"; + const exhausted = `${SUBMIT_TOOL} was rejected ${this.#rejected} times; last: ${problems.slice(0, 5).join("; ")}; submitted fields: ${names}`; + return { ok: false, message, exhausted }; + } + + /** Records the first accepted output; false when one was already recorded. */ + accept(output: unknown): boolean { + if (this.#accepted) return false; + this.#accepted = { output }; + return true; + } +} diff --git a/app/runner/core/test-script.ts b/app/runner/core/test-script.ts new file mode 100644 index 0000000..2ccb891 --- /dev/null +++ b/app/runner/core/test-script.ts @@ -0,0 +1,52 @@ +import type { RunnerInput, TestTurn } from "./protocol.js"; +import { SUBMIT_TOOL } from "./submit.js"; + +/** + * The `--test-stub` script when the input has none: lists the working directory if the agent has `ls`, then + * submits a minimal value built from the output schema. Exercises a harness end to end without a model (snapshot + * self-checks, tests). + */ +export function defaultTestScript(input: RunnerInput): TestTurn[] { + const turns: TestTurn[] = []; + if (input.tools.includes("ls")) turns.push({ text: "[test-stub] Listing the working directory.", calls: [{ name: "ls", args: {} }] }); + turns.push({ calls: [{ name: SUBMIT_TOOL, args: sampleFromSchema(input.output_schema) as Record }] }); + return turns; +} + +/** A small value that satisfies common JSON Schema keywords (enough for test-stub runs, not a general generator). */ +function sampleFromSchema(schema: unknown): unknown { + if (typeof schema !== "object" || schema === null) return null; + const s = schema as Record; + if ("const" in s) return s.const; + if (Array.isArray(s.enum) && s.enum.length > 0) return s.enum[0]; + if (s.default !== undefined) return s.default; + const variants = (s.anyOf ?? s.oneOf) as unknown[] | undefined; + if (Array.isArray(variants) && variants.length > 0) { + const nonNull = variants.find((v) => (v as { type?: unknown })?.type !== "null"); + return sampleFromSchema(nonNull ?? variants[0]); + } + const types = Array.isArray(s.type) ? s.type : [s.type]; + const type = types.find((t) => t !== "null") ?? types[0]; + switch (type) { + case "object": { + const props = (s.properties ?? {}) as Record; + const required = Array.isArray(s.required) ? (s.required as string[]) : []; + return Object.fromEntries(required.map((key) => [key, sampleFromSchema(props[key])])); + } + case "array": + return Array.from({ length: typeof s.minItems === "number" ? s.minItems : 0 }, () => sampleFromSchema(s.items)); + case "string": { + const min = typeof s.minLength === "number" ? s.minLength : 0; + return "[test-stub]".padEnd(min, "."); + } + case "integer": + case "number": + return typeof s.minimum === "number" && s.minimum > 0 ? s.minimum : 0; + case "boolean": + return false; + case "null": + return null; + default: + return null; + } +} diff --git a/app/runner/keys.ts b/app/runner/keys.ts deleted file mode 100644 index 56450f5..0000000 --- a/app/runner/keys.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** Each supported provider's key env vars, first one preferred. Every one of them is removed before tools run. */ -export const PROVIDER_KEY_ENV: Readonly> = { - anthropic: ["ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_AUTH_TOKEN"], - google: ["GEMINI_API_KEY"], - openai: ["OPENAI_API_KEY"], -}; - -const ALL_PROVIDER_KEY_ENV: readonly string[] = [...new Set(Object.values(PROVIDER_KEY_ENV).flat())]; - -/** The selected provider's own (first) key var, never another provider's. The host reads the key it sends with this. */ -export function providerApiKey(provider: string, env: Readonly>): string | undefined { - const name = PROVIDER_KEY_ENV[provider]?.[0]; - return (name && env[name]?.trim()) || undefined; -} - -/** - * Removes every provider's key vars from `env`, so none reach the agent's tool subprocesses, and returns their - * values for redaction. Every provider's vars go, not just the selected one: ANTHROPIC_OAUTH_TOKEN, for one, would - * take precedence inside Pi. The runner's only key is the one on stdin. - */ -export function scrubProviderKeys(env: NodeJS.ProcessEnv): string[] { - const removed: string[] = []; - for (const name of ALL_PROVIDER_KEY_ENV) { - const value = env[name]; - if (value) removed.push(value); - delete env[name]; - } - return removed; -} diff --git a/app/runner/main.ts b/app/runner/main.ts index b837f22..16269d8 100644 --- a/app/runner/main.ts +++ b/app/runner/main.ts @@ -1,17 +1,20 @@ import { mkdirSync, realpathSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; -import { PROVIDER_KEY_ENV, scrubProviderKeys } from "./keys.js"; -import { RUNNER_EXIT } from "./protocol.js"; -import { createEmitter, createModels, errorMessage, PI_VERSION, parseInput, runSession } from "./session.js"; -import { defaultTestStub, type TestStub } from "./test-stub.js"; +import type { Agent } from "../agents/define.js"; +import { AGENTS } from "../agents/registry.js"; +import { createEmitter, errorMessage } from "./core/emit.js"; +import { parseInput } from "./core/input.js"; +import { scrubKeys } from "./core/keys.js"; +import { PROTOCOL_VERSION, RUNNER_EXIT } from "./core/protocol.js"; +import { defaultTestScript } from "./core/test-script.js"; interface MainIO { stdin: AsyncIterable; /** Resolves once `chunk` is flushed. */ write(chunk: string): Promise; - /** Replaces the default `--test-stub` responses (tests). */ - testStub?: TestStub; + /** Replaces the bundled agents (tests). */ + agents?: readonly Agent[]; } const processIO = (): MainIO => ({ @@ -27,10 +30,12 @@ function useScratch(scratchDir: string): void { /** * `runner.mjs [--key-stdin] [--test-stub] [--self-test]`: RunnerInput on stdin, RunnerEvent JSONL on stdout. With - * `--key-stdin`, the first line of stdin is the model key and the rest is the RunnerInput. Returns the exit code. + * `--key-stdin`, the first line of stdin is the model key and the rest is the RunnerInput. The input's `agent_id` + * selects the agent, and so its harness, from the bundle. Returns the exit code. */ export async function main(argv: string[], io: MainIO = processIO()): Promise { const startedAt = Date.now(); + const agents = io.agents ?? AGENTS; const emit = createEmitter(io.write); const fail = async (code: "invalid_input" | "agent_error", message: string) => { emit({ type: "error", code, message }); @@ -52,7 +57,11 @@ export async function main(argv: string[], io: MainIO = processIO()): Promise a.harness.credential), + )) + emit.redact(secret); // stdout carries only RunnerEvent JSONL; route any library console output to stderr. console.log = console.info = console.debug = console.error; @@ -67,7 +76,7 @@ export async function main(argv: string[], io: MainIO = processIO()): Promise a.id === input.agent_id); + if (!agent) return fail("invalid_input", `unknown agent "${input.agent_id}": the runner bundle doesn't define it`); + if (input.test_script && !args["test-stub"]) return fail("invalid_input", "test_script is only honored with --test-stub"); try { if (input.scratch_dir) useScratch(input.scratch_dir); - return await runSession({ + return await agent.harness.run({ input, emit, apiKey, - testStub: args["test-stub"] ? (io.testStub ?? defaultTestStub) : undefined, + testScript: args["test-stub"] ? (input.test_script ?? defaultTestScript(input)) : undefined, startedAt, }); } catch (err) { @@ -96,14 +108,29 @@ export async function main(argv: string[], io: MainIO = processIO()): Promise { - const models = createModels(); +/** Checks every bundled harness and prints one JSON line: `ok`, the protocol, and each harness's label and version. */ +async function selfTest(io: MainIO, agents: readonly Agent[]): Promise { + const harnesses = [...new Map(agents.map((a) => [`${a.harness.name} ${a.harness.label}`, a.harness])).values()]; const errors: string[] = []; - for (const p of Object.keys(PROVIDER_KEY_ENV)) { - if (!models.getModels(p).length) errors.push(`no models for provider "${p}"`); + const reported: { name: string; label: string; version: string }[] = []; + for (const harness of harnesses) { + try { + const { version, problems } = await harness.selfTest(); + reported.push({ name: harness.name, label: harness.label, version }); + errors.push(...problems); + } catch (err) { + reported.push({ name: harness.name, label: harness.label, version: "unknown" }); + errors.push(`${harness.name}: ${errorMessage(err)}`); + } } const ok = errors.length === 0; - const report = { ok, pi_version: PI_VERSION, node: process.version, ...(ok ? {} : { errors }) }; + const report = { + ok, + protocol_version: PROTOCOL_VERSION, + node: process.version, + harnesses: reported, + ...(ok ? {} : { errors }), + }; await io.write(`${JSON.stringify(report)}\n`); return ok ? RUNNER_EXIT.ok : RUNNER_EXIT.error; } diff --git a/app/runner/session.ts b/app/runner/pi/session.ts similarity index 56% rename from app/runner/session.ts rename to app/runner/pi/session.ts index c1a0228..4431261 100644 --- a/app/runner/session.ts +++ b/app/runner/pi/session.ts @@ -1,4 +1,3 @@ -import { statSync } from "node:fs"; import { setTimeout as sleep } from "node:timers/promises"; import { type AfterToolCallContext, Agent, type AgentEvent } from "@earendil-works/pi-agent-core"; import agentCorePackage from "@earendil-works/pi-agent-core/package.json" with { type: "json" }; @@ -13,14 +12,23 @@ import { type Usage as PiUsage, } from "@earendil-works/pi-ai"; import { builtinModels } from "@earendil-works/pi-ai/providers/all"; -import { RUNNER_EXIT, type RunnerErrorCode, type RunnerEvent, RunnerInput, type ThinkingLevel, type Usage } from "./protocol.js"; -import { createSubmitTool, MAX_SUBMIT_ATTEMPTS, outputSchemaProblem, SUBMIT_TOOL } from "./submit.js"; -import { registerTestStubModel, type TestStub } from "./test-stub.js"; -import { createTools } from "./tools.js"; +import { type Emitter, errorMessage, previewText } from "../core/emit.js"; +import type { HarnessRun } from "../core/harness.js"; +import { RUNNER_EXIT, type RunnerErrorCode, type Usage } from "../core/protocol.js"; +import { SUBMIT_TOOL, Submission } from "../core/submit.js"; +import { contentText, createTools } from "../tools/index.js"; +import { createSubmitTool } from "./submit-tool.js"; +import { registerTestStubModel } from "./test-stub.js"; -/** Reported in `started` and `--self-test`. */ export const PI_VERSION: string = agentCorePackage.version; +export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high"; + +export interface PiSessionOptions { + model: Model; + thinkingLevel: ThinkingLevel; +} + const MAX_OUTPUT_TOKENS = 16_384; /** Passed to the agent so the max_tokens cap below stays in step with Pi's thinking budget. */ const THINKING_BUDGETS = { minimal: 1024, low: 2048, medium: 8192, high: 16_384 } as const; @@ -35,56 +43,6 @@ const RETRY_BASE_DELAY_MS = 2_000; const SUBMIT_REMINDER = `You stopped without calling ${SUBMIT_TOOL}. Call it now, exactly once, with your final output built from what you have found so far.`; -export const errorMessage = (err: unknown): string => (err instanceof Error ? err.message : String(err)); - -// ── events ────────────────────────────────────────────────────────────── - -type WithoutAt = E extends unknown ? Omit : never; -type EventBody = WithoutAt; - -interface Emitter { - (event: EventBody): void; - readonly terminated: boolean; - /** Replaces `secret` with `[REDACTED]` in every later line. */ - redact(secret: string): void; - /** Resolves once every line written so far has been flushed. */ - flush(): Promise; -} - -/** Writes one RunnerEvent per line. Drops everything after the first terminal (`result`/`error`) event. */ -export function createEmitter(write: (chunk: string) => Promise): Emitter { - const secrets: string[] = []; - let pending = Promise.resolve(); - let terminated = false; - const emit = (event: EventBody) => { - if (terminated) return; - terminated = event.type === "result" || event.type === "error"; - const { type, ...fields } = event; - let line = JSON.stringify({ type, at: new Date().toISOString(), ...fields }); - for (const secret of secrets) line = line.replaceAll(secret, "[REDACTED]"); - pending = pending.then(() => write(`${line}\n`)).catch(() => {}); - }; - return Object.assign(emit, { - get terminated() { - return terminated; - }, - redact(secret: string) { - if (secret) secrets.push(secret); - }, - flush: () => pending, - }); -} - -const PREVIEW_CHARS = 2000; -const TRUNCATED = "…[truncated]"; - -function outputPreview(result: unknown): string { - const content = (result as { content?: unknown } | undefined)?.content; - if (!Array.isArray(content)) return ""; - const text = content.map((c: { type?: string; text?: string }) => (c?.type === "text" ? (c.text ?? "") : `[${c?.type ?? "unknown"}]`)).join("\n"); - return text.length > PREVIEW_CHARS ? text.slice(0, PREVIEW_CHARS - TRUNCATED.length) + TRUNCATED : text; -} - /** * Undefined for all-zero usage (e.g. a request aborted before it was sent). When the provider reports no cost * (the test stub never does), the cost is computed from the model's prices. @@ -103,8 +61,6 @@ function toUsage(u: PiUsage, model: Model): Usage | undefined { return typeof cost === "number" && Number.isFinite(cost) && cost >= 0 ? { ...usage, cost_usd: cost } : usage; } -// ── models ────────────────────────────────────────────────────────────── - /** The built-in provider catalog with in-memory credentials: the key is passed per request, nothing is stored. */ export function createModels(): MutableModels { return builtinModels({ credentials: new InMemoryCredentialStore() }); @@ -117,71 +73,27 @@ function capMaxTokens(model: Model, level: ThinkingLevel): Model { return { ...model, maxTokens: Math.min(model.maxTokens, MAX_OUTPUT_TOKENS + thinking) }; } -// ── input ─────────────────────────────────────────────────────────────── - -/** Validates a RunnerInput, including what Zod can't see: the output schema's root and the directories. */ -export function parseInput(json: unknown): RunnerInput | string { - const parsed = RunnerInput.safeParse(json); - if (!parsed.success) { - return `invalid RunnerInput: ${parsed.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ")}`; - } - const input = parsed.data; - const schemaProblem = outputSchemaProblem(input.output_schema); - if (schemaProblem) return schemaProblem; - if (input.tools.length > 0) { - if (!input.cwd) return "cwd is required when tools is non-empty"; - if (!input.scratch_dir) return "scratch_dir is required when tools is non-empty"; - if (!isDirectory(input.cwd)) return `cwd ${input.cwd} is not a directory`; - } - return input; -} - -function isDirectory(path: string): boolean { - try { - return statSync(path).isDirectory(); - } catch { - return false; - } -} - -// ── session ───────────────────────────────────────────────────────────── - type Failure = { code: RunnerErrorCode; message: string }; -interface SessionOptions { - /** Already validated with parseInput. */ - input: RunnerInput; - emit: Emitter; - /** Passed with every request; unused with `testStub`. */ - apiKey?: string; - /** Replaces the model with deterministic test responses (tests, snapshot self-checks). */ - testStub?: TestStub; - /** Epoch ms the `timeout_ms` budget is measured from (default: now). */ - startedAt?: number; - /** Aborting stops the run with `agent_error`. */ - signal?: AbortSignal; -} - -/** Runs one agent and emits `started` … then exactly one `result` or `error`. Returns the exit code. */ -export async function runSession(opts: SessionOptions): Promise { - const { input, emit } = opts; - const startedAt = opts.startedAt ?? Date.now(); +/** Runs one agent on Pi and emits `started` … then exactly one `result` or `error`. Returns the exit code. */ +export async function runPiSession(run: HarnessRun, opts: PiSessionOptions): Promise { + const { input, emit, startedAt } = run; const fail = (code: RunnerErrorCode, message: string) => { emit({ type: "error", code, message }); return code === "invalid_input" ? RUNNER_EXIT.invalid_input : RUNNER_EXIT.error; }; const models = createModels(); - const requested = models.getModel(input.model_provider, input.model); + const requested = models.getModel(opts.model.provider, opts.model.id); if (!requested) { - return fail("invalid_input", `unknown model "${input.model_provider}/${input.model}" (not in the Pi ${PI_VERSION} registry)`); + return fail("invalid_input", `unknown model "${opts.model.provider}/${opts.model.id}" (not in the Pi ${PI_VERSION} registry)`); } - if (!opts.testStub && !opts.apiKey) { - return fail("agent_error", `no API key for provider "${input.model_provider}": pass it with --key-stdin`); + if (!run.testScript && !run.apiKey) { + return fail("agent_error", `no API key for provider "${opts.model.provider}": pass it with --key-stdin`); } - const model = capMaxTokens(opts.testStub ? registerTestStubModel(models, opts.testStub(input), requested) : requested, input.thinking_level); + const model = capMaxTokens(run.testScript ? registerTestStubModel(models, run.testScript, requested) : requested, opts.thinkingLevel); - let submission: { output: unknown } | undefined; + const submission = new Submission(input.output_schema); let failure: Failure | undefined; const stopped = new AbortController(); const tools = @@ -193,34 +105,16 @@ export async function runSession(opts: SessionOptions): Promise { initialState: { systemPrompt: input.system_prompt, model, - thinkingLevel: input.thinking_level, - tools: [ - ...tools, - createSubmitTool(input.output_schema, { - onAccepted: (output) => { - submission = { output }; - }, - onRejected: (attempts, problems, fields) => { - if (attempts < MAX_SUBMIT_ATTEMPTS) return; - // Schema paths, rule names, and field names only: TypeBox's errors don't quote the submitted values. - const last = problems.slice(0, 5).join("; "); - const names = - fields - .slice(0, 20) - .map((f) => f.slice(0, 40)) - .join(", ") || "none"; - stop({ code: "invalid_submission", message: `${SUBMIT_TOOL} was rejected ${attempts} times; last: ${last}; submitted fields: ${names}` }); - }, - }), - ], + thinkingLevel: opts.thinkingLevel, + tools: [...tools, createSubmitTool(input.output_schema, submission, (message) => stop({ code: "invalid_submission", message }))], }, streamFn: (m, context, options) => models.streamSimple(m, context, options), - getApiKey: () => opts.apiKey, + getApiKey: () => run.apiKey, thinkingBudgets: THINKING_BUDGETS, afterToolCall: async (ctx) => (endsBatch(ctx) || failure !== undefined ? { terminate: true } : undefined), }); - emit({ type: "started", agent_id: input.agent_id, model: `${model.provider}/${model.id}`, pi_version: PI_VERSION }); + emit({ type: "started", agent_id: input.agent_id, harness: "pi", harness_version: PI_VERSION, model: `${model.provider}/${model.id}` }); function stop(reason?: Failure) { if (reason) failure ??= reason; @@ -245,13 +139,13 @@ export async function runSession(opts: SessionOptions): Promise { if (event.type !== "turn_end") return; turns++; // Backstop for batches that still continue after a submission (e.g. a sibling call failed validation). - if (submission || failure) stop(); + if (submission.accepted || failure) stop(); else if (turns >= input.max_turns) stop({ code: "max_turns", message: `turn limit (${input.max_turns}) reached without a submission` }); }); const onAbort = () => stop({ code: "agent_error", message: "the run was cancelled" }); - if (opts.signal?.aborted) onAbort(); - opts.signal?.addEventListener("abort", onAbort, { once: true }); + if (run.signal?.aborted) onAbort(); + run.signal?.addEventListener("abort", onAbort, { once: true }); const budgetMs = input.timeout_ms - Math.min(SAFETY_MARGIN_MS, Math.floor(input.timeout_ms / 10)); let deadline: NodeJS.Timeout | undefined; @@ -270,7 +164,7 @@ export async function runSession(opts: SessionOptions): Promise { const conversation = (async () => { if (stopped.signal.aborted) return; await prompt(input.message); - if (!submission && !failure && stoppedOnItsOwn(agent)) await prompt(SUBMIT_REMINDER); + if (!submission.accepted && !failure && stoppedOnItsOwn(agent)) await prompt(SUBMIT_REMINDER); })(); let thrown: unknown; await Promise.race([ @@ -282,10 +176,10 @@ export async function runSession(opts: SessionOptions): Promise { clearTimeout(deadline); clearTimeout(grace); unsubscribe(); - opts.signal?.removeEventListener("abort", onAbort); + run.signal?.removeEventListener("abort", onAbort); - if (submission) { - emit({ type: "result", output: submission.output, turns }); + if (submission.accepted) { + emit({ type: "result", output: submission.accepted.output, turns }); return RUNNER_EXIT.ok; } failure ??= diagnose(agent, thrown); @@ -304,7 +198,7 @@ function forward(event: AgentEvent, emit: Emitter, model: Model) { tool_call_id: event.toolCallId, tool: event.toolName, is_error: event.isError, - output_preview: outputPreview(event.result), + output_preview: previewText(contentText((event.result as { content?: unknown } | undefined)?.content)), }); break; case "message_end": { diff --git a/app/runner/pi/submit-tool.ts b/app/runner/pi/submit-tool.ts new file mode 100644 index 0000000..c05b4e0 --- /dev/null +++ b/app/runner/pi/submit-tool.ts @@ -0,0 +1,42 @@ +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import type { TSchema } from "typebox"; +import { SUBMIT_ACCEPTED, SUBMIT_DUPLICATE, SUBMIT_TOOL, type Submission, submitDescription, submitParameters } from "../core/submit.js"; + +/** + * Pi's submit tool: its parameters are the agent's output schema. Submissions are checked in `prepareArguments`, + * which runs before Pi's own argument validation (which would otherwise reject a call without counting it, and + * coerce values the host's Zod schema would refuse). `onExhausted` gets the failure message after the last + * allowed rejection. + */ +export function createSubmitTool( + schema: Record, + submission: Submission, + onExhausted: (message: string) => void, +): AgentTool { + /** The arguments as checked, before Pi's validation coerces them; calls run one at a time (sequential). */ + let checked: unknown; + return { + name: SUBMIT_TOOL, + label: "Submit", + description: submitDescription(schema), + parameters: submitParameters(schema) as TSchema, + executionMode: "sequential", + prepareArguments(args) { + if (submission.accepted) return args; + const verdict = submission.check(args); + if (verdict.ok) { + checked = verdict.checked; + return args; + } + if (verdict.exhausted) onExhausted(verdict.exhausted); + throw new Error(verdict.message); + }, + async execute(_toolCallId, params) { + const output = checked ?? params; + if (!submission.accept(output)) { + return { content: [{ type: "text", text: SUBMIT_DUPLICATE }], details: submission.accepted?.output, terminate: true }; + } + return { content: [{ type: "text", text: SUBMIT_ACCEPTED }], details: output, terminate: true }; + }, + }; +} diff --git a/app/runner/pi/test-stub.ts b/app/runner/pi/test-stub.ts new file mode 100644 index 0000000..5a7bf45 --- /dev/null +++ b/app/runner/pi/test-stub.ts @@ -0,0 +1,33 @@ +import { + type Api, + type FauxResponseStep, + fauxAssistantMessage, + fauxProvider, + fauxText, + fauxToolCall, + type Model, + type MutableModels, +} from "@earendil-works/pi-ai"; +import type { TestTurn } from "../core/protocol.js"; + +const TEST_STUB_MODEL = "test-stub"; + +/** + * Adds a provider that plays `turns` instead of calling a model. It carries the requested model's prices, so the + * runner can cost the stub's token estimates. + */ +export function registerTestStubModel(models: MutableModels, turns: readonly TestTurn[], priced: Model): Model { + const { input, output, cacheRead, cacheWrite } = priced.cost; + const stub = fauxProvider({ + provider: TEST_STUB_MODEL, + models: [{ id: TEST_STUB_MODEL, reasoning: false, cost: { input, output, cacheRead, cacheWrite } }], + }); + models.setProvider(stub.provider); + stub.setResponses(turns.map(toResponse)); + return stub.getModel() as Model; +} + +function toResponse(turn: TestTurn): FauxResponseStep { + const content = [...(turn.text ? [fauxText(turn.text)] : []), ...turn.calls.map((c) => fauxToolCall(c.name, c.args))]; + return fauxAssistantMessage(content, { stopReason: turn.calls.length > 0 ? "toolUse" : "stop" }); +} diff --git a/app/runner/submit.ts b/app/runner/submit.ts deleted file mode 100644 index 352000f..0000000 --- a/app/runner/submit.ts +++ /dev/null @@ -1,100 +0,0 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; -import type { TSchema } from "typebox"; -import { Value } from "typebox/value"; - -export const SUBMIT_TOOL = "submit"; -export const MAX_SUBMIT_ATTEMPTS = 3; - -type JsonOutputSchema = Record; - -/** - * Why `schema` can't be a submit tool's parameters, or undefined if it can. Pi's Anthropic adapter sends only - * the top-level `properties` and `required`, so the root must be a plain object schema, not a union or a ref. - */ -export function outputSchemaProblem(schema: JsonOutputSchema): string | undefined { - if (schema.type !== "object") return 'output_schema must have type "object" at its root'; - const props = schema.properties; - if (typeof props !== "object" || props === null || Array.isArray(props)) { - return "output_schema must have a properties object at its root"; - } - try { - Value.Check(schema as TSchema, {}); - } catch (err) { - return `output_schema is not a usable JSON Schema: ${err instanceof Error ? err.message : String(err)}`; - } - return undefined; -} - -type Verdict = { ok: true } | { ok: false; problems: string[] }; - -/** - * Checks a submission against the output JSON Schema with TypeBox, which validates plain JSON Schema (including - * what `z.toJSONSchema` emits). A usability check so the model can fix its submission: the host revalidates - * the accepted output with the agent's Zod schema, and that check is authoritative. - */ -function checkSubmission(schema: JsonOutputSchema, candidate: unknown): Verdict { - if (Value.Check(schema as TSchema, candidate)) return { ok: true }; - const problems = [...Value.Errors(schema as TSchema, candidate)].map((e) => `${e.instancePath || "(root)"}: ${e.message}`); - return { ok: false, problems: problems.length > 0 ? [...new Set(problems)] : ["(root): does not match the output schema"] }; -} - -interface SubmitHandlers { - onAccepted(output: unknown): void; - /** Called with the number of rejected attempts so far, what was wrong with the latest one, and its top-level field names. */ - onRejected(attempts: number, problems: readonly string[], fields: readonly string[]): void; -} - -/** The tool's parameters as the model sees them: the output schema without JSON Schema metadata. */ -function toolParameters(schema: JsonOutputSchema): TSchema { - const { $schema: _schema, $id: _id, ...rest } = schema; - return rest as TSchema; -} - -/** - * The generic submit tool: its parameters are the agent's output schema. Submissions are checked in - * `prepareArguments`, which runs before Pi's own argument validation (which would otherwise reject a call - * without counting it, and coerce values the host's Zod schema would refuse). - */ -export function createSubmitTool(schema: JsonOutputSchema, handlers: SubmitHandlers): AgentTool { - let accepted: { output: unknown } | undefined; - /** The arguments as checked, before Pi's validation coerces them; calls run one at a time (sequential). */ - let checked: unknown; - let rejected = 0; - const what = typeof schema.description === "string" && schema.description ? ` ${schema.description}` : ""; - return { - name: SUBMIT_TOOL, - label: "Submit", - description: - `Submit your final output.${what} Call it exactly once, as your final action, on its own. The arguments are ` + - "checked against the schema; a rejected submission records nothing, so fix every listed problem and call it " + - `again (${MAX_SUBMIT_ATTEMPTS} attempts in total).`, - parameters: toolParameters(schema), - executionMode: "sequential", - prepareArguments(args) { - if (accepted) return args; - const verdict = checkSubmission(schema, args); - if (verdict.ok) { - checked = structuredClone(args); - return args; - } - rejected++; - const remaining = MAX_SUBMIT_ATTEMPTS - rejected; - const fields = args && typeof args === "object" && !Array.isArray(args) ? Object.keys(args) : []; - handlers.onRejected(rejected, verdict.problems, fields); - throw new Error( - `${SUBMIT_TOOL} rejected (attempt ${rejected} of ${MAX_SUBMIT_ATTEMPTS}); nothing was recorded. ` + - (remaining > 0 ? "Fix every problem below and call it again:\n" : "No attempts remain:\n") + - verdict.problems.map((p) => `- ${p}`).join("\n"), - ); - }, - async execute(_toolCallId, params) { - if (accepted) { - return { content: [{ type: "text", text: "Already submitted; this call was ignored." }], details: accepted.output, terminate: true }; - } - const output = checked ?? params; - accepted = { output }; - handlers.onAccepted(output); - return { content: [{ type: "text", text: "Submission recorded. You are done." }], details: output, terminate: true }; - }, - }; -} diff --git a/app/runner/test-stub.ts b/app/runner/test-stub.ts deleted file mode 100644 index 4ab2eaa..0000000 --- a/app/runner/test-stub.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { - type Api, - type FauxResponseStep as TestStubResponse, - fauxAssistantMessage as testStubAssistantMessage, - fauxProvider as testStubProvider, - fauxText as testStubText, - fauxToolCall as testStubToolCall, - type Model, - type MutableModels, -} from "@earendil-works/pi-ai"; -import type { RunnerInput } from "./protocol.js"; -import { SUBMIT_TOOL } from "./submit.js"; - -const TEST_STUB_MODEL = "test-stub"; -export type TestStub = (input: RunnerInput) => TestStubResponse[]; - -/** - * Adds a provider that returns deterministic test responses instead of calling a model. It carries the requested - * model's prices, so the runner can cost the stub's token estimates. - */ -export function registerTestStubModel(models: MutableModels, responses: TestStubResponse[], priced: Model): Model { - const { input, output, cacheRead, cacheWrite } = priced.cost; - const stub = testStubProvider({ - provider: TEST_STUB_MODEL, - models: [{ id: TEST_STUB_MODEL, reasoning: false, cost: { input, output, cacheRead, cacheWrite } }], - }); - models.setProvider(stub.provider); - stub.setResponses(responses); - return stub.getModel() as Model; -} - -const toolUse = { stopReason: "toolUse" } as const; - -/** - * Deterministic `--test-stub` run: lists the working directory if the agent has `ls`, then submits a minimal value - * built from the output schema. Exercises the runner end to end without a model (snapshot self-checks, tests). - */ -export const defaultTestStub: TestStub = (input) => { - const responses: TestStubResponse[] = []; - if (input.tools.includes("ls")) { - responses.push(testStubAssistantMessage([testStubText("[test-stub] Listing the working directory."), testStubToolCall("ls", {})], toolUse)); - } - const output = sampleFromSchema(input.output_schema) as Record; - responses.push(testStubAssistantMessage(testStubToolCall(SUBMIT_TOOL, output), toolUse)); - return responses; -}; - -/** A small value that satisfies common JSON Schema keywords (enough for test-stub runs, not a general generator). */ -function sampleFromSchema(schema: unknown): unknown { - if (typeof schema !== "object" || schema === null) return null; - const s = schema as Record; - if ("const" in s) return s.const; - if (Array.isArray(s.enum) && s.enum.length > 0) return s.enum[0]; - if (s.default !== undefined) return s.default; - const variants = (s.anyOf ?? s.oneOf) as unknown[] | undefined; - if (Array.isArray(variants) && variants.length > 0) { - const nonNull = variants.find((v) => (v as { type?: unknown })?.type !== "null"); - return sampleFromSchema(nonNull ?? variants[0]); - } - const types = Array.isArray(s.type) ? s.type : [s.type]; - const type = types.find((t) => t !== "null") ?? types[0]; - switch (type) { - case "object": { - const props = (s.properties ?? {}) as Record; - const required = Array.isArray(s.required) ? (s.required as string[]) : []; - return Object.fromEntries(required.map((key) => [key, sampleFromSchema(props[key])])); - } - case "array": - return Array.from({ length: typeof s.minItems === "number" ? s.minItems : 0 }, () => sampleFromSchema(s.items)); - case "string": { - const min = typeof s.minLength === "number" ? s.minLength : 0; - return "[test-stub]".padEnd(min, "."); - } - case "integer": - case "number": - return typeof s.minimum === "number" && s.minimum > 0 ? s.minimum : 0; - case "boolean": - return false; - case "null": - return null; - default: - return null; - } -} diff --git a/app/runner/tools.ts b/app/runner/tools/index.ts similarity index 75% rename from app/runner/tools.ts rename to app/runner/tools/index.ts index 3228126..da246a8 100644 --- a/app/runner/tools.ts +++ b/app/runner/tools/index.ts @@ -13,10 +13,13 @@ import { type ReadToolDetails, } from "@earendil-works/pi-agent-core"; import { NodeExecutionEnv } from "@earendil-works/pi-agent-core/node"; -import { Type } from "typebox"; -import type { ToolName } from "./protocol.js"; +import { type TSchema, Type } from "typebox"; +import { Value } from "typebox/value"; +import type { ToolName } from "../core/protocol.js"; import { rejectReason } from "./shell-policy.js"; +export { rejectReason } from "./shell-policy.js"; + const DEFAULT_TIMEOUT_S = 120; const MAX_TIMEOUT_S = 1800; const LS_DEFAULT_LIMIT = 500; @@ -71,11 +74,53 @@ export function createTools(opts: ToolOptions): AgentTool[] { return [...new Set(opts.tools)].map((name) => factories[name]()); } -type HarnessTool = ReturnType | ReturnType; +/** A factory tool as any harness registers it: JSON Schema parameters and a text result, no library types. */ +export interface FactoryTool { + name: ToolName; + description: string; + /** JSON Schema of the arguments. */ + parameters: Record; + /** Validates `args`, runs the tool, and never throws: a failure comes back with `isError`. */ + execute(args: unknown, signal?: AbortSignal): Promise<{ text: string; isError: boolean }>; +} + +let nextCallId = 0; + +/** + * The tools named in `opts.tools` as FactoryTools: the same implementations, environment, policy, and limits as + * `createTools`, so every harness gives an agent identical tool behavior. + */ +export function createFactoryTools(opts: ToolOptions): FactoryTool[] { + return createTools(opts).map((tool) => ({ + name: tool.name as ToolName, + description: tool.description, + parameters: tool.parameters as Record, + async execute(args, signal) { + if (!Value.Check(tool.parameters as TSchema, args)) { + const problems = [...Value.Errors(tool.parameters as TSchema, args)].map((e) => `${e.instancePath || "(root)"}: ${e.message}`); + return { text: `Invalid arguments for ${tool.name}:\n${problems.map((p) => `- ${p}`).join("\n")}`, isError: true }; + } + try { + const result = await tool.execute(`factory-${++nextCallId}`, args, signal); + return { text: contentText(result.content), isError: false }; + } catch (err) { + return { text: err instanceof Error ? err.message : String(err), isError: true }; + } + }, + })); +} + +/** A tool result's text content; other content types become a placeholder. */ +export function contentText(content: unknown): string { + if (!Array.isArray(content)) return ""; + return content.map((c: { type?: string; text?: string }) => (c?.type === "text" ? (c.text ?? "") : `[${c?.type ?? "unknown"}]`)).join("\n"); +} + +type ContextTool = ReturnType | ReturnType; /** Binds one of Pi's harness tools (which take the execution context per call) to `context`. */ // biome-ignore lint/suspicious/noExplicitAny: see createTools. -function withContext(inner: HarnessTool, context: ExecutionToolContext): AgentTool { +function withContext(inner: ContextTool, context: ExecutionToolContext): AgentTool { return { ...inner, // biome-ignore lint/suspicious/noExplicitAny: the params match inner.parameters, validated by Pi. diff --git a/app/runner/shell-policy.ts b/app/runner/tools/shell-policy.ts similarity index 100% rename from app/runner/shell-policy.ts rename to app/runner/tools/shell-policy.ts diff --git a/app/sandbox/sandbox.ts b/app/sandbox/sandbox.ts index 3d951f0..3d822dd 100644 --- a/app/sandbox/sandbox.ts +++ b/app/sandbox/sandbox.ts @@ -11,6 +11,7 @@ import { computeLockfileHash, credentialFile, credentialHelperArgs, + ensureRunnerBundle, readToken, resolveBaseHead, runInstall, @@ -68,7 +69,8 @@ export async function openSandbox(repo: RepoContext, ref: SandboxRef, deps: Open const id = await createFromSnapshot(client, repo, deps); try { await client.waitRunning(id); - const info = await verifySandbox(client, id, { waitReady: true }); + await ensureRunnerBundle(client, id); + const info = await verifySandbox(client, id); // verify.sh can only probe /proc as root; a sandbox where it couldn't check never gets a model key. if (!info.runnerNonDumpable) throw new Error(`could not confirm the runner is non-dumpable in ${id}`); const manifest = SnapshotManifestSchema.parse(JSON.parse((await client.download(id, `${APP_DIR}/manifest.json`)).toString("utf8"))); diff --git a/app/sandbox/snapshot.ts b/app/sandbox/snapshot.ts index c058f48..7f2cf1d 100644 --- a/app/sandbox/snapshot.ts +++ b/app/sandbox/snapshot.ts @@ -1,9 +1,11 @@ import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; import { cp, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; +import { gzipSync } from "node:zlib"; import { ClientError, SandboxSnapshotNotFoundError } from "@renderinc/sdk"; import type { SandboxSnapshot } from "@renderinc/sdk/experimental"; import { create as createTar } from "tar"; @@ -265,9 +267,58 @@ async function createSnapshot(client: SandboxClient, sandboxId: string, name: st return snap; } -/** A gzipped tar of the runtime scripts and a freshly built, self-contained runner bundle. */ +// ── Runner bundle ─────────────────────────────────────────────────────── + +const RUNNER_PATH = `${APP_DIR}/runner/runner.mjs`; +let built: Promise<{ text: string; sha256: string }> | undefined; + +/** The runner bundle this host deploys, built once per process. */ +export function runnerBundle(): Promise<{ text: string; sha256: string }> { + built ??= bundleRunner().then( + (text) => ({ text, sha256: createHash("sha256").update(text).digest("hex") }), + (err: unknown) => { + built = undefined; + throw err; + }, + ); + return built; +} + +/** + * Replaces a restored sandbox's runner with this host's bundle when they differ, so a change to an agent's + * harness or model reaches existing snapshots without a rebuild. It runs before verify.sh, which self-tests the + * runner, and installs the file root-owned and read-only to the runner user, as the build does. Returns whether + * it replaced the bundle. + */ +export async function ensureRunnerBundle(client: SandboxClient, id: string): Promise { + const { text, sha256 } = await runnerBundle(); + const quick = { timeoutMs: client.t.verifyTimeoutMs }; + const current = (await client.check(id, `sha256sum ${RUNNER_PATH}`, { ...quick, waitReady: true })).trim().split(/\s+/)[0]; + if (current === sha256) return false; + const stage = `/tmp/factory-runner-${randomUUID()}`; + await client.check(id, `mkdir -m 700 ${stage}`, quick); + try { + // Gzipped for the same reason as the staging archive: the upload endpoint rejects the bundle's raw JavaScript. + await client.upload(id, `${stage}/runner.mjs.gz`, gzipSync(text)); + await client.check( + id, + [ + `gunzip -c ${stage}/runner.mjs.gz > ${stage}/runner.mjs`, + `[ "$(sha256sum ${stage}/runner.mjs | cut -d' ' -f1)" = ${sha256} ]`, + `install -o root -g root -m 0644 ${stage}/runner.mjs ${RUNNER_PATH}`, + ].join(" && "), + quick, + ); + } finally { + await client.exec(id, `rm -rf ${stage}`, quick).catch(() => {}); + } + log("info", "runner bundle replaced", { sandbox: id, from: current || null, to: sha256 }); + return true; +} + +/** A gzipped tar of the runtime scripts and the runner bundle. */ async function stagingArchive(): Promise { - const bundle = await bundleRunner(); + const bundle = (await runnerBundle()).text; const dir = await mkdtemp(join(tmpdir(), "factory-stage-")); try { await mkdir(join(dir, "runner")); diff --git a/docs/architecture.md b/docs/architecture.md index 2dab38f..e47656e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -117,12 +117,14 @@ A base-branch push builds a repository snapshot with the runner, a base checkout, and installed dependencies. A GitHub task then: 1. restores the newest snapshot; -2. verifies runtime permissions and process hardening; -3. fetches the exact requested ref with a temporary read-only token; -4. removes the token and verifies its removal; -5. reinstalls dependencies when the lockfiles or install command changed; -6. runs agents and repository commands as an unprivileged user; and -7. closes the Sandbox in `finally`. +2. replaces the snapshot's runner bundle with this deployment's when their + hashes differ, root-owned and read-only to the runner user; +3. verifies runtime permissions and process hardening; +4. fetches the exact requested ref with a temporary read-only token; +5. removes the token and verifies its removal; +6. reinstalls dependencies when the lockfiles or install command changed; +7. runs agents and repository commands as an unprivileged user; and +8. closes the Sandbox in `finally`. Render terminates a cancelled task without running `finally`. For that case, each GitHub task records its Sandbox ID in the status comment. A replacement @@ -138,7 +140,7 @@ prepared state, not trusted current state. An agent definition fixes: - its prompt; -- model tier and thinking level; +- its harness: the agent library and model that run it; - available tools; - output schema; - turn limit; and @@ -147,6 +149,15 @@ An agent definition fixes: The model loop and every model-directed tool run in the Sandbox beside the checkout. Agent tools never run on the Workflows host. +A harness adapter in `app/harness/` wraps an agent library's own +configuration. Agent definitions and their harnesses are bundled into the +Sandbox runner, which runs the agent named in its input on that agent's +harness. The host and the runner speak a versioned contract +(`app/runner/core/protocol.ts`): JSON on stdin, with the key on the first line, +and JSON Lines events on stdout. Every harness gives the model the factory's +own tools and `submit`, so tool authority and the read-only shell policy don't +depend on the library. + Every agent finishes through a typed `submit` tool. The runner validates the submission against JSON Schema. The host validates it again with the agent's Zod schema. Repositories, refs, issue numbers, pull requests, branches, and @@ -251,7 +262,8 @@ Start with: 5. [`app/workflows/autopilot/index.ts`](../app/workflows/autopilot/index.ts) — parent/subtask orchestration. 6. [`app/agents/run.ts`](../app/agents/run.ts) — host-side agent execution. -7. [`app/runner/main.ts`](../app/runner/main.ts) — Sandbox runner entry point. +7. [`app/runner/main.ts`](../app/runner/main.ts) — Sandbox runner entry point; + [`app/harness/`](../app/harness/) — agent library adapters. 8. [`app/sandbox/sandbox.ts`](../app/sandbox/sandbox.ts) — Sandbox lifecycle. 9. [`app/changeset/index.ts`](../app/changeset/index.ts) — validated publishing. diff --git a/docs/configuration.md b/docs/configuration.md index af615bb..862edc7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,10 +31,6 @@ export default defineFactoryConfig({ }, commandPrefix: "/factory", minimumPermission: "write", - models: { - medium: "anthropic/claude-sonnet-5", - large: "anthropic/claude-opus-5", - }, limits: { repairRounds: 2, maxChangedFiles: 200, @@ -79,8 +75,6 @@ After adding a repository: | `labels.autopilot` | `factory:autopilot` | Runs all three stages through a reviewed draft PR | | `commandPrefix` | `/factory` | Prefix for commands such as `/factory cancel` | | `minimumPermission` | `write` | Live repository permission required to start work | -| `models.medium` | `anthropic/claude-sonnet-5` | Triage and PR Reviewer model | -| `models.large` | `anthropic/claude-opus-5` | Spec writer and implementer model | | `limits.repairRounds` | `2` | Repair attempts after failed verification | | `limits.maxChangedFiles` | `200` | Maximum files in a published change | | `limits.maxFileBytes` | `1000000` | Maximum size of one changed file | @@ -99,22 +93,38 @@ remove old ones. Keep `sandbox.timeoutSeconds` at least as long as the longest Workflows task. -## Models +## Agent harnesses and models + +Each agent runs on a harness: an agent library and its configuration, built by +an adapter in [`app/harness/`](../app/harness/). The built-in agents share two +harnesses from [`app/agents/models.ts`](../app/agents/models.ts): + +```typescript +export const medium = pi({ model: getBuiltinModel("anthropic", "claude-sonnet-5") }); +export const large = pi({ model: getBuiltinModel("anthropic", "claude-opus-5") }); +``` + +Triage and PR Reviewer use `medium`; the spec writer and implementer use +`large`. To change a model, edit these values, or give one agent its own +harness in its definition. Model IDs are type-checked against the library's +catalog, so a typo fails `npm run typecheck`. The `pi` adapter also takes +`thinkingLevel` (`off`, `minimal`, `low`, `medium`, or `high`; default +`medium`). -Agent definitions select the `medium` or `large` tier. -`factory.config.ts` maps each tier to a `provider/model` supported by -`@earendil-works/pi-ai`, the model client used by the Sandbox runner. +Each harness declares the env var of its key, its `credential`. Setup and +`doctor` derive the keys they manage from the agents' harnesses, so a harness that needs a new key +needs no other configuration change. The `pi` adapter supports Anthropic +(`ANTHROPIC_API_KEY`), Google (`GEMINI_API_KEY`), and OpenAI +(`OPENAI_API_KEY`). After a harness change that needs a new key: -Changing a model mapping requires a Workflows redeploy. To add another -provider: +1. add the key to local `.env`; +2. run `npm run setup:render` to add it to the `software-factory-workflows` + env group; +3. deploy `factory-workflows`; and +4. run `npm run doctor -- --service workflows`. -1. choose a registered `provider/model`; -2. find its key name in [`app/runner/keys.ts`](../app/runner/keys.ts); -3. add the key to `factory-workflows` in [`render.yaml`](../render.yaml); -4. add it to [`.env.example`](../.env.example) and - `SERVICE_ENV` in `scripts/lib/requirements.ts`; -5. set the key in Render; and -6. run `npm run doctor -- --service workflows`. +Every harness gives the model the factory's own tools, so an agent's tools and +the read-only shell policy behave the same on any harness. Use a key dedicated to this deployment with a provider-enforced spend limit. The factory has no hard dollar budget. @@ -124,11 +134,12 @@ The factory has no hard dollar budget. | Change | Required action | |---|---| | Repository, label, permission, or publishing limit | Deploy both services | -| Model mapping | Deploy `factory-workflows` | -| Prompt or Workflows service code | Deploy `factory-workflows` | +| Agent harness or model | Deploy `factory-workflows` | +| Prompt, agent definition, or Workflows service code | Deploy `factory-workflows` | | Gateway routing or validation | Deploy `factory-gateway` | | Label, App permission, or subscribed event | Run `npm run setup:github`; update the App when prompted | -| `app/runner/` or `sandbox/` | Deploy Workflows, then refresh each repository snapshot | +| `app/runner/` or `app/harness/` | Deploy `factory-workflows`; each Sandbox gets the new runner bundle when it opens | +| `sandbox/` | Deploy `factory-workflows`, then refresh each repository snapshot | | Repository `install` command or lockfiles | Refresh the snapshot, or let the next task reinstall | ## Snapshots @@ -261,7 +272,7 @@ service. Every other deployed value comes from an | `RENDER_WORKSPACE_ID` | Workspace that owns Sandboxes and snapshots | | `GITHUB_APP_ID` | Authenticate the GitHub App | | `GITHUB_APP_PRIVATE_KEY` | Mint installation tokens | -| `ANTHROPIC_API_KEY` | Call the default models | +| `ANTHROPIC_API_KEY` | Call the default models; each agent's harness declares the key it needs | | `GITHUB_API_URL` | Optional GitHub API base URL | `GITHUB_API_URL` is not in the Blueprint. GitHub Enterprise Server users must diff --git a/docs/runbooks/extend.md b/docs/runbooks/extend.md index 082b84c..8518e0b 100644 --- a/docs/runbooks/extend.md +++ b/docs/runbooks/extend.md @@ -1,7 +1,7 @@ # Extension runbook -Use this runbook when adding an agent, a Render Workflows task, or a GitHub -trigger. It is written for coding agents working in a fork of +Use this runbook when adding an agent, an agent harness, a Render Workflows +task, or a GitHub trigger. It is written for coding agents working in a fork of software-factory. Read [Architecture](../architecture.md) first. Do not change the authority @@ -15,6 +15,7 @@ Inspect: - `factory.config.ts`; - `app/agents/`; +- `app/harness/`; - `app/workflows/`; - `app/schemas/`; - `app/gateway/routes.ts`; @@ -29,6 +30,8 @@ Decide which change you need: - **Agent only:** a new prompt, tool set, and typed result used by an existing task. +- **Agent harness:** another agent library or model client behind the runner + contract. - **Claiming task:** work attached to one issue or pull request that must stop when another run takes over. - **Plain task:** maintenance work without an issue or pull request claim, @@ -73,10 +76,12 @@ app/agents// Define the agent: ```typescript +import { medium } from "../models.js"; + export const finder = defineAgent({ id: "finder", prompt: "finder/prompt.md", - model: "medium", + harness: medium, tools: ["read", "ls", "bash"], output: Finding, maxTurns: 40, @@ -87,21 +92,24 @@ The definition fixes the agent's authority: - `id` must be unique and stable; - `prompt` is relative to `app/agents/`; -- `model` is `medium` or `large`; +- `harness` is a value from `app/agents/models.ts` or one built with an + adapter from `app/harness/`; - `tools` may include `read`, `ls`, `bash`, `edit`, and `write`; -- `output` is a bounded strict Zod object; -- `maxTurns` is a positive model-call limit; and -- `thinking` is optional. +- `output` is a bounded strict Zod object; and +- `maxTurns` is a positive model-call limit. -Grant `edit` or `write` only when the agent must change the checkout. The -read-only shell policy in `app/runner/tools.ts` guides read-only agents; it is -not a security boundary. +Grant `edit` or `write` only when the agent must change the checkout. +The read-only shell policy in `app/runner/tools/shell-policy.ts` guides +read-only agents; it is not a security boundary. The prompt should explain the job, evidence requirements, limits, and when to call `submit`. Put repository-, issue-, and run-specific data in the message passed to `runAgent`, not in the prompt. -Export the agent from `app/agents/index.ts`. +Export the agent from `app/agents/index.ts` and add it to `AGENTS` in +`app/agents/registry.ts`. The Sandbox runner is bundled from that list and +runs only the agents in it. Agent modules are bundled into the runner, so they +must not import host-only code. ### 3. Call the agent @@ -122,6 +130,34 @@ list only if write tools are intentional. Add task-level tests for the user-visible result and meaningful failures. Use fakes; tests must not call GitHub, Render, or a model provider. +## Add an agent harness + +A harness puts an agent library behind the runner contract in +[`app/runner/core/`](../../app/runner/core/). The runner calls it inside the +Sandbox with a validated input and the key from stdin; the host reads only what +it declares. + +1. Add an adapter in `app/harness/.ts` that takes the library's own + configuration and returns a `Harness` (`app/harness/types.ts`): + - `name` and `label` name the library and model before a run; + - `credential` is the env var on the Workflows service that holds its key; + the host sends its value into the Sandbox; and + - `run` and `selfTest` are the runtime (`app/runner/core/harness.ts`), + which the runner calls inside the Sandbox. +2. Give the model only the factory's tools, through `createFactoryTools` + (`app/runner/tools/`), and a `submit` tool checked by `Submission` + (`app/runner/core/submit.ts`). Never offer the library's own shell, file, + network, or subagent tools: their authority isn't the factory's. +3. Emit `started`, then exactly one `result` or `error`, and stop at + `max_turns` and `timeout_ms`. +4. Honor `testScript` (set by `--test-stub`) by playing its turns instead of + calling a model, if the library has a fake model; then add the harness to + `HARNESSES` in `tests/helpers/runner.ts` so it runs the conformance suite. +5. Export the adapter from `app/harness/index.ts` and use it in an agent + definition or `app/agents/models.ts`. + +`tests/agents-catalog.test.ts` fails if the bundle imports host-only code. + ## Add a GitHub task Use `defineGitHubTask()` for work attached to one issue or pull request. It @@ -287,7 +323,8 @@ Before handing off, confirm: - the new agent has the minimum tools it needs; - model output cannot select an external write target; - the gateway still imports no GitHub auth, Sandbox, agent, or workflow code; -- `app/runner/` still imports nothing from the rest of `app/`; +- the runner bundle (`app/runner/`, `app/harness/`, agent definitions, and + schemas) still imports no host-only code; - GitHub tasks record their Sandbox before running an agent; - tests cover the user-visible success and important failures; and - documentation and setup requirements changed when the App contract changed. @@ -301,7 +338,12 @@ Before handing off, confirm: `scripts/lib/requirements.ts`. Add it to `.env`, rerun `npm run setup:render`, and redeploy the services that link its group. See [Environment groups](../configuration.md#environment-groups). -- Changes under `app/runner/` or `sandbox/` also require a snapshot refresh for +- A harness whose key is new needs the key in `.env`, `npm run setup:render`, + and a Workflows deploy. Setup and `doctor` derive the key list from the + agents' harnesses. +- Changes under `app/runner/`, `app/harness/`, or agent definitions reach + Sandboxes with the Workflows deploy: each Sandbox gets the new runner bundle + when it opens. Changes under `sandbox/` also require a snapshot refresh for every configured repository. State the required deploy and refresh actions in the handoff. diff --git a/factory.config.ts b/factory.config.ts index 5594ecc..2444fbf 100644 --- a/factory.config.ts +++ b/factory.config.ts @@ -1,5 +1,3 @@ -export type ModelTier = "medium" | "large"; - export interface RepoConfig { /** Branch that specs are implemented against and snapshots are built from. */ baseBranch: string; @@ -17,8 +15,6 @@ export interface FactoryConfig { commandPrefix: string; /** GitHub permission the triggering actor needs on the repository. */ minimumPermission: "triage" | "write" | "maintain" | "admin"; - /** "provider/model" per tier, from Pi's model registry. */ - models: Record; limits: { /** Fix rounds after a failed verify in Spec-to-PR. */ repairRounds: number; @@ -53,10 +49,6 @@ export default defineFactoryConfig({ }, commandPrefix: "/factory", minimumPermission: "write", - models: { - medium: "anthropic/claude-sonnet-5", - large: "anthropic/claude-opus-5", - }, limits: { repairRounds: 2, maxChangedFiles: 200, diff --git a/scripts/doctor.ts b/scripts/doctor.ts index ac51ef8..5605f83 100644 --- a/scripts/doctor.ts +++ b/scripts/doctor.ts @@ -26,7 +26,7 @@ if (!services) { const out = new Reporter(); try { - process.exitCode = await runDoctor({ env: process.env, services, repos: Object.keys(config.repos), models: config.models }, out); + process.exitCode = await runDoctor({ env: process.env, services, repos: Object.keys(config.repos) }, out); } catch (error) { out.fail(`doctor crashed: ${errorMessage(error)}`); process.exitCode = 1; diff --git a/scripts/lib/doctor.ts b/scripts/lib/doctor.ts index 9862312..3c52ccb 100644 --- a/scripts/lib/doctor.ts +++ b/scripts/lib/doctor.ts @@ -1,35 +1,28 @@ /** doctor: read-only checks of one or both services' environment, the GitHub App, and Workflows tasks. */ +import type { Agent } from "../../app/agents/define.js"; +import { AGENTS } from "../../app/agents/registry.js"; import { TASKS } from "../../app/config/index.js"; -import { PROVIDER_KEY_ENV } from "../../app/runner/keys.js"; import { appCredentialsFrom, type Env, GitHubApp, type ScriptDeps, splitRepo } from "./github.js"; import { PLACEHOLDER_WEBHOOK_URL } from "./manifest.js"; import { errorMessage, type Reporter } from "./output.js"; -import { - envGroupFor, - LABELS, - missingEvents, - missingPermissions, - SECRET_ENV, - SERVICE_ENV, - type Service, - WEBHOOK_PATH, -} from "./requirements.js"; +import { envGroupFor, LABELS, missingEvents, missingPermissions, SECRET_ENV, SERVICE_ENV, type Service, WEBHOOK_PATH } from "./requirements.js"; import { appSettingsUrl } from "./setup-github.js"; export interface DoctorOptions extends ScriptDeps { services: readonly Service[]; repos: readonly string[]; - /** config.models: "provider/model" per tier. */ - models: Readonly>; + /** The agents whose harnesses to check; defaults to every agent the factory runs. */ + agents?: readonly Agent[]; renderApiUrl?: string; } export async function runDoctor(opts: DoctorOptions, out: Reporter): Promise { const { env } = opts; - out.addSecrets(...SECRET_ENV.map((name) => env[name]), ...modelKeys(opts.models).map((name) => env[name])); + const agents = opts.agents ?? AGENTS; + out.addSecrets(...SECRET_ENV.map((name) => env[name]), ...agents.map((agent) => env[agent.harness.credential])); checkEnv(opts, out); - if (opts.services.includes("workflows")) checkModels(opts, out); + if (opts.services.includes("workflows")) checkHarnesses(env, agents, out); await checkGitHub(opts, out); await checkTasks(opts, out); return out.summary(); @@ -55,41 +48,20 @@ function setFix(service: Service, name: string): string { return `Set it in the ${group.name} env group: add it to .env and run \`npm run setup:render\``; } -/** The env var each configured provider's key is read from, in config order. */ -function modelKeys(models: Readonly>): string[] { - const names = Object.values(models) - .map((id) => PROVIDER_KEY_ENV[id.split("/")[0] ?? ""]?.[0]) - .filter((name): name is string => !!name); - return [...new Set(names)]; -} - -function checkModels({ env, models }: DoctorOptions, out: Reporter): void { - out.heading("Model keys"); - const tiersByProvider = new Map(); - for (const [tier, id] of Object.entries(models)) { - const slash = id.indexOf("/"); - if (slash <= 0) { - out.fail(`models.${tier} is not "provider/model": ${id}`); - continue; - } - const provider = id.slice(0, slash); - tiersByProvider.set(provider, [...(tiersByProvider.get(provider) ?? []), tier]); +/** Each harness the agents use, the agents on it, and whether its key is set. */ +function checkHarnesses(env: Env, agents: readonly Agent[], out: Reporter): void { + out.heading("Agent harnesses"); + const groups = new Map(); + for (const agent of agents) { + const key = `${agent.harness.name} ${agent.harness.label}`; + const group = groups.get(key) ?? { agent, ids: [] }; + group.ids.push(agent.id); + groups.set(key, group); } - for (const [provider, tiers] of tiersByProvider) { - const keyName = PROVIDER_KEY_ENV[provider]?.[0]; - const label = `${provider} (${tiers.join(", ")})`; - if (!keyName) { - out.fail(`${label}: unknown provider`, "Use a provider from app/runner/keys.ts"); - continue; - } - if (env[keyName]?.trim()) out.pass(`${label}: ${keyName} is set`); - else out.fail(`${label}: ${keyName} is not set`, `${setFix("workflows", keyName)}; use a spend-capped key`); - if (!SERVICE_ENV.workflows.includes(keyName)) { - out.warn( - `setup:render doesn't manage ${keyName}`, - "Add it to the software-factory-workflows group in ENV_GROUPS and SERVICE_ENV (scripts/lib/requirements.ts)", - ); - } + for (const [label, { agent, ids }] of groups) { + const { credential } = agent.harness; + if (env[credential]?.trim()) out.pass(`${label} (${ids.join(", ")}): ${credential} is set`); + else out.fail(`${label} (${ids.join(", ")}): ${credential} is not set`, `${setFix("workflows", credential)}; use a spend-capped key`); } } diff --git a/scripts/lib/requirements.ts b/scripts/lib/requirements.ts index 6a6f9b7..1765e93 100644 --- a/scripts/lib/requirements.ts +++ b/scripts/lib/requirements.ts @@ -1,4 +1,5 @@ /** What the factory needs from GitHub and from each service's environment, in one place for setup and doctor. */ +import { AGENTS } from "../../app/agents/registry.js"; import { config } from "../../app/config/index.js"; import { WORKING_LABEL } from "../../app/claims/status.js"; @@ -60,10 +61,13 @@ export const LABELS: readonly LabelSpec[] = [ export type Service = "gateway" | "workflows"; +/** The credential each agent's harness declares (app/agents/), once each, in agent order. Workflows sends them to its sandboxes. */ +export const HARNESS_KEY_ENV: readonly string[] = [...new Set(AGENTS.map((agent) => agent.harness.credential))]; + /** Each service's env vars, exactly as render.yaml declares them (through its env groups or directly). */ export const SERVICE_ENV: Readonly> = { gateway: ["GITHUB_WEBHOOK_SECRET", "RENDER_API_KEY", "RENDER_WORKFLOW_SLUG"], - workflows: ["RENDER_API_KEY", "RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", "ANTHROPIC_API_KEY"], + workflows: ["RENDER_API_KEY", "RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", ...HARNESS_KEY_ENV], }; export interface EnvGroupSpec { @@ -84,7 +88,7 @@ export const ENV_GROUPS: readonly EnvGroupSpec[] = [ { name: "software-factory-workflows", services: ["workflows"], - keys: ["RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", "ANTHROPIC_API_KEY"], + keys: ["RENDER_WORKSPACE_ID", "GITHUB_APP_ID", "GITHUB_APP_PRIVATE_KEY", ...HARNESS_KEY_ENV], }, ]; diff --git a/tests/agents-catalog.test.ts b/tests/agents-catalog.test.ts index b3f8989..0cbc203 100644 --- a/tests/agents-catalog.test.ts +++ b/tests/agents-catalog.test.ts @@ -1,8 +1,25 @@ import { describe, expect, it } from "vitest"; -import { implementer, prReviewer, specWriter, triage } from "../app/agents/index.js"; +import { AGENTS, implementer, prReviewer, specWriter, triage } from "../app/agents/index.js"; +import { bundleInputs } from "../app/runner/bundle.js"; const agents = [triage, specWriter, implementer, prReviewer]; +describe("agent registry", () => { + it("bundles every built-in agent, each with a non-empty prompt", () => { + expect(AGENTS.map((agent) => agent.id)).toEqual(agents.map((agent) => agent.id)); + for (const agent of AGENTS) expect(agent.prompt.trim(), agent.id).not.toBe(""); + }); + + it("keeps host-only code out of the sandbox runner bundle", { timeout: 60_000 }, async () => { + const app = (await bundleInputs()).filter((path) => !path.includes("node_modules/")); + const hostOnly = app.filter( + (path) => /^app\/(github|sandbox|workflows|claims|gateway|events|config|entrypoints)\//.test(path) || path === "app/agents/run.ts", + ); + expect(hostOnly).toEqual([]); + expect(app).toContain("app/agents/registry.ts"); + }); +}); + describe("agent authority", () => { it("gives every agent a unique id and prompt file", () => { const ids = agents.map((agent) => agent.id); diff --git a/tests/agents.test.ts b/tests/agents.test.ts index 08bbf82..5bc3ac4 100644 --- a/tests/agents.test.ts +++ b/tests/agents.test.ts @@ -1,8 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { z } from "zod"; -import { defineAgent, runAgent } from "../app/agents/index.js"; +import { defineAgent, large, runAgent } from "../app/agents/index.js"; import { emit } from "../app/events/index.js"; -import { RunnerInput } from "../app/runner/protocol.js"; +import { PROTOCOL_VERSION, RunnerInput } from "../app/runner/core/protocol.js"; import { REPO_DIR } from "../app/sandbox/types.js"; import { eventLine, FakeSandbox } from "./helpers/fake-sandbox.js"; @@ -20,11 +20,10 @@ const Report = z.object({ const reviewer = defineAgent({ id: "reviewer", prompt: PROMPT, - model: "large", + harness: large, tools: ["read", "ls", "bash"], output: Report, maxTurns: 40, - thinking: "high", }); const emitted = () => vi.mocked(emit).mock.calls.map(([e]) => e); @@ -48,7 +47,7 @@ describe("defineAgent", () => { const cannedRun = (output: unknown) => [ - eventLine({ type: "started", agent_id: "reviewer", model: "anthropic/claude-opus-5", pi_version: "0.83.0" }), + eventLine({ type: "started", agent_id: "reviewer", harness: "pi", harness_version: "0.83.0", model: "anthropic/claude-opus-5" }), eventLine({ type: "usage", usage: { input_tokens: 1000, output_tokens: 200, cache_read_tokens: 50, cache_write_tokens: 0, cost_usd: 0.01 } }), eventLine({ type: "tool_call", tool_call_id: "c1", tool: "ls", args: {} }), eventLine({ type: "usage", usage: { input_tokens: 1500, output_tokens: 100, cache_read_tokens: 0, cache_write_tokens: 10, cost_usd: 0.02 } }), @@ -76,6 +75,7 @@ describe("runAgent in a sandbox", () => { expect(sandbox.uploads.get(`${runDir}/key`)).toBe(HOST_KEY); const input = RunnerInput.parse(JSON.parse(sandbox.uploads.get(`${runDir}/input.json`) as string)); expect(input).toMatchObject({ + protocol_version: PROTOCOL_VERSION, agent_id: "reviewer", system_prompt: reviewer.prompt, message: "Review it.", @@ -89,7 +89,15 @@ describe("runAgent in a sandbox", () => { expect(emitted()[0]).toMatchObject({ type: "agent.finished", traceId: "trace-2", - payload: { outcome: "ok", turns: 2, model: "anthropic/claude-opus-5", costUsd: run.costUsd, usage: run.usage }, + payload: { + outcome: "ok", + turns: 2, + harness: "pi", + harnessVersion: "0.83.0", + model: "anthropic/claude-opus-5", + costUsd: run.costUsd, + usage: run.usage, + }, }); }); }); diff --git a/tests/helpers/fake-sandbox.ts b/tests/helpers/fake-sandbox.ts index 74f1ff9..c6f986d 100644 --- a/tests/helpers/fake-sandbox.ts +++ b/tests/helpers/fake-sandbox.ts @@ -1,4 +1,4 @@ -import type { RunnerEvent } from "../../app/runner/protocol.js"; +import type { RunnerEvent } from "../../app/runner/core/protocol.js"; import type { ExecOptions, ExecResult, Sandbox } from "../../app/sandbox/types.js"; type WithoutAt = E extends unknown ? Omit : never; diff --git a/tests/helpers/runner.ts b/tests/helpers/runner.ts index 00116c0..9d2fc4c 100644 --- a/tests/helpers/runner.ts +++ b/tests/helpers/runner.ts @@ -4,13 +4,18 @@ import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Readable } from "node:stream"; -import { fauxAssistantMessage as testStubAssistantMessage, fauxToolCall as testStubToolCall } from "@earendil-works/pi-ai"; +import { getBuiltinModel } from "@earendil-works/pi-ai/providers/all"; +import { z } from "zod"; +import { type Agent, defineAgent } from "../../app/agents/define.js"; +import { pi } from "../../app/harness/index.js"; +import type { Harness } from "../../app/harness/types.js"; +import { PROTOCOL_VERSION, RunnerEvent, type RunnerInput, type TestTurn, type ToolName } from "../../app/runner/core/protocol.js"; import { bundleRunner } from "../../app/runner/bundle.js"; import { main } from "../../app/runner/main.js"; -import { RunnerEvent, type RunnerInput, type ToolName } from "../../app/runner/protocol.js"; -import type { TestStub } from "../../app/runner/test-stub.js"; +import { scriptedHarness } from "./scripted-harness.js"; export const SUBMIT = "submit"; +export const TEST_AGENT_ID = "test-agent"; /** The output schema most runner tests use: what `z.toJSONSchema` emits for a small object. */ const OUTPUT_SCHEMA = { @@ -26,6 +31,24 @@ const OUTPUT_SCHEMA = { } as const; export const GOOD = { summary: "All good.", files: ["README.md"], confidence: "high" }; +/** A test agent on `harness`; the runner takes its tools and schema from the input, so these are placeholders. */ +export function testAgent(harness: Harness): Agent { + return defineAgent({ + id: TEST_AGENT_ID, + prompt: "../../tests/fixtures/agents/echo.md", + harness, + tools: ["read", "ls", "bash", "edit", "write"], + output: z.object({ summary: z.string() }), + maxTurns: 20, + }); +} + +/** Every harness the conformance suite runs against. */ +export const HARNESSES: ReadonlyArray<{ name: string; harness: Harness }> = [ + { name: "pi", harness: pi({ model: getBuiltinModel("anthropic", "claude-sonnet-5"), thinkingLevel: "off" }) }, + { name: "scripted", harness: scriptedHarness }, +]; + export interface Workspace { cwd: string; scratchDir: string; @@ -55,13 +78,11 @@ export function makeWorkspace(): Workspace { export function makeInput(ws: Workspace | undefined, overrides: Partial = {}): RunnerInput { const tools: ToolName[] = overrides.tools ?? (ws ? ["read", "ls", "bash"] : []); return { + protocol_version: PROTOCOL_VERSION, trace_id: randomUUID(), - agent_id: "test-agent", + agent_id: TEST_AGENT_ID, system_prompt: "You are a test agent.", message: "Do the thing.", - model_provider: "anthropic", - model: "claude-sonnet-4-5", - thinking_level: "off", tools, output_schema: OUTPUT_SCHEMA, max_turns: 20, @@ -89,22 +110,28 @@ export function terminal(events: RunnerEvent[]) { export const toolResults = (events: RunnerEvent[]) => events.flatMap((e) => (e.type === "tool_result" ? [e] : [])); -/** Runs main() in-process with deterministic model responses. */ -export async function runWithTestStub(input: unknown, testStub: TestStub, extraArgs: string[] = []) { +/** One scripted model turn that makes one tool call. */ +export const call = (name: string, args: Record): TestTurn => ({ calls: [{ name, args }] }); + +/** + * Runs main() in-process through the runner contract (stdin in, JSONL out), with `harness` behind the test agent + * and `turns` as the scripted model. `input` may be a string to control stdin exactly (e.g. `--key-stdin`). + */ +export async function runContract(harness: Harness, input: RunnerInput | string, turns?: TestTurn[], extraArgs: string[] = []) { + let stdin: string; + if (typeof input === "string") stdin = input; + else stdin = JSON.stringify(turns ? { ...input, test_script: turns } : input); let stdout = ""; const code = await main(["--test-stub", ...extraArgs], { - stdin: Readable.from([typeof input === "string" ? input : JSON.stringify(input)]), + stdin: Readable.from([stdin]), write: async (chunk) => { stdout += chunk; }, - testStub, + agents: [testAgent(harness)], }); return { code, stdout, events: parseEvents(stdout) }; } -const toolUse = { stopReason: "toolUse" } as const; -export const call = (name: string, args: Record) => testStubAssistantMessage(testStubToolCall(name, args), toolUse); - let runnerPath: string | undefined; /** Bundles the runner into a fresh temp dir, outside the repo, so no node_modules is reachable from it. */ diff --git a/tests/helpers/scripted-harness.ts b/tests/helpers/scripted-harness.ts new file mode 100644 index 0000000..5d91813 --- /dev/null +++ b/tests/helpers/scripted-harness.ts @@ -0,0 +1,80 @@ +import { previewText } from "../../app/runner/core/emit.js"; +import type { Harness } from "../../app/harness/types.js"; +import type { HarnessRun } from "../../app/runner/core/harness.js"; +import { RUNNER_EXIT } from "../../app/runner/core/protocol.js"; +import { SUBMIT_ACCEPTED, SUBMIT_DUPLICATE, SUBMIT_TOOL, Submission } from "../../app/runner/core/submit.js"; +import { createFactoryTools } from "../../app/runner/tools/index.js"; + +/** + * A harness with no agent library: it plays the test script through the shared factory tools and Submission. It + * stands in for a third-party library in the conformance suite, proving a harness can be swapped without host or + * runner changes. + */ +export const scriptedHarness: Harness = { + name: "scripted", + label: "scripted", + credential: "SCRIPTED_API_KEY", + run: runScripted, + selfTest: async () => ({ version: "1.0.0", problems: [] }), +}; + +async function runScripted({ input, emit, testScript }: HarnessRun): Promise { + emit({ type: "started", agent_id: input.agent_id, harness: "scripted", harness_version: "1.0.0", model: "scripted" }); + if (!testScript) { + emit({ type: "error", code: "agent_error", message: "the scripted harness runs only with --test-stub" }); + return RUNNER_EXIT.error; + } + const tools = new Map( + input.cwd && input.scratch_dir + ? createFactoryTools({ tools: input.tools, cwd: input.cwd, scratchDir: input.scratch_dir }).map((t) => [t.name as string, t]) + : [], + ); + const submission = new Submission(input.output_schema); + let turns = 0; + let callId = 0; + for (const turn of testScript) { + if (turn.text) emit({ type: "assistant_text", text: turn.text }); + for (const call of turn.calls) { + const id = `call-${++callId}`; + emit({ type: "tool_call", tool_call_id: id, tool: call.name, args: call.args }); + const result = (text: string, isError: boolean) => + emit({ type: "tool_result", tool_call_id: id, tool: call.name, is_error: isError, output_preview: previewText(text) }); + if (call.name === SUBMIT_TOOL) { + if (submission.accepted) { + result(SUBMIT_DUPLICATE, false); + continue; + } + const verdict = submission.check(call.args); + if (verdict.ok) { + submission.accept(verdict.checked); + result(SUBMIT_ACCEPTED, false); + continue; + } + result(verdict.message, true); + if (verdict.exhausted) { + emit({ type: "error", code: "invalid_submission", message: verdict.exhausted }); + return RUNNER_EXIT.error; + } + continue; + } + const tool = tools.get(call.name); + if (!tool) { + result(`Tool ${call.name} not found`, true); + continue; + } + const out = await tool.execute(call.args); + result(out.text, out.isError); + } + turns++; + if (submission.accepted) { + emit({ type: "result", output: submission.accepted.output, turns }); + return RUNNER_EXIT.ok; + } + if (turns >= input.max_turns) { + emit({ type: "error", code: "max_turns", message: `turn limit (${input.max_turns}) reached without a submission` }); + return RUNNER_EXIT.error; + } + } + emit({ type: "error", code: "no_submission", message: `the model stopped without calling ${SUBMIT_TOOL}` }); + return RUNNER_EXIT.error; +} diff --git a/tests/runner.test.ts b/tests/runner.test.ts index b59cc80..f015ebb 100644 --- a/tests/runner.test.ts +++ b/tests/runner.test.ts @@ -2,16 +2,18 @@ import { existsSync, readFileSync } from "node:fs"; import { rm } from "node:fs/promises"; import { dirname, join } from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { rejectReason } from "../app/runner/shell-policy.js"; +import { PROTOCOL_VERSION } from "../app/runner/core/protocol.js"; +import { rejectReason } from "../app/runner/tools/shell-policy.js"; import { buildRunner, call, GOOD, + HARNESSES, makeInput, makeWorkspace, parseEvents, runCli, - runWithTestStub, + runContract, SUBMIT, terminal, toolResults, @@ -25,22 +27,24 @@ beforeAll(async () => { afterAll(() => rm(runnerDir, { recursive: true, force: true })); -const ENV_KEYS = ["ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "OPENAI_API_KEY", "RENDER_API_KEY"]; +const ENV_KEYS = ["ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "OPENAI_API_KEY", "CURSOR_API_KEY", "RENDER_API_KEY"]; afterEach(() => { for (const name of ENV_KEYS) delete process.env[name]; }); +// The bundled runner runs only the agents bundled into it; triage is one, on Pi. describe("bundled runner (runner.mjs)", { timeout: 60_000 }, () => { - it("--test-stub lists the working directory and submits a value built from the output schema", () => { + it("--test-stub runs a bundled agent on its harness: lists the working directory and submits a schema-built value", () => { const ws = makeWorkspace(); - const { code, stdout, stderr } = runCli(["--test-stub"], JSON.stringify(makeInput(ws))); + const { code, stdout, stderr } = runCli(["--test-stub"], JSON.stringify(makeInput(ws, { agent_id: "triage" }))); const events = parseEvents(stdout); expect(events[0]).toEqual({ type: "started", at: expect.any(String), - agent_id: "test-agent", + agent_id: "triage", + harness: "pi", + harness_version: "0.83.0", model: "test-stub/test-stub", - pi_version: "0.83.0", }); expect(toolResults(events).map((r) => [r.tool, r.is_error])).toEqual([ ["ls", false], @@ -52,7 +56,7 @@ describe("bundled runner (runner.mjs)", { timeout: 60_000 }, () => { }); it("runs an agent with no tools and no cwd: the model gets only submit", () => { - const { code, stdout } = runCli(["--test-stub"], JSON.stringify(makeInput(undefined))); + const { code, stdout } = runCli(["--test-stub"], JSON.stringify(makeInput(undefined, { agent_id: "triage" }))); const events = parseEvents(stdout); expect(toolResults(events).map((r) => r.tool)).toEqual([SUBMIT]); expect(terminal(events).type).toBe("result"); @@ -62,20 +66,49 @@ describe("bundled runner (runner.mjs)", { timeout: 60_000 }, () => { it("--key-stdin takes the key from the first line of stdin and the input from the rest, and never echoes the key", () => { const ws = makeWorkspace(); const key = "sk-ant-test-stdin-0123456789abcdef"; - const { code, stdout, stderr } = runCli(["--test-stub", "--key-stdin"], ` ${key} \r\n${JSON.stringify(makeInput(ws))}`); + const input = JSON.stringify(makeInput(ws, { agent_id: "triage" })); + const { code, stdout, stderr } = runCli(["--test-stub", "--key-stdin"], ` ${key} \r\n${input}`); expect(terminal(parseEvents(stdout))).toMatchObject({ type: "result", output: { summary: "[test-stub]" } }); expect(stdout + stderr).not.toContain(key); expect(code).toBe(0); }); + + it("rejects an agent the bundle doesn't define, input from another protocol version, and a script without --test-stub", () => { + const unknown = runCli(["--test-stub"], JSON.stringify(makeInput(undefined, { agent_id: "nope" }))); + expect(terminal(parseEvents(unknown.stdout))).toMatchObject({ code: "invalid_input", message: expect.stringContaining('unknown agent "nope"') }); + expect(unknown.code).toBe(2); + + const old = runCli(["--test-stub"], JSON.stringify({ ...makeInput(undefined, { agent_id: "triage" }), protocol_version: 1 })); + expect(terminal(parseEvents(old.stdout))).toMatchObject({ + code: "invalid_input", + message: expect.stringContaining(`speaks protocol ${PROTOCOL_VERSION}, but the input is protocol 1`), + }); + + const scripted = runCli([], JSON.stringify({ ...makeInput(undefined, { agent_id: "triage" }), test_script: [call(SUBMIT, GOOD)] })); + expect(terminal(parseEvents(scripted.stdout))).toMatchObject({ code: "invalid_input", message: "test_script is only honored with --test-stub" }); + }); + + it("--self-test reports the protocol and every bundled harness", () => { + const { code, stdout } = runCli(["--self-test"], ""); + const report = JSON.parse(stdout.trim()); + expect(report).toMatchObject({ ok: true, protocol_version: PROTOCOL_VERSION }); + expect(report.harnesses).toEqual([ + { name: "pi", label: "anthropic/claude-sonnet-5", version: "0.83.0" }, + { name: "pi", label: "anthropic/claude-opus-5", version: "0.83.0" }, + ]); + expect(code).toBe(0); + }); }); -describe("submit", { timeout: 30_000 }, () => { - it("returns every problem to the model, and accepts the corrected submission", async () => { +// The runner contract every harness must honor. A new harness joins HARNESSES in tests/helpers/runner.ts. +describe.each(HARNESSES)("$name harness conformance", { timeout: 30_000 }, ({ harness }) => { + it("returns every submit problem to the model, and accepts the corrected submission", async () => { const ws = makeWorkspace(); - const { code, events } = await runWithTestStub(makeInput(ws), () => [ + const { code, events } = await runContract(harness, makeInput(ws), [ call(SUBMIT, { summary: "x", files: "README.md", confidence: "maybe", extra: 1 }), call(SUBMIT, GOOD), ]); + expect(events[0]).toMatchObject({ type: "started", harness: harness.name, harness_version: (await harness.selfTest()).version }); const [first, second] = toolResults(events); expect(first).toMatchObject({ tool: SUBMIT, is_error: true }); expect(first?.output_preview).toContain("attempt 1 of 3"); @@ -85,10 +118,10 @@ describe("submit", { timeout: 30_000 }, () => { expect(code).toBe(0); }); - it("stops after the last rejected attempt and says what was wrong with it", async () => { + it("stops after the last rejected submission and says what was wrong with it", async () => { const ws = makeWorkspace(); const bad = call(SUBMIT, { summary: "ok summary", files: "README.md" }); - const { code, events } = await runWithTestStub(makeInput(ws), () => [bad, bad, bad, call(SUBMIT, GOOD)]); + const { code, events } = await runContract(harness, makeInput(ws), [bad, bad, bad, call(SUBMIT, GOOD)]); const end = terminal(events); expect(end).toMatchObject({ type: "error", code: "invalid_submission" }); const message = end.type === "error" ? end.message : ""; @@ -97,12 +130,10 @@ describe("submit", { timeout: 30_000 }, () => { expect(message).not.toContain("ok summary"); expect(code).toBe(1); }); -}); -describe("run limits", { timeout: 30_000 }, () => { it("stops after max_turns without a submission", async () => { const ws = makeWorkspace(); - const { code, events } = await runWithTestStub(makeInput(ws, { max_turns: 3 }), () => [ + const { code, events } = await runContract(harness, makeInput(ws, { max_turns: 3 }), [ ...Array.from({ length: 5 }, () => call("ls", {})), call(SUBMIT, GOOD), ]); @@ -110,57 +141,80 @@ describe("run limits", { timeout: 30_000 }, () => { expect(toolResults(events)).toHaveLength(3); expect(code).toBe(1); }); -}); -// Tests that inspect a command's environment have it write to a file: Pi stops reading a command's stdout -// 100 ms after the process exits, so under load the tool result can come back empty. -describe("credentials", { timeout: 30_000 }, () => { - it("takes the key from stdin and keeps every secret out of the tool env", async () => { + // Tests that inspect a command's environment have it write to a file: Pi stops reading a command's stdout + // 100 ms after the process exits, so under load the tool result can come back empty. + it("takes the key from stdin and keeps every secret out of the tool env and the output", async () => { const ws = makeWorkspace(); const stdinKey = "sk-ant-test-STDINKEY-0123456789"; process.env.ANTHROPIC_API_KEY = "sk-ant-test-ENVKEY-9876543210"; process.env.ANTHROPIC_OAUTH_TOKEN = "oauth-token-value"; process.env.OPENAI_API_KEY = "sk-openai-not-for-the-model"; + process.env.CURSOR_API_KEY = "cursor-not-for-the-model"; process.env.RENDER_API_KEY = "rnd-not-for-the-model"; - const { code, stdout, events } = await runWithTestStub( - `${stdinKey}\n${JSON.stringify(makeInput(ws, { tools: ["bash", "write"] }))}`, - () => [call("bash", { command: "env > $HOME/env.txt" }), call(SUBMIT, GOOD)], - ["--key-stdin"], - ); + const input = { + ...makeInput(ws, { tools: ["bash", "write"] }), + test_script: [call("bash", { command: "env > $HOME/env.txt" }), call(SUBMIT, GOOD)], + }; + const { code, stdout, events } = await runContract(harness, `${stdinKey}\n${JSON.stringify(input)}`, undefined, ["--key-stdin"]); expect(toolResults(events)[0]?.is_error).toBe(false); const env = readFileSync(join(ws.scratchDir, "env.txt"), "utf8"); expect(env).toContain(`HOME=${ws.scratchDir}`); - for (const leaked of ["sk-ant", "sk-openai", "ANTHROPIC", "OPENAI", "oauth-token-value", "RENDER_API_KEY", "rnd-not"]) + for (const leaked of ["sk-ant", "sk-openai", "ANTHROPIC", "OPENAI", "CURSOR", "oauth-token-value", "RENDER_API_KEY", "rnd-not"]) expect(env).not.toContain(leaked); expect(stdout).not.toContain(stdinKey); expect(process.env.ANTHROPIC_API_KEY).toBeUndefined(); expect(code).toBe(0); }); -}); -describe("tools", { timeout: 30_000 }, () => { - it("read-only agents: bash rejects git push and rm, and runs read-only commands", async () => { + it("read-only agents: bash rejects git push, rm, and redirects into the checkout, and runs read-only commands", async () => { const ws = makeWorkspace(); - const { events } = await runWithTestStub(makeInput(ws), () => [ + const { events } = await runContract(harness, makeInput(ws), [ call("bash", { command: "git push origin main" }), call("bash", { command: "rm README.md" }), call("bash", { command: "echo hi > notes.txt" }), call("bash", { command: "git log -1 --format=%H > $HOME/head.txt" }), + call("write", { path: "notes.txt", content: "not allowed" }), call(SUBMIT, GOOD), ]); - const [push, rmCall, redirect, log] = toolResults(events); + const [push, rmCall, redirect, log, write] = toolResults(events); expect(push).toMatchObject({ is_error: true, output_preview: expect.stringContaining("git push is not allowed") }); expect(rmCall).toMatchObject({ is_error: true, output_preview: expect.stringContaining("rm is only allowed on files in the scratch directory") }); expect(redirect).toMatchObject({ is_error: true, output_preview: expect.stringContaining("redirecting output to notes.txt") }); expect(log).toMatchObject({ is_error: false }); + expect(write).toMatchObject({ tool: "write", is_error: true }); expect(readFileSync(join(ws.scratchDir, "head.txt"), "utf8").trim()).toBe(ws.commit); expect(existsSync(join(ws.cwd, "README.md"))).toBe(true); expect(existsSync(join(ws.cwd, "notes.txt"))).toBe(false); }); - it("the read-only shell policy lets inspection and test commands through and steers away from obvious writes", () => { + it("agents with edit and write create and edit files in cwd, and bash is unrestricted", async () => { + const ws = makeWorkspace(); + const { events } = await runContract(harness, makeInput(ws, { tools: ["read", "bash", "edit", "write"] }), [ + call("write", { path: "src/new.ts", content: "export const added = true;\n" }), + call("edit", { path: "src/app.ts", edits: [{ oldText: "41", newText: "42" }] }), + call("bash", { command: "rm README.md && git status --porcelain > $HOME/status.txt" }), + call("read", { path: "src/app.ts" }), + call(SUBMIT, GOOD), + ]); + expect(toolResults(events).map((r) => [r.tool, r.is_error])).toEqual([ + ["write", false], + ["edit", false], + ["bash", false], + ["read", false], + [SUBMIT, false], + ]); + expect(readFileSync(join(ws.cwd, "src/new.ts"), "utf8")).toBe("export const added = true;\n"); + expect(readFileSync(join(ws.cwd, "src/app.ts"), "utf8")).toBe("export const answer = 42;\n"); + expect(existsSync(join(ws.cwd, "README.md"))).toBe(false); + expect(readFileSync(join(ws.scratchDir, "status.txt"), "utf8")).toContain("D README.md"); + }); +}); + +describe("read-only shell policy", () => { + it("lets inspection and test commands through and steers away from obvious writes", () => { const scratch = "/factory/scratch"; const allowed = [ "rg -n 'a|b; c' src", @@ -185,26 +239,4 @@ describe("tools", { timeout: 30_000 }, () => { for (const command of allowed) expect(rejectReason(command, scratch), command).toBeUndefined(); for (const command of rejected) expect(rejectReason(command, scratch), command).toBeDefined(); }); - - it("agents with edit and write create and edit files in cwd, and bash is unrestricted", async () => { - const ws = makeWorkspace(); - const { events } = await runWithTestStub(makeInput(ws, { tools: ["read", "bash", "edit", "write"] }), () => [ - call("write", { path: "src/new.ts", content: "export const added = true;\n" }), - call("edit", { path: "src/app.ts", edits: [{ oldText: "41", newText: "42" }] }), - call("bash", { command: "rm README.md && git status --porcelain > $HOME/status.txt" }), - call("read", { path: "src/app.ts" }), - call(SUBMIT, GOOD), - ]); - expect(toolResults(events).map((r) => [r.tool, r.is_error])).toEqual([ - ["write", false], - ["edit", false], - ["bash", false], - ["read", false], - [SUBMIT, false], - ]); - expect(readFileSync(join(ws.cwd, "src/new.ts"), "utf8")).toBe("export const added = true;\n"); - expect(readFileSync(join(ws.cwd, "src/app.ts"), "utf8")).toBe("export const answer = 42;\n"); - expect(existsSync(join(ws.cwd, "README.md"))).toBe(false); - expect(readFileSync(join(ws.scratchDir, "status.txt"), "utf8")).toContain("D README.md"); - }); }); diff --git a/tests/sandbox-render.test.ts b/tests/sandbox-render.test.ts index 2b67b07..46cea29 100644 --- a/tests/sandbox-render.test.ts +++ b/tests/sandbox-render.test.ts @@ -1,5 +1,7 @@ +import { createHash } from "node:crypto"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; +import { gunzipSync } from "node:zlib"; import { SandboxSnapshotNotFoundError } from "@renderinc/sdk"; import type { SandboxExecEvent } from "@renderinc/sdk/experimental"; import { list as listTar } from "tar"; @@ -7,7 +9,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { config, type RepoContext } from "../app/config/index.js"; import { buildSnapshot, type OpenSandboxDeps, openSandbox, type SandboxApi, SandboxClient, type SnapshotManifest } from "../app/sandbox/index.js"; +const FAKE_BUNDLE = "// fake runner bundle\n"; vi.mock("../app/runner/bundle.js", () => ({ bundleRunner: async () => "// fake runner bundle\n" })); +const sha256 = (text: string) => createHash("sha256").update(text).digest("hex"); const TOKEN = "ghs_factoryReadToken0000000000000000000"; const BASE_SHA = "1".repeat(40); @@ -346,6 +350,30 @@ describe("openSandbox", () => { expect(fake.calls.at(-1)).toEqual({ op: "terminate", id: "sbx-1" }); }); + it("replaces a stale runner bundle with this host's, root-owned and checked, before verifying the sandbox", async () => { + await openSandbox(repo(), { kind: "branch", name: "main" }, deps()); + const upload = fake.calls.find((c) => c.op === "upload" && /^\/tmp\/factory-runner-[0-9a-f-]{36}\/runner\.mjs\.gz$/.test(c.path)) as { + path: string; + data: Buffer; + }; + expect(gunzipSync(upload.data).toString("utf8")).toBe(FAKE_BUNDLE); + const stage = upload.path.replace(/\/runner\.mjs\.gz$/, ""); + const install = fake.commands().find((c) => c.includes(`install -o root -g root -m 0644 ${stage}/runner.mjs /opt/factory/app/runner/runner.mjs`)); + expect(install).toContain(`= ${sha256(FAKE_BUNDLE)} ]`); + expect(fake.execIndex("install -o root")).toBeLessThan(fake.execIndex("verify.sh")); + expect(fake.commands()).toContain(`rm -rf ${stage}`); + }); + + it("leaves a runner bundle that matches this host's alone", async () => { + fake.overrides = (_id, command) => + command.startsWith("sha256sum /opt/factory/app/runner/runner.mjs") + ? [out(`${sha256(FAKE_BUNDLE)} /opt/factory/app/runner/runner.mjs\n`), exit(0)] + : []; + await openSandbox(repo(), { kind: "branch", name: "main" }, deps()); + expect(fake.calls.some((c) => c.op === "upload" && c.path.includes("factory-runner-"))).toBe(false); + expect(fake.execIndex("install -o root")).toBe(-1); + }); + it("refuses a sandbox where the runner could not be confirmed non-dumpable", async () => { fake.overrides = (_id, command) => (command.includes("verify.sh") ? [out(info({ runnerNonDumpable: false })), exit(0)] : []); await expect(openSandbox(repo(), { kind: "branch", name: "main" }, deps())).rejects.toThrow("non-dumpable"); diff --git a/tests/scripts-doctor.test.ts b/tests/scripts-doctor.test.ts index 7d3a6ae..1766e30 100644 --- a/tests/scripts-doctor.test.ts +++ b/tests/scripts-doctor.test.ts @@ -105,7 +105,6 @@ async function doctor(overrides: Partial & { world?: World } = {} sleep: async () => {}, services: ["gateway", "workflows"], repos: ["acme/web"], - models: { medium: "anthropic/sonnet", large: "anthropic/opus" }, ...rest, }, out,