Add a Pydantic AI harness adapter

.github/workflows/ci.yml

@@ -23,6 +23,26 @@ jobs:
       - run: shellcheck sandbox/*.sh sandbox/build/*.sh
       - run: npm audit --omit=dev --audit-level=high
 
+  # The pydantic harness's leg of the runner conformance suite, against the locked Python environment.
+  python:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .node-version
+          cache: npm
+      - uses: astral-sh/setup-uv@v6
+      - run: npm ci
+      - name: Install the locked Python environment
+        working-directory: python
+        run: uv sync --frozen --python 3.11
+      - name: Check that requirements.txt matches uv.lock
+        working-directory: python
+        run: uv export --frozen --no-header --format requirements-txt --hashes | diff -u requirements.txt -
+      - run: npx vitest run tests/runner.test.ts tests/harness-pydantic.test.ts
+
   # Validation checks plans, regions, and conflicts against a real workspace, so it needs an API key.
   # Set the RENDER_API_KEY secret and the RENDER_WORKSPACE_ID variable to turn it on; forks without them skip it.
   blueprint:

AGENTS.md

@@ -4,6 +4,9 @@ This file applies to the whole repository.
 
 ## Use the right guide
 
+- Agent libraries, models, or harness adapters:
+  [docs/configuration.md#agent-harnesses-and-models](docs/configuration.md#agent-harnesses-and-models)
+  and [docs/runbooks/extend.md](docs/runbooks/extend.md#add-an-agent-harness)
 - First deployment:
   [docs/runbooks/setup.md](docs/runbooks/setup.md)
 - Repositories, labels, models, permissions, limits, or snapshots:
@@ -31,8 +34,10 @@ database.
 
 Two dependency rules are mandatory:
 
-1. `app/runner/` imports nothing from the rest of `app/`. It is bundled into
-   a self-contained Sandbox program.
+1. The Sandbox runner bundle (`app/runner/`, `app/harness/`, the agent
+   definitions in `app/agents/registry.ts`, and `app/schemas/`) imports no
+   host-only code: GitHub, Sandboxes, Workflows, claims, events, or config.
+   `tests/agents-catalog.test.ts` enforces it.
 2. `app/gateway/` does not import GitHub authentication, Sandboxes, agents, or
    workflows. The gateway holds no GitHub credential and runs no model.
 
@@ -93,8 +98,13 @@ strict output schemas.
   credentials or a model key to the gateway.
 - App permission, event, or label: update
   `scripts/lib/requirements.ts`, setup behavior, and doctor tests.
-- `app/runner/` or `sandbox/`: refresh every repository snapshot after
-  deployment.
+- Agent harness: declare the env var of its key as the adapter's
+  `credential`; setup and `doctor` derive the key list. It must give the
+  model only the factory's tools; add it to `HARNESSES` in
+  `tests/helpers/runner.ts` if it can run the conformance suite.
+- `sandbox/`: refresh every repository snapshot after deployment. Changes to
+  `app/runner/`, `app/harness/`, or agents reach Sandboxes with the Workflows
+  deploy, because each Sandbox replaces a stale runner bundle when it opens.
 
 ## Verify
 

README.md

@@ -58,15 +58,16 @@ questions, the workflow stops and continues when a person replies.
 
 The built-in agents and tasks are examples of the template's extension model:
 
-- Add or replace an agent with a Markdown prompt, a model tier, a bounded Zod
-  output schema, and the minimum tools it needs.
+- Add or replace an agent with a Markdown prompt, a harness (agent library and
+  model), a bounded Zod output schema, and the minimum tools it needs.
+- Run agents on another agent library by adding a harness adapter.
 - Define a GitHub task for work attached to an issue or pull request.
 - Add a plain Workflows task for maintenance or other work without a GitHub
   claim.
 - Route new GitHub events, labels, or commands through the authenticated
   gateway.
-- Change repositories, verification commands, models, permissions, and limits
-  in `factory.config.ts`.
+- Change repositories, verification commands, permissions, and limits in
+  `factory.config.ts`, and agent models in `app/agents/models.ts`.
 
 The Sandbox runner, claim lifecycle, GitHub client, formatting helpers, and
 publishing boundary are reusable building blocks. Start with the

app/agents/define.ts

@@ -1,46 +1,61 @@
 import { readFileSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { z } from "zod";
-import type { ModelTier } from "../config/index.js";
-import type { ThinkingLevel, ToolName } from "../runner/protocol.js";
-import { outputSchemaProblem } from "../runner/submit.js";
+import type { Harness } from "../harness/types.js";
+import type { ToolName } from "../runner/core/protocol.js";
+import { outputSchemaProblem } from "../runner/core/submit.js";
 
 export type OutputSchema = z.ZodObject;
 
 export interface AgentSpec<Tools extends readonly ToolName[], Output extends OutputSchema> {
 	id: string;
 	/** Path to the agent's Markdown prompt, relative to app/agents/. */
 	prompt: string;
-	model: ModelTier;
+	/** The agent library and model that run it, from an adapter in app/harness/. */
+	harness: Harness;
 	/** Tools the agent may call in its sandbox. */
 	tools: Tools;
 	output: Output;
 	maxTurns: number;
-	thinking?: ThinkingLevel;
 }
 
 export interface Agent<Tools extends readonly ToolName[] = readonly ToolName[], Output extends OutputSchema = OutputSchema>
 	extends Omit<AgentSpec<Tools, Output>, "prompt"> {
 	/** Absolute path of the prompt file. */
 	promptPath: string;
-	/** The prompt file's contents, verbatim: the agent's system prompt. */
-	prompt: string;
+	/** The prompt file's contents, verbatim: the agent's system prompt. Read on first use, on the host. */
+	readonly prompt: string;
 	/** `output` as JSON Schema (input side, so fields with defaults are optional): the runner's submit tool takes it. */
 	outputJsonSchema: Record<string, unknown>;
 }
 
 const AGENTS_DIR = new URL("./", import.meta.url);
 
-/** Reads the prompt file once, as a string (no frontmatter, no parsing), and converts the output schema. */
+/**
+ * Converts the output schema. Agent modules are also bundled into the sandbox runner, where prompt files don't
+ * exist, so the prompt is read lazily.
+ */
 export function defineAgent<const Tools extends readonly ToolName[], Output extends OutputSchema>(
 	spec: AgentSpec<Tools, Output>,
 ): Agent<Tools, Output> {
 	if (!Number.isInteger(spec.maxTurns) || spec.maxTurns <= 0) throw new Error(`agent ${spec.id}: maxTurns must be a positive integer`);
 	const promptPath = fileURLToPath(new URL(spec.prompt, AGENTS_DIR));
-	const prompt = readFileSync(promptPath, "utf8");
-	if (!prompt.trim()) throw new Error(`agent ${spec.id}: prompt file ${promptPath} is empty`);
 	const outputJsonSchema = z.toJSONSchema(spec.output, { io: "input" }) as Record<string, unknown>;
 	const problem = outputSchemaProblem(outputJsonSchema);
 	if (problem) throw new Error(`agent ${spec.id}: ${problem}`);
-	return { ...spec, promptPath, prompt, outputJsonSchema };
+	let prompt: string | undefined;
+	const { prompt: _relative, ...rest } = spec;
+	return {
+		...rest,
+		promptPath,
+		outputJsonSchema,
+		get prompt() {
+			if (prompt === undefined) {
+				const text = readFileSync(promptPath, "utf8");
+				if (!text.trim()) throw new Error(`agent ${spec.id}: prompt file ${promptPath} is empty`);
+				prompt = text;
+			}
+			return prompt;
+		},
+	};
 }

app/agents/implementer/index.ts

@@ -1,5 +1,6 @@
 import { Implementation } from "../../schemas/index.js";
 import { defineAgent } from "../define.js";
+import { large } from "../models.js";
 
 /**
  * Implements the approved spec in the sandbox's working tree and runs the verify commands itself. A repair run
@@ -8,7 +9,7 @@ import { defineAgent } from "../define.js";
 export const implementer = defineAgent({
 	id: "implementer",
 	prompt: "implementer/prompt.md",
-	model: "large",
+	harness: large,
 	tools: ["read", "ls", "bash", "edit", "write"],
 	output: Implementation,
 	maxTurns: 150,

app/agents/index.ts

@@ -1,6 +1,8 @@
 export { type Agent, type AgentSpec, defineAgent, type OutputSchema } from "./define.js";
 export { implementer } from "./implementer/index.js";
+export { large, medium } from "./models.js";
 export { prReviewer } from "./pr-reviewer/index.js";
+export { AGENTS } from "./registry.js";
 export { AgentError, type AgentErrorCode, type AgentRun, type RunAgentOptions, runAgent } from "./run.js";
 export { specWriter } from "./spec-writer/index.js";
 export { triage } from "./triage/index.js";

app/agents/models.ts

@@ -0,0 +1,10 @@
+import { getBuiltinModel } from "@earendil-works/pi-ai/providers/all";
+import { pi } from "../harness/index.js";
+
+/**
+ * The harnesses the built-in agents use. Change an agent's library or model here, or give an agent its own
+ * harness in its definition. Each value carries its library's own configuration, so a typo in a model ID is a
+ * type error.
+ */
+export const medium = pi({ model: getBuiltinModel("anthropic", "claude-sonnet-5") });
+export const large = pi({ model: getBuiltinModel("anthropic", "claude-opus-5") });

app/agents/pr-reviewer/index.ts

@@ -1,11 +1,12 @@
 import { Review } from "../../schemas/index.js";
 import { defineAgent } from "../define.js";
+import { medium } from "../models.js";
 
 /** Reviews a pull request across correctness, security, performance, and tests. Read-only, in the PR Reviewer's sandbox. */
 export const prReviewer = defineAgent({
 	id: "pr-reviewer",
 	prompt: "pr-reviewer/prompt.md",
-	model: "medium",
+	harness: medium,
 	tools: ["read", "ls", "bash"],
 	output: Review,
 	maxTurns: 100,

app/agents/registry.ts

@@ -0,0 +1,11 @@
+import type { Agent } from "./define.js";
+import { implementer } from "./implementer/index.js";
+import { prReviewer } from "./pr-reviewer/index.js";
+import { specWriter } from "./spec-writer/index.js";
+import { triage } from "./triage/index.js";
+
+/**
+ * Every agent the factory runs. The sandbox runner is bundled from this list and runs an agent only if it is here.
+ * Bundled modules must not import host-only code (GitHub, Sandbox, Workflows, claims, events, config).
+ */
+export const AGENTS: readonly Agent[] = [triage, specWriter, implementer, prReviewer];

app/agents/run.ts

@@ -1,10 +1,8 @@
 import { randomUUID } from "node:crypto";
 import type { z } from "zod";
-import { modelFor } from "../config/index.js";
 import { emit } from "../events/index.js";
-import { providerApiKey } from "../runner/keys.js";
-import { type RunnerErrorCode, RunnerEvent, type RunnerInput, type ToolName, type Usage } from "../runner/protocol.js";
-import { errorMessage } from "../runner/session.js";
+import { errorMessage } from "../runner/core/emit.js";
+import { PROTOCOL_VERSION, type RunnerErrorCode, RunnerEvent, type RunnerInput, type ToolName, type Usage } from "../runner/core/protocol.js";
 import { APP_DIR, REPO_DIR, SCRATCH_DIR, type Sandbox } from "../sandbox/types.js";
 import type { Agent, OutputSchema } from "./define.js";
 
@@ -55,7 +53,7 @@ export async function runAgent<Tools extends readonly ToolName[], Output extends
 	opts: RunAgentOptions,
 ): Promise<AgentRun<Output>> {
 	const { sandbox, ...base } = opts;
-	const { provider, model } = modelFor(agent.model);
+	const { harness } = agent;
 	const timeoutMs = base.timeoutMs ?? DEFAULT_TIMEOUT_MS;
 	const finish = (outcome: string, summary: Summary, message?: string) =>
 		emit({
@@ -66,7 +64,9 @@ export async function runAgent<Tools extends readonly ToolName[], Output extends
 			type: "agent.finished",
 			payload: {
 				agentId: agent.id,
-				model: `${provider}/${model}`,
+				harness: harness.name,
+				...(summary.harnessVersion ? { harnessVersion: summary.harnessVersion } : {}),
+				model: summary.model ?? harness.label,
 				outcome,
 				turns: summary.turns,
 				usage: summary.usage,
@@ -80,22 +80,20 @@ export async function runAgent<Tools extends readonly ToolName[], Output extends
 	const failure = (code: AgentErrorCode, text: string) => new AgentError(agent.id, code, text);
 	try {
 		const runnerInput: RunnerInput = {
+			protocol_version: PROTOCOL_VERSION,
 			trace_id: base.traceId,
 			agent_id: agent.id,
 			system_prompt: agent.prompt,
 			message,
-			model_provider: provider,
-			model,
-			thinking_level: agent.thinking ?? "medium",
 			tools: [...agent.tools],
 			output_schema: agent.outputJsonSchema,
 			max_turns: agent.maxTurns,
 			timeout_ms: timeoutMs,
 			cwd: REPO_DIR,
 			scratch_dir: SCRATCH_DIR,
 		};
-		const apiKey = providerApiKey(provider, process.env);
-		if (!apiKey) throw failure("agent_error", `no API key for provider "${provider}" in the environment`);
+		const apiKey = process.env[harness.credential]?.trim();
+		if (!apiKey) throw failure("agent_error", `${harness.credential} is not set: the ${harness.name} harness needs it`);
 
 		const stdout = await runInSandbox(sandbox, runnerInput, apiKey, timeoutMs, base.signal);
 		summary = summarize(stdout.text);
@@ -170,6 +168,9 @@ interface Summary {
 	costUsd: number;
 	turns: number;
 	terminal?: Extract<RunnerEvent, { type: "result" | "error" }>;
+	/** From the runner's `started` event: what actually ran, which the host can't always know. */
+	harnessVersion?: string;
+	model?: string;
 	/** Lines that aren't valid RunnerEvents. */
 	invalidLines: number;
 }
@@ -197,7 +198,10 @@ function summarize(jsonl: string): Summary {
 			continue;
 		}
 		if (summary.terminal) continue;
-		if (event.type === "usage") {
+		if (event.type === "started") {
+			summary.harnessVersion = event.harness_version;
+			summary.model = event.model;
+		} else if (event.type === "usage") {
 			const u = summary.usage;
 			u.input_tokens += event.usage.input_tokens;
 			u.output_tokens += event.usage.output_tokens;

app/agents/spec-writer/index.ts

@@ -1,11 +1,12 @@
 import { SpecDraft } from "../../schemas/index.js";
 import { defineAgent } from "../define.js";
+import { large } from "../models.js";
 
 /** Researches the repository (read-only) and drafts or revises the spec. Runs in triage's sandbox after a "draft" decision. */
 export const specWriter = defineAgent({
 	id: "spec-writer",
 	prompt: "spec-writer/prompt.md",
-	model: "large",
+	harness: large,
 	tools: ["read", "ls", "bash"],
 	output: SpecDraft,
 	maxTurns: 60,

app/agents/triage/index.ts

@@ -1,11 +1,12 @@
 import { TriageDecision } from "../../schemas/index.js";
 import { defineAgent } from "../define.js";
+import { medium } from "../models.js";
 
 /** Reads the repository (read-only) and decides whether the issue is ready for a spec. Runs in a sandbox at the base branch. */
 export const triage = defineAgent({
 	id: "triage",
 	prompt: "triage/prompt.md",
-	model: "medium",
+	harness: medium,
 	tools: ["read", "ls", "bash"],
 	output: TriageDecision,
 	maxTurns: 40,

app/config/index.ts

@@ -1,6 +1,6 @@
-import config, { type ModelTier, type RepoConfig } from "../../factory.config.js";
+import config, { type RepoConfig } from "../../factory.config.js";
 
-export { config, type ModelTier, type RepoConfig };
+export { config, type RepoConfig };
 
 /** One repository a run acts on, resolved from the webhook that started it. */
 export interface RepoContext {
@@ -29,13 +29,6 @@ export function requireRepo(fullName: string, installationId: number): RepoConte
 	return repo;
 }
 
-export function modelFor(tier: ModelTier): { provider: string; model: string } {
-	const id = config.models[tier];
-	const slash = id.indexOf("/");
-	if (slash <= 0) throw new Error(`models.${tier} must be "provider/model", got "${id}"`);
-	return { provider: id.slice(0, slash), model: id.slice(slash + 1) };
-}
-
 export function requireEnv(name: string): string {
 	const value = process.env[name]?.trim();
 	if (!value) throw new Error(`${name} is not set`);

app/harness/index.ts

@@ -0,0 +1,4 @@
+export type { HarnessRun, HarnessRuntime, HarnessSelfTest } from "../runner/core/harness.js";
+export { type PiOptions, pi, type ThinkingLevel } from "./pi.js";
+export { type PydanticOptions, pydantic } from "./pydantic.js";
+export type { Harness, HarnessProvision } from "./types.js";

app/harness/pi.ts

@@ -0,0 +1,37 @@
+import type { Api, Model } from "@earendil-works/pi-ai";
+import { createModels, PI_VERSION, runPiSession, type ThinkingLevel } from "../runner/pi/session.js";
+import type { Harness } from "./types.js";
+
+export type { ThinkingLevel };
+
+/** Each provider's key env var, for the providers the factory supports through Pi. */
+export const PI_PROVIDER_KEY_ENV: Readonly<Record<string, string>> = {
+	anthropic: "ANTHROPIC_API_KEY",
+	google: "GEMINI_API_KEY",
+	openai: "OPENAI_API_KEY",
+};
+
+export interface PiOptions {
+	/** A model from Pi's catalog, e.g. `getBuiltinModel("anthropic", "claude-opus-5")`. */
+	model: Model<Api>;
+	/** Default "medium". */
+	thinkingLevel?: ThinkingLevel;
+}
+
+/** Runs an agent on Pi (`@earendil-works/pi-agent-core`) with the factory's tools. */
+export function pi(opts: PiOptions): Harness {
+	const { provider, id } = opts.model;
+	const credential = PI_PROVIDER_KEY_ENV[provider];
+	if (!credential) throw new Error(`pi: provider "${provider}" is not supported; use one of ${Object.keys(PI_PROVIDER_KEY_ENV).join(", ")}`);
+	const thinkingLevel = opts.thinkingLevel ?? "medium";
+	return {
+		name: "pi",
+		label: `${provider}/${id}`,
+		credential,
+		run: (run) => runPiSession(run, { model: opts.model, thinkingLevel }),
+		async selfTest() {
+			const known = createModels().getModel(provider, id);
+			return { version: PI_VERSION, problems: known ? [] : [`pi: model ${provider}/${id} is not in the Pi ${PI_VERSION} registry`] };
+		},
+	};
+}