From 5741bd73b115cd4a07a66fb98cd66e60243c3654 Mon Sep 17 00:00:00 2001 From: zszz3 <91608029+zszz3@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:09:10 +0800 Subject: [PATCH 1/5] fix(runtime): stabilize ToolSearch across provider transports Select fixed declarations by transport capabilities instead of a Flash allowlist, retaining native anchored additions where Pi supports them. Keep execution activation private to Desktop so folding adapters do not rewrite their leading instructions after each search. Exercise all Desktop-selectable adapters through the runtime tool loop, with compatible-route HTTP and process coverage and independent activation restoration and permission checks. --- docs/adr/chronological-system-transcript.md | 36 ++++--- docs/spec/03-runtime/02-agent-runtime.md | 29 ++++-- docs/spec/06-delivery/04-e2e-test-plan.md | 17 +++- .../zh-CN/spec/03-runtime/02-agent-runtime.md | 9 +- .../spec/06-delivery/04-e2e-test-plan.md | 17 +++- .../src/fixed-tool-declarations.test.ts | 25 ++++- .../src/fixed-tool-declarations.ts | 27 +++-- .../src/fixed-tool-providers.test.ts | 98 +++++++++++++++++++ .../src/fixed-tool-runtime.test.ts | 91 +++-------------- .../agent-runtime/src/pi-runtime-messages.ts | 15 +++ packages/agent-runtime/src/runtime.test.ts | 66 ++++++------- packages/agent-runtime/src/runtime.ts | 2 +- .../src/system-transcript-runtime.test.ts | 4 + .../src/test-helpers/fixed-tool-fixture.ts | 77 +++++++++++++++ scripts/e2e-fixed-tool-declarations.mjs | 9 +- 15 files changed, 372 insertions(+), 150 deletions(-) create mode 100644 packages/agent-runtime/src/fixed-tool-providers.test.ts create mode 100644 packages/agent-runtime/src/test-helpers/fixed-tool-fixture.ts diff --git a/docs/adr/chronological-system-transcript.md b/docs/adr/chronological-system-transcript.md index 659ec00a28..1177064d94 100644 --- a/docs/adr/chronological-system-transcript.md +++ b/docs/adr/chronological-system-transcript.md @@ -53,20 +53,34 @@ Desktop allowlist; remove the hunk when an upgraded Pi catalog carries it. The model/API/endpoint binding check still excludes aliases and relays. Native tool-addition and tool-change flags are not enabled by this correction. -## Fixed declarations for the verified Flash route - -For the exact official `deepseek-flash` / `openai-completions` binding with -verified chronological system support, declare the complete current tool catalog -in deterministic name order from the first request. ToolSearch changes execution -activation only. This is a Desktop declaration policy, not an additional Pi -transport capability or an endpoint switch. Other bindings keep on-demand -schema publication; native tool-state flags alone do not prove cache stability. +## Stable declarations across provider transports + +Choose the declaration strategy by the bound Pi transport capabilities, not a +model-name allowlist. Responses (OpenAI/Codex) with verified system support +and `additional_tools` or client tool search, Chat Completions with verified +system/tool additions, and Pi's transcript transport retain native chronological +additions. Other transports declare the complete current catalog in deterministic +name order from the first request. In particular Anthropic's native transition +blocks still grow its request-level schemas, so they use fixed declarations. +This policy also covers compatible relays without enabling unsupported native +message roles or switching their configured API. + +Desktop currently exposes Chat Completions, Responses, Codex Responses, +Anthropic Messages, Gemini and Pi Messages bindings. This change does not add +new selectable Azure, Vertex, Bedrock or Mistral native bindings; models offered +through existing compatible endpoints follow that endpoint's adapter. Keep activation separate from declarations in a versioned `tool_activation` system section. The section records active deferred names and a SHA-256 identity of the account, model, API, endpoint, declarations and deferred-name set. Updates append after tool results and persist through the existing Host journal and -compaction checkpoint. Restoration never interprets the complete declaration +compaction checkpoint. Before provider conversion, omit this Desktop-only +activation section and any resulting empty metadata-only message. Preserve all +other instruction sections, content and tool deltas. Providers that fold system +messages must not rewrite their leading instructions just because execution +activation changed. ToolSearch results tell the model which tools were activated; +uncertain models may search again. Canonical persisted history is not mutated. +Restoration never interprets the complete declaration snapshot as permission to execute every tool. Successful ToolSearch results newer than the saved activation section recover an interrupted activation. Invalid, unknown-version or mismatched state grants no activation. Legacy @@ -79,8 +93,8 @@ removed tools cannot be invoked. Temporary prompt replacement must preserve the activation metadata. Current runtime activation remains authoritative between prompts; declarations do not re-grant revoked activation. -DeepSeek limits a request to 128 functions. If the full catalog exceeds that -limit, or its estimated prompt/schema cost leaves less than the ordinary +Use 128 functions as a conservative shared fixed-catalog ceiling, including +DeepSeek Chat Completions' limit. If the full catalog exceeds that ceiling, or its estimated prompt/schema cost leaves less than the ordinary retained-tail budget below the automatic compaction threshold, use the existing on-demand path and log the fallback reason. Never truncate a catalog. Context estimation charges the full declared catalog while fixed declarations are active. diff --git a/docs/spec/03-runtime/02-agent-runtime.md b/docs/spec/03-runtime/02-agent-runtime.md index 139ca9421a..a0269ea9b0 100644 --- a/docs/spec/03-runtime/02-agent-runtime.md +++ b/docs/spec/03-runtime/02-agent-runtime.md @@ -1412,10 +1412,9 @@ grammar and validated against another fails every call. ### 7.1 Active tool context and on-demand loading (D185, ADR 0048) -The sidecar builds one complete tool registry. By default, each provider request -declares the mode's core set plus activated deferred tools. The verified Flash -binding uses the fixed-declaration policy below, while preserving the same -execution activation rules: +The sidecar builds one complete tool registry. Native anchored-addition routes +declare core tools and add activated deferred tools in place. Other routes use +the fixed-declaration policy below. Both preserve these execution activation rules: - Agent: `Read`, `Bash`, `Edit`, and `Write` (matching pi's coding-agent core) - Agent: `Skill` whenever the skill catalog is non-empty (D404, ADR 0230) — the @@ -1454,9 +1453,13 @@ results from deferred tools also restore their names. Failed results, missing-result placeholders and assistant/user prose never activate tools. Only names in the current mode's deferred catalog are eligible. -For the exact official `deepseek-flash` Chat Completions binding with verified -mid-conversation system support, the runtime instead declares the complete -catalog in deterministic name order on the first request. ToolSearch changes +Select by the bound transport, not a model name. Responses with verified system +support plus `supportsAdditionalTools` or `supportsToolSearch`, Chat Completions +with verified system/tool additions, and Pi Messages retain native additions. +Other bindings, including Anthropic Messages, Gemini, ordinary Chat Completions, +older Responses/Codex models and compatible relays, declare the complete catalog +in deterministic name order on the first request. Anthropic's native tool-change +blocks still grow request-level schemas, so do not exempt them. ToolSearch changes activation without changing the declared schemas. A visible schema does not permit execution: inactive deferred calls are rejected before extension hooks and the Host; activated calls still require the existing mode and Host checks. @@ -1466,7 +1469,12 @@ Fixed declarations persist separately from activation. A version-1 `tool_activation` section records active names and a fingerprint of the account, model, API, endpoint, schema catalog and deferred set. Activation changes append at the continuation boundary, and the existing system journal/checkpoint saves -both declarations and activation. Restore only validated activation for a +both declarations and activation. Strip the activation section only from the +provider projection, dropping metadata-only empty messages but preserving all +other sections/content/tool deltas. This prevents folding APIs from moving an +activation change into the leading prompt; persisted state remains complete. +Model guidance uses successful ToolSearch results, not private activation JSON. +Restore only validated activation for a matching fingerprint, plus successful ToolSearch results newer than that state; never activate tools merely because the full snapshot declared them. Malformed, unknown-version and mismatched activation state fail closed. A catalog/schema, @@ -1476,8 +1484,9 @@ activation. Removal immediately removes the tool from executable registration. If the full catalog exceeds 128 functions or its prompt/schema estimate cannot leave the normal retained-tail budget below the compaction threshold, retain on-demand declarations and emit a diagnostic explaining that ToolSearch cache -stability is not guaranteed. Do not truncate tools. Other models and unverified -routes retain the existing Pi projection. First-request schema overhead increases; +stability is not guaranteed. The 128-function ceiling is conservative across +fixed-catalog APIs. Do not truncate tools or opt unknown endpoints into native +capabilities. First-request schema overhead increases; cache stability does not imply that short conversations become cheaper. For user-visible HTML deliverables, the default system prompt asks the agent to diff --git a/docs/spec/06-delivery/04-e2e-test-plan.md b/docs/spec/06-delivery/04-e2e-test-plan.md index 5a64266872..0faa5a6ef7 100644 --- a/docs/spec/06-delivery/04-e2e-test-plan.md +++ b/docs/spec/06-delivery/04-e2e-test-plan.md @@ -16487,7 +16487,7 @@ renderer's durable transcript reads. No real model or provider is contacted. regressions also cover legacy identities and genuine account/model changes. Cache percentages are observations, not deterministic pass thresholds. -### E2E-FIXED-TOOL-DECLARATIONS: Stable Flash schemas with independent activation +### E2E-FIXED-TOOL-DECLARATIONS: Stable schemas across transports with independent activation - Fixture: production AgentSidecar and isolated Host, official Pi Flash binding, and a child-process fetch boundary redirected to local HTTP/SSE. Credentials @@ -16520,6 +16520,21 @@ renderer's durable transcript reads. No real model or provider is contacted. through successful recovery. Terminal failure and Stop retain their existing closure behavior. Covered by the parameterized runtime overflow user-path test. +- Cross-provider acceptance: `fixed-tool-providers.test.ts` enters the real runtime + prompt/ToolSearch/execution path and captures each Desktop-selectable Pi + adapter's actual serialized payload at `onPayload`, before network dispatch. + Cover Chat Completions, Anthropic (native system on/off), Responses and Codex + (fallback, additional tools, client tool search), Gemini and Pi Messages. + Search A, execute A, search B, execute B, then finish: all five requests retain + their top-level schema state and prior semantic message prefix. Native routes + retain deferred schema additions. Activation JSON never reaches the provider. + This proves request construction, not server cache hits or paid API acceptance. +- The local HTTP/SSE fixture additionally covers official Flash, unflagged Chat + Completions and a compatible relay. Canonical activation restoration, denial + before Host execution, mode/account/catalog invalidation and compaction remain + required. Fixed declarations increase first-request size; oversized catalogs + explicitly fall back without a cache-stability guarantee. + #### E2E-262: Transcript path:line opens and scrolls the host file viewer diff --git a/docs/zh-CN/spec/03-runtime/02-agent-runtime.md b/docs/zh-CN/spec/03-runtime/02-agent-runtime.md index 03346fba5d..485f191b10 100644 --- a/docs/zh-CN/spec/03-runtime/02-agent-runtime.md +++ b/docs/zh-CN/spec/03-runtime/02-agent-runtime.md @@ -983,7 +983,7 @@ sidecar 最多激活四个匹配项，并将名称写入 canonical Deferred activation remains sticky within a live runtime. Restoration uses successful activation evidence and the current catalog; old declarations do not -re-grant tools revoked from the live activation set. For official bound Flash, +re-grant tools revoked from the live activation set. For routes without verified native anchored tool additions, full declarations and execution activation are independent: versioned `tool_activation` sections carry the account/model/API/endpoint/catalog identity and active names through restart and compaction. Only matching, valid state and @@ -992,7 +992,12 @@ epochs fail closed. Inactive declared tools are blocked before extension/Host execution, and activation never bypasses mode or approval checks. The full catalog is deterministic from the first request. More than 128 tools or an insufficient context budget falls back to on-demand declarations with a -diagnostic, without truncation. Other bindings retain their existing projection. +diagnostic, without truncation. Responses and Chat Completions bindings with verified anchored additions, and +Pi Messages, retain native incremental publication. Anthropic native tool changes +still grow request schemas and therefore use fixed declarations. Strip only the +private activation section from provider projection, preserving canonical state +and all other instructions. This also stabilizes ToolSearch on folding APIs and +compatible relays without enabling new transport capabilities. Fixed declarations may increase total cost for short conversations. See the English section 7.1 and the chronological-system-transcript ADR for the complete contract. diff --git a/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md b/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md index 51006ac2b9..ae32e27420 100644 --- a/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md +++ b/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md @@ -9290,7 +9290,7 @@ the latest destination. These assertions measure work counts, not device FPS. - **里程碑：** M6+ - **状态：** 单测和源码契约覆盖（`update-cache.test.mjs`、`auto-update.test.mjs`）；仍需 Windows 安装器/E2E 验证。 -### E2E-FIXED-TOOL-DECLARATIONS: Stable Flash schemas with independent activation +### E2E-FIXED-TOOL-DECLARATIONS: Stable schemas across transports with independent activation - Fixture: production AgentSidecar and isolated Host, official Pi Flash binding, and a child-process fetch boundary redirected to local HTTP/SSE. Credentials @@ -9323,6 +9323,21 @@ the latest destination. These assertions measure work counts, not device FPS. through successful recovery. Terminal failure and Stop retain their existing closure behavior. Covered by the parameterized runtime overflow user-path test. +- Cross-provider acceptance: `fixed-tool-providers.test.ts` enters the real runtime + prompt/ToolSearch/execution path and captures each Desktop-selectable Pi + adapter's actual serialized payload at `onPayload`, before network dispatch. + Cover Chat Completions, Anthropic (native system on/off), Responses and Codex + (fallback, additional tools, client tool search), Gemini and Pi Messages. + Search A, execute A, search B, execute B, then finish: all five requests retain + their top-level schema state and prior semantic message prefix. Native routes + retain deferred schema additions. Activation JSON never reaches the provider. + This proves request construction, not server cache hits or paid API acceptance. +- The local HTTP/SSE fixture additionally covers official Flash, unflagged Chat + Completions and a compatible relay. Canonical activation restoration, denial + before Host execution, mode/account/catalog invalidation and compaction remain + required. Fixed declarations increase first-request size; oversized catalogs + explicitly fall back without a cache-stability guarantee. + #### E2E-262：聊天 path:line 引用打开文件并滚动到目标行 - **前提：** 隔离 Electron/Chromium、活动工作区和会话、可用的随应用打包文件管理器视图，以及确定性的文件系统 IPC fixture。 diff --git a/packages/agent-runtime/src/fixed-tool-declarations.test.ts b/packages/agent-runtime/src/fixed-tool-declarations.test.ts index 8b54969b9a..451fd277ca 100644 --- a/packages/agent-runtime/src/fixed-tool-declarations.test.ts +++ b/packages/agent-runtime/src/fixed-tool-declarations.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "vitest"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; +import type { AgentMessage, AgentTool } from "@earendil-works/pi-agent-core"; import { DEEPSEEK_MODELS } from "@earendil-works/pi-ai/providers/deepseek.models"; import { Type, type Api, type Model } from "@earendil-works/pi-ai"; import { toolDeclarationPolicy, toolActivationSection, restoredToolActivation, TOOL_ACTIVATION_SECTION } from "./fixed-tool-declarations.js"; import { replaceSystemPrompt, systemTranscriptCheckpoint } from "./system-transcript.js"; +import { convertToLlm } from "./pi-runtime-messages.js"; const model = Object.values(DEEPSEEK_MODELS).find((model) => model.id === "deepseek-flash")!; const tool = (name: string): AgentTool => ({ name, label: name, description: "Fixture tool", parameters: Type.Object({}), @@ -13,6 +14,24 @@ const policy = (tools = [tool("Alpha"), tool("Beta")], overrides: Partial, tools, deferred, prompt, "fixture-account"); describe("fixed tool declaration policy", () => { + it("persists activation without projecting it into model instructions", () => { + const current = policy(); + const messages: AgentMessage[] = [ + { role: "system" as const, content: "", timestamp: 1, toolsAdded: current.tools, + sections: { runtime: "Rules", [TOOL_ACTIVATION_SECTION]: toolActivationSection(current.key, new Set()) } }, + { role: "system" as const, content: "", timestamp: 2, + sections: { [TOOL_ACTIVATION_SECTION]: toolActivationSection(current.key, new Set(["Alpha"])) } }, + { role: "system" as const, content: "Changed instruction", timestamp: 3, + sections: { obsolete: null, [TOOL_ACTIVATION_SECTION]: null }, toolsRemoved: [{ name: "Beta" }] }, + ]; + const before = JSON.stringify(messages); + const projected = convertToLlm(messages); + expect(projected).toHaveLength(2); + expect(projected[0]).toMatchObject({ sections: { runtime: "Rules" }, toolsAdded: current.tools }); + expect(projected[1]).toMatchObject({ content: "Changed instruction", sections: { obsolete: null }, toolsRemoved: [{ name: "Beta" }] }); + expect(JSON.stringify(projected)).not.toContain(TOOL_ACTIVATION_SECTION); + expect(JSON.stringify(messages)).toEqual(before); + }); it("has a deterministic declaration order and snapshot identity", () => { const first = policy(); const second = policy([tool("Beta"), tool("Alpha")]); @@ -23,8 +42,8 @@ describe("fixed tool declaration policy", () => { it.each([ { id: "deepseek-pro" }, { api: "openai-responses" }, { baseUrl: "https://relay.invalid" }, { baseUrl: "https://api.deepseek.com/v1" }, { compat: { supportsMidConvoSystemMessages: false } }, - ] as Partial>[])("does not enable unverified bindings: %j", (overrides) => { - expect(policy(undefined, overrides).tools).toBeUndefined(); + ] as Partial>[])("stabilizes declarations without assuming transcript capabilities: %j", (overrides) => { + expect(policy(undefined, overrides).tools?.map((tool) => tool.name)).toEqual(["Alpha", "Beta"]); expect(policy(undefined, overrides).fallback).toBeUndefined(); }); it("falls back without truncating catalogs beyond the provider's function limit", () => { diff --git a/packages/agent-runtime/src/fixed-tool-declarations.ts b/packages/agent-runtime/src/fixed-tool-declarations.ts index 721f39663e..cf3a450fd1 100644 --- a/packages/agent-runtime/src/fixed-tool-declarations.ts +++ b/packages/agent-runtime/src/fixed-tool-declarations.ts @@ -17,7 +17,24 @@ export type ToolDeclarationPolicy = { fallback?: "tool-count" | "context-budget"; }; -/** The verified Flash route has chronological system updates, but no tool deltas. */ +/** Only these Pi transports keep new schemas out of the request's initial tools. */ +function hasAnchoredToolAdditions(model: Model): boolean { + if (model.api === "pi-messages") return true; + const compat = model.compat; + if (!compat || !("supportsMidConvoSystemMessages" in compat) + || compat.supportsMidConvoSystemMessages !== true) return false; + if (model.api === "openai-completions") { + return "supportsMidConvoToolAdditions" in compat && compat.supportsMidConvoToolAdditions === true; + } + if (["openai-responses", "openai-codex-responses"].includes(model.api)) { + return ("supportsAdditionalTools" in compat && compat.supportsAdditionalTools === true) + || ("supportsToolSearch" in compat && compat.supportsToolSearch === true); + } + // Anthropic's native tool-change blocks still grow its request-level schemas. + return false; +} + +/** Keep native anchored additions, otherwise stabilize the complete catalog. */ export function toolDeclarationPolicy( model: Model, tools: readonly AgentTool[], deferred: ReadonlySet, prompt: string, accountId: string, ): ToolDeclarationPolicy { @@ -26,11 +43,9 @@ export function toolDeclarationPolicy( account: accountId, model: model.id, api: model.api, endpoint: model.baseUrl.replace(/\/+$/, ""), tools: ordered.map(toToolDeclaration), deferred: [...deferred].sort(), })).digest("hex"); - if (model.id !== "deepseek-flash" || model.api !== "openai-completions" - || model.baseUrl.replace(/\/+$/, "") !== "https://api.deepseek.com" - || !model.compat || !("supportsMidConvoSystemMessages" in model.compat) - || model.compat.supportsMidConvoSystemMessages !== true) return { key }; - // DeepSeek Chat Completions permits at most 128 functions. Never truncate. + if (hasAnchoredToolAdditions(model)) return { key }; + // Conservative shared ceiling, including Chat Completions' 128-function limit. + // Larger catalogs retain on-demand loading; never truncate or raise API limits. if (ordered.length > 128) return { key, fallback: "tool-count" }; const budget = contextBudgetLimitsFor(model); const tokens = estimateOutputCapInputTokens({ messages: [], systemPrompt: prompt, tools: ordered }, model); diff --git a/packages/agent-runtime/src/fixed-tool-providers.test.ts b/packages/agent-runtime/src/fixed-tool-providers.test.ts new file mode 100644 index 0000000000..12f8b24395 --- /dev/null +++ b/packages/agent-runtime/src/fixed-tool-providers.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import type { Agent } from "@earendil-works/pi-agent-core"; +import { createAssistantMessageEventStream, type Api, type Model, type AssistantMessage, type StreamFunction } from "@earendil-works/pi-ai"; +import { DEEPSEEK_MODELS } from "@earendil-works/pi-ai/providers/deepseek.models"; +import { modelConfigFromPi } from "./model-capabilities.js"; +import { runtimeFixture, pluginTools } from "./test-helpers/fixed-tool-fixture.js"; + +const base = Object.values(DEEPSEEK_MODELS).find((model) => model.id === "deepseek-flash")!; +const routes: { api: Api; compat?: Model["compat"]; native?: boolean; label: string }[] = [ + { label: "Chat Completions", api: "openai-completions" }, + { label: "Claude without native transitions", api: "anthropic-messages" }, + { label: "Claude with native transitions", api: "anthropic-messages", compat: { supportsMidConvoSystemMessages: true, supportsMidConvoToolChanges: true } }, + { label: "OpenAI Responses fallback", api: "openai-responses" }, + { label: "Codex fallback", api: "openai-codex-responses" }, + { label: "Gemini", api: "google-generative-ai" }, + { label: "Kimi native additions", api: "openai-completions", native: true, compat: { supportsMidConvoSystemMessages: true, supportsMidConvoToolAdditions: true } }, + { label: "OpenAI native additions", api: "openai-responses", native: true, compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true } }, + { label: "Codex native search", api: "openai-codex-responses", native: true, compat: { supportsMidConvoSystemMessages: true, supportsToolSearch: true } }, + { label: "Codex native additions", api: "openai-codex-responses", native: true, compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true } }, + { label: "Pi transcript", api: "pi-messages", native: true }, +]; +// Cache markers move with the last message; they are not model input text. +function semantic(value: unknown): unknown { + if (Array.isArray(value)) return value.map(semantic); + if (value && typeof value === "object") return Object.fromEntries(Object.entries(value) + .filter(([key]) => key !== "cache_control" && key !== "cachePoint") + .map(([key, item]) => [key, semantic(item)])); + return value; +} +function payloadView(value: unknown): { tools: unknown; system: unknown; messages: unknown[] } { + const p = value as Record; + const config = p.config as Record | undefined; + const context = p.context as Record | undefined; + return { tools: p.tools ?? p.toolConfig ?? config?.tools, + system: p.system ?? p.instructions ?? config?.systemInstruction, + messages: (p.messages ?? p.input ?? p.contents ?? context?.messages ?? []) as unknown[] }; +} + +describe("ToolSearch user path through every Desktop-selectable Pi request adapter", () => { + it.each(routes)("preserves schemas and history for $label", async ({ api, compat, native }) => { + const model = { ...base, id: "fixture-model", provider: "fixture", api, + baseUrl: "https://fixture.invalid", compat, reasoning: false } as Model; + const adapter: { stream: StreamFunction } = await import(`@earendil-works/pi-ai/api/${api}`); + // Synthetic token satisfies Codex's local parser; no real auth is read. + const key = `fixture.${Buffer.from(JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "fixture" } })).toString("base64")}.fixture`; + const f = runtimeFixture([], pluginTools, { id: "fixture", name: "Fixture", modelId: model.id, + baseUrl: model.baseUrl, apiKey: key, authKind: "api_key", supportsReasoning: false, + supportedThinkingLevels: ["off"], modelConfig: modelConfigFromPi(model) }); + const requests: ReturnType[] = []; + const captureErrors: string[] = []; + const calls: { name: string; arguments: Record }[] = [ + { name: "ToolSearch", arguments: { query: "plugin_alpha" } }, { name: "plugin_alpha", arguments: {} }, + { name: "ToolSearch", arguments: { query: "plugin_beta" } }, { name: "plugin_beta", arguments: {} }, + ]; + const agent = (f.runtime as unknown as { agent: Agent }).agent; + agent.streamFunction = async (resolved, context) => { + if (resolved.api !== api) captureErrors.push(`Wrong adapter: ${resolved.api}`); + let captured: unknown; + // Run the serializer and stop at the external request boundary, before + // network access or cloud credential lookup. + const probe = adapter.stream(resolved, context, { apiKey: key, maxTokens: 1024, + onPayload: (payload) => { captured = payload; throw new Error("fixture payload captured"); }, + }); + const outcome = await probe.result(); + if (captured === undefined) captureErrors.push(outcome.errorMessage ?? "No payload"); + else if (JSON.stringify(captured).includes("tool_activation")) captureErrors.push("Activation metadata leaked"); + requests.push(payloadView(semantic(captured ?? {}))); + const call = calls.shift(); + const message: AssistantMessage = { role: "assistant", api, provider: resolved.provider, model: resolved.id, + content: call ? [{ type: "toolCall", id: `call-${requests.length}`, ...call }] : [{ type: "text", text: "Done." }], + stopReason: call ? "toolUse" : "stop", timestamp: Date.now(), + usage: { input: 10, output: 2, cacheRead: 0, cacheWrite: 0, totalTokens: 12, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } }; + const stream = createAssistantMessageEventStream(); + stream.push({ type: "done", reason: call ? "toolUse" : "stop", message }); + return stream; + }; + try { + await f.prompt(); + expect(captureErrors).toEqual([]); + expect(f.errors).toEqual([]); + expect(f.executed).toEqual(["plugin_alpha", "plugin_beta"]); + expect(requests).toHaveLength(5); + expect(requests[0].messages.length).toBeGreaterThan(0); + if (!native) expect(JSON.stringify(requests[0].tools)).toContain("plugin_beta"); + else { + expect(JSON.stringify(requests[0].tools) ?? "").not.toContain("plugin_beta"); + expect(JSON.stringify(requests.at(-1)?.messages)).toContain("plugin_beta"); + } + for (let index = 1; index < requests.length; index++) { + expect(requests[index].tools).toEqual(requests[0].tools); + expect(requests[index].system).toEqual(requests[0].system); + const previous = requests[index - 1].messages; + expect(requests[index].messages.slice(0, previous.length)).toEqual(previous); + } + } finally { await f.runtime.dispose(); } + }); +}); diff --git a/packages/agent-runtime/src/fixed-tool-runtime.test.ts b/packages/agent-runtime/src/fixed-tool-runtime.test.ts index 2c86736cea..4d32bb64ec 100644 --- a/packages/agent-runtime/src/fixed-tool-runtime.test.ts +++ b/packages/agent-runtime/src/fixed-tool-runtime.test.ts @@ -1,82 +1,9 @@ -import { randomUUID } from "node:crypto"; -import { createServer } from "node:http"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { DEEPSEEK_MODELS } from "@earendil-works/pi-ai/providers/deepseek.models"; import { systemTranscriptCheckpoint } from "./system-transcript.js"; import { readSystemMessage } from "./system-transcript-journal.js"; import type { UiMessage } from "@pi-desktop/shared"; -import { modelConfigFromPi } from "./model-capabilities.js"; -import { DesktopAgentRuntime, type PluginToolDef, type RuntimeProviderConfig } from "./runtime.js"; +import { flashProvider, pluginTools, wireFixture, runtimeFixture, type Payload } from "./test-helpers/fixed-tool-fixture.js"; -function flashProvider(): RuntimeProviderConfig { - const model = Object.values(DEEPSEEK_MODELS).find((model) => model.id === "deepseek-flash")!; - return { id: "flash-fixture", name: "Flash", modelId: model.id, baseUrl: model.baseUrl, - apiKey: "fixture", authKind: "api_key", supportsReasoning: false, supportedThinkingLevels: ["off"], - modelConfig: modelConfigFromPi(model) }; -} -const pluginTools: PluginToolDef[] = ["plugin_alpha", "plugin_beta"].map((name) => ({ - name, description: `${name} synthetic probe`, parameters: { type: "object", properties: {}, required: [] }, risk: "low", -})); -type Payload = { tools: { function: { name: string } }[]; messages: { role: string; content?: unknown }[] }; -type Call = { name: string; args?: Record }; -async function wireFixture(calls: Call[]) { - const requests: Payload[] = []; - const fetch = globalThis.fetch; - const server = createServer(async (req, res) => { - const chunks: Buffer[] = []; - for await (const chunk of req) chunks.push(Buffer.from(chunk)); - requests.push(JSON.parse(Buffer.concat(chunks).toString())); - const call = calls.shift(); - const delta = call ? { role: "assistant", tool_calls: [{ index: 0, id: randomUUID(), - type: "function", function: { name: call.name, arguments: JSON.stringify(call.args ?? {}) } }] } - : { role: "assistant", content: "Done." }; - res.writeHead(200, { "content-type": "text/event-stream" }); - res.end(`data: ${JSON.stringify({ choices: [{ index: 0, delta, finish_reason: call ? "tool_calls" : "stop" }] })}\n\ndata: [DONE]\n\n`); - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - const address = server.address(); - if (!address || typeof address === "string") throw new Error("Missing HTTP address"); - vi.stubGlobal("fetch", ((_url, init) => fetch(`http://127.0.0.1:${address.port}`, init)) satisfies typeof fetch); - return { requests, close: async () => { - vi.unstubAllGlobals(); - await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); - } }; -} -function runtimeFixture(history: UiMessage[] = [], tools = pluginTools, provider = flashProvider(), denied = false) { - const rows = structuredClone(history); - const executed: string[] = []; - const errors: unknown[] = []; - const runtime = new DesktopAgentRuntime({ - sessionId: "fixed-tools", mode: "agent", provider, thinkingLevel: "off", history: rows, pluginTools: tools, - commandShell: { id: "bash", label: "Bash", dialect: "posix", available: true, isDefault: true }, - host: { call: async (method: string, params?: unknown): Promise => { - if (method === "session.appendMessage") rows.push((params as { message: UiMessage }).message); - else if (method === "tools.execute") { - executed.push((params as { toolName: string }).toolName); - return denied ? { ok: false, denied: true, content: "Permission denied" } as T - : { ok: true, content: "Synthetic success" } as T; - } else throw new Error(`Unexpected host method: ${method}`); - return undefined as T; - } }, - onEvent: ({ event }) => { - if (event.type === "error") errors.push(event.error); - if (event.type === "tool_start") rows.push({ id: event.toolCallId, role: "tool", content: "", - toolCallId: event.toolCallId, toolName: event.toolName, toolArgs: event.args, createdAt: new Date().toISOString() }); - if (event.type === "tool_end") { - const row = rows.find((row) => row.id === event.toolCallId)!; - row.toolResult = event.result; row.isError = event.isError; row.toolStatus = event.isError ? "error" : "success"; - } - if (event.type === "message_end") { - const index = rows.findIndex((row) => row.id === event.message.id); - if (index < 0) rows.push(event.message); else rows[index] = event.message; - } - }, - }); - return { runtime, rows, executed, errors, prompt: async (id = "user-1") => { - rows.push({ id, role: "user", content: "Run the synthetic probes", createdAt: new Date().toISOString() }); - await runtime.prompt("Run the synthetic probes", id, `turn-${id}`); - } }; -} afterEach(() => vi.unstubAllGlobals()); describe("fixed Flash declarations through runtime and HTTP/SSE", () => { it("rejects a declared but inactive tool without invoking Host", async () => { @@ -171,7 +98,13 @@ describe("fixed Flash declarations through runtime and HTTP/SSE", () => { const wire = await wireFixture([{ name: "ToolSearch", args: { query: "plugin_alpha" } }]); const f = runtimeFixture([], pluginTools, { ...flashProvider(), baseUrl: "https://relay.invalid" }); let history: UiMessage[]; - try { await f.prompt(); history = structuredClone(f.rows); } + try { await f.prompt(); history = structuredClone(f.rows).map((row) => { + if (!row.modelSystem) return row; + const state = readSystemMessage(row.modelSystem.messageJson); + delete state.sections?.tool_activation; + if (state.toolsAdded) state.toolsAdded = state.toolsAdded.filter((tool) => tool.name !== "plugin_beta"); + return { ...row, modelSystem: { ...row.modelSystem, messageJson: JSON.stringify(state) } }; + }); } finally { await f.runtime.dispose(); await wire.close(); } const resumed = await wireFixture([{ name: "plugin_alpha" }, { name: "plugin_beta" }]); const restored = runtimeFixture(history!); @@ -194,12 +127,16 @@ describe("fixed Flash declarations through runtime and HTTP/SSE", () => { } finally { await f.runtime.dispose(); await wire.close(); } }); - it("keeps the complete request prefix while searching and executing two different tools", async () => { + it.each([ + { name: "official Flash", provider: flashProvider() }, + { name: "unflagged Chat Completions", provider: { ...flashProvider(), modelId: "fixture-chat", modelConfig: undefined } }, + { name: "compatible relay", provider: { ...flashProvider(), baseUrl: "https://relay.invalid/v1" } }, + ])("keeps the complete request prefix for $name while searching and executing two different tools", async ({ provider }) => { const wire = await wireFixture([ { name: "ToolSearch", args: { query: "plugin_alpha" } }, { name: "plugin_alpha" }, { name: "ToolSearch", args: { query: "plugin_beta" } }, { name: "plugin_beta" }, ]); - const f = runtimeFixture(); + const f = runtimeFixture([], pluginTools, provider); try { await f.prompt(); expect(f.errors).toEqual([]); diff --git a/packages/agent-runtime/src/pi-runtime-messages.ts b/packages/agent-runtime/src/pi-runtime-messages.ts index a595da9603..eace63930a 100644 --- a/packages/agent-runtime/src/pi-runtime-messages.ts +++ b/packages/agent-runtime/src/pi-runtime-messages.ts @@ -1,6 +1,7 @@ import type { Message } from "@earendil-works/pi-ai"; import { hostedSearchReplayProjection } from "@earendil-works/pi-ai/utils/hosted-search"; import type { AgentMessage } from "@earendil-works/pi-agent-core"; +import { TOOL_ACTIVATION_SECTION } from "./fixed-tool-declarations.js"; export type CompactionSummaryMessage = { role: "compactionSummary"; @@ -136,6 +137,20 @@ export function convertToLlm(messages: AgentMessage[]): Message[] { })); break; case "system": + // Activation is Desktop execution metadata, not a model instruction. + // Persist it canonically, but omit it before Pi folds system messages: + // otherwise each ToolSearch rewrites the prefix on non-native APIs. + if (runtimeMessage.sections && TOOL_ACTIVATION_SECTION in runtimeMessage.sections) { + const sections = Object.fromEntries(Object.entries(runtimeMessage.sections) + .filter(([name]) => name !== TOOL_ACTIVATION_SECTION)); + if (Object.keys(sections).length || textFromContent(runtimeMessage.content) + || runtimeMessage.toolsAdded?.length || runtimeMessage.toolsRemoved?.length) { + converted.push(asProviderMessage({ ...runtimeMessage, sections })); + } + break; + } + converted.push(asProviderMessage(runtimeMessage)); + break; case "user": case "assistant": case "toolResult": diff --git a/packages/agent-runtime/src/runtime.test.ts b/packages/agent-runtime/src/runtime.test.ts index e84ea5fa4d..ebd4db947c 100644 --- a/packages/agent-runtime/src/runtime.test.ts +++ b/packages/agent-runtime/src/runtime.test.ts @@ -1962,7 +1962,7 @@ describe("DesktopAgentRuntime deferred tool catalog", () => { }, ], }); - const tools = (runtime as any).agent.state.tools as Array<{ name: string }>; + const tools = (runtime as unknown as { activeTools(): Array<{ name: string }> }).activeTools(); const names = tools.map((tool) => tool.name); expect(names).toEqual([ @@ -2162,7 +2162,7 @@ describe("DesktopAgentRuntime deferred tool catalog", () => { expect(result.details.activated).toEqual(["BrowserPreview"]); expect(result.details.addedToolNames).toEqual(["BrowserPreview"]); expect(agent.state.tools.some((tool: any) => tool.name === "BrowserPreview")).toBe( - false, + true, ); const next = await (runtime as any).prepareNextTurn({ @@ -2184,9 +2184,8 @@ describe("DesktopAgentRuntime deferred tool catalog", () => { expect(next.context.tools.some((tool: any) => tool.name === "BrowserPreview")).toBe( true, ); - // The Pi loop declares changes immediately before conversion; preparation - // only changes the executable tool catalog. - expect(getCurrentTools(next.context.messages).some((tool) => tool.name === "BrowserPreview")).toBe(false); + // The full catalog is stable; preparation updates execution activation. + expect(getCurrentTools(next.context.messages).some((tool) => tool.name === "BrowserPreview")).toBe(true); await runtime.dispose(); }); @@ -2214,9 +2213,8 @@ describe("DesktopAgentRuntime deferred tool catalog", () => { expect(agent.state.tools.some((tool: any) => tool.name === "BrowserPreview")).toBe( true, ); - // Reset retains executability; Pi declares it at the next dispatch, not - // while this test calls preparation helpers outside the agent loop. - expect(getCurrentTools(agent.state.messages).some((tool) => tool.name === "BrowserPreview")).toBe(false); + // The full schema was already declared; activation remains sticky. + expect(getCurrentTools(agent.state.messages).some((tool) => tool.name === "BrowserPreview")).toBe(true); await runtime.dispose(); }); }); @@ -6864,10 +6862,10 @@ describe("DesktopAgentRuntime inline context compaction", () => { // The point of this family: the window is bought back without paying for a // summary, so no provider request is made at all. - expect(agent.state.messages[0]).toEqual(prefix); + expect(agent.state.messages[0]).toMatchObject({ ...prefix, timestamp: expect.any(Number) }); expect(getCurrentTools(agent.state.messages)).toEqual(agent.state.tools.map(toToolDeclaration)); - expect(getCurrentSystemMessage(agent.state.messages)?.sections).toEqual({ rules: "Keep checkpoint rules" }); - expect((runtime as any).rebuiltAgentContext().messages[0]).toEqual(prefix); + expect(getCurrentSystemMessage(agent.state.messages)?.sections).toMatchObject({ rules: "Keep checkpoint rules" }); + expect((runtime as any).rebuiltAgentContext().messages[0]).toMatchObject({ ...prefix, timestamp: expect.any(Number) }); expect(generateCompaction).not.toHaveBeenCalled(); const compaction = host.call.mock.calls.find( ([method]) => method === "session.appendCompaction", @@ -9039,7 +9037,7 @@ describe("DesktopAgentRuntime deferred tool restore (#225)", () => { status: "complete", }; const hasTool = (runtime: DesktopAgentRuntime, name: string) => - (runtime as any).agent.state.tools.some((tool: any) => tool.name === name); + (runtime as unknown as { activeTools(): Array<{ name: string }> }).activeTools().some((tool) => tool.name === name); it("keeps a tool active across prompts while its ToolSearch activation is in context", async () => { const runtime = createRuntime({ history: [assistantRow, searchRow()] }); @@ -9165,31 +9163,29 @@ describe("DesktopAgentRuntime deferred tool restore (#225)", () => { (runtime as any).resetDeferredToolsForPrompt(); expect(hasTool(runtime, "BrowserPreview")).toBe(false); - (runtime as any).fullEntries.push( - ...(createRuntime({ - history: [ - assistantRow, - { - id: "tool-preview-ok", - role: "tool", - content: "", - createdAt: now(), - status: "complete", - toolName: "BrowserPreview", - toolCallId: "call-preview-ok", - toolStatus: "success", - toolArgs: {}, - toolResult: { content: [{ type: "text", text: "opened" }] }, - }, - ], - }) as any).fullEntries, - ); - (runtime as any).resetDeferredToolsForPrompt(); - expect(hasTool(runtime, "BrowserPreview")).toBe(true); + const restored = createRuntime({ + history: [ + assistantRow, + { + id: "tool-preview-ok", + role: "tool", + content: "", + createdAt: now(), + status: "complete", + toolName: "BrowserPreview", + toolCallId: "call-preview-ok", + toolStatus: "success", + toolArgs: {}, + toolResult: { content: [{ type: "text", text: "opened" }] }, + }, + ], + }); + expect(hasTool(restored, "BrowserPreview")).toBe(true); + await restored.dispose(); await runtime.dispose(); }); - it("restores the activation again after a mode round trip", async () => { + it("requires activation again after a fixed-catalog mode round trip", async () => { const runtime = createRuntime({ history: [assistantRow, searchRow()] }); (runtime as any).resetDeferredToolsForPrompt(); expect(hasTool(runtime, "BrowserPreview")).toBe(true); @@ -9197,7 +9193,7 @@ describe("DesktopAgentRuntime deferred tool restore (#225)", () => { runtime.setMode("plan"); runtime.setMode("agent"); - expect(hasTool(runtime, "BrowserPreview")).toBe(true); + expect(hasTool(runtime, "BrowserPreview")).toBe(false); await runtime.dispose(); }); }); diff --git a/packages/agent-runtime/src/runtime.ts b/packages/agent-runtime/src/runtime.ts index 97fa98d249..680d433fdf 100644 --- a/packages/agent-runtime/src/runtime.ts +++ b/packages/agent-runtime/src/runtime.ts @@ -3890,7 +3890,7 @@ Do not invent objections or turn speculative risks into blockers. Stop when the } return [ "# On-demand tools", - `The following capabilities are available on demand. Call ${TOOL_SEARCH_NAME} with an exact tool name or a short capability description before using an on-demand tool that has not been activated. A visible schema is not activation; the tool_activation section, when present, records active names.`, + `The following capabilities are available on demand. Call ${TOOL_SEARCH_NAME} with an exact tool name or a short capability description before using an on-demand tool that has not been activated. A visible schema is not activation. Successful ToolSearch results identify activated tools; if unsure, search again.`, ...lines, ].join("\n"); } diff --git a/packages/agent-runtime/src/system-transcript-runtime.test.ts b/packages/agent-runtime/src/system-transcript-runtime.test.ts index 15ced3662a..0e039d584a 100644 --- a/packages/agent-runtime/src/system-transcript-runtime.test.ts +++ b/packages/agent-runtime/src/system-transcript-runtime.test.ts @@ -2,10 +2,14 @@ import { describe, expect, it } from "vitest"; import type { Agent } from "@earendil-works/pi-agent-core"; import { createAssistantMessageEventStream, getCurrentTools, getCurrentSystemPrompt, type AssistantMessage, type Message } from "@earendil-works/pi-ai"; import type { UiMessage } from "@pi-desktop/shared"; +import { modelConfigFromPi } from "./model-capabilities.js"; +import { DEEPSEEK_MODELS } from "@earendil-works/pi-ai/providers/deepseek.models"; import { DesktopAgentRuntime, type RuntimeProviderConfig } from "./runtime.js"; const provider: RuntimeProviderConfig = { id: "fixture", name: "Fixture", modelId: "fixture", apiKey: "", authKind: "none", + modelConfig: modelConfigFromPi({ ...Object.values(DEEPSEEK_MODELS)[0], id: "fixture", provider: "fixture", baseUrl: "https://fixture.invalid/v1", + compat: { supportsMidConvoSystemMessages: true, supportsMidConvoToolAdditions: true } }), baseUrl: "https://fixture.invalid/v1", supportsReasoning: false, supportedThinkingLevels: ["off"], }; const skill = { id: "fixture/notes", name: "First catalog", description: "Summarize notes" }; diff --git a/packages/agent-runtime/src/test-helpers/fixed-tool-fixture.ts b/packages/agent-runtime/src/test-helpers/fixed-tool-fixture.ts new file mode 100644 index 0000000000..c48f4622db --- /dev/null +++ b/packages/agent-runtime/src/test-helpers/fixed-tool-fixture.ts @@ -0,0 +1,77 @@ +import { randomUUID } from "node:crypto"; +import { createServer } from "node:http"; +import { vi } from "vitest"; +import { DEEPSEEK_MODELS } from "@earendil-works/pi-ai/providers/deepseek.models"; +import type { UiMessage } from "@pi-desktop/shared"; +import { modelConfigFromPi } from "../model-capabilities.js"; +import { DesktopAgentRuntime, type PluginToolDef, type RuntimeProviderConfig } from "../runtime.js"; + +export function flashProvider(): RuntimeProviderConfig { + const model = Object.values(DEEPSEEK_MODELS).find((model) => model.id === "deepseek-flash")!; + return { id: "flash-fixture", name: "Flash", modelId: model.id, baseUrl: model.baseUrl, + apiKey: "fixture", authKind: "api_key", supportsReasoning: false, supportedThinkingLevels: ["off"], + modelConfig: modelConfigFromPi(model) }; +} +export const pluginTools: PluginToolDef[] = ["plugin_alpha", "plugin_beta"].map((name) => ({ + name, description: `${name} synthetic probe`, parameters: { type: "object", properties: {}, required: [] }, risk: "low", +})); +export type Payload = { tools: { function: { name: string } }[]; messages: { role: string; content?: unknown }[] }; +type Call = { name: string; args?: Record }; +export async function wireFixture(calls: Call[]) { + const requests: Payload[] = []; + const fetch = globalThis.fetch; + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + requests.push(JSON.parse(Buffer.concat(chunks).toString())); + const call = calls.shift(); + const delta = call ? { role: "assistant", tool_calls: [{ index: 0, id: randomUUID(), + type: "function", function: { name: call.name, arguments: JSON.stringify(call.args ?? {}) } }] } + : { role: "assistant", content: "Done." }; + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(`data: ${JSON.stringify({ choices: [{ index: 0, delta, finish_reason: call ? "tool_calls" : "stop" }] })}\n\ndata: [DONE]\n\n`); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing HTTP address"); + vi.stubGlobal("fetch", ((_url, init) => fetch(`http://127.0.0.1:${address.port}`, init)) satisfies typeof fetch); + return { requests, close: async () => { + vi.unstubAllGlobals(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } }; +} +export function runtimeFixture(history: UiMessage[] = [], tools = pluginTools, provider = flashProvider(), denied = false) { + const rows = structuredClone(history); + const executed: string[] = []; + const errors: unknown[] = []; + const runtime = new DesktopAgentRuntime({ + sessionId: "fixed-tools", mode: "agent", provider, thinkingLevel: "off", history: rows, pluginTools: tools, + commandShell: { id: "bash", label: "Bash", dialect: "posix", available: true, isDefault: true }, + host: { call: async (method: string, params?: unknown): Promise => { + if (method === "session.appendMessage") rows.push((params as { message: UiMessage }).message); + else if (method === "tools.execute") { + executed.push((params as { toolName: string }).toolName); + return denied ? { ok: false, denied: true, content: "Permission denied" } as T + : { ok: true, content: "Synthetic success" } as T; + } else throw new Error(`Unexpected host method: ${method}`); + return undefined as T; + } }, + onEvent: ({ event }) => { + if (event.type === "error") errors.push(event.error); + if (event.type === "tool_start") rows.push({ id: event.toolCallId, role: "tool", content: "", + toolCallId: event.toolCallId, toolName: event.toolName, toolArgs: event.args, createdAt: new Date().toISOString() }); + if (event.type === "tool_end") { + const row = rows.find((row) => row.id === event.toolCallId)!; + row.toolResult = event.result; row.isError = event.isError; row.toolStatus = event.isError ? "error" : "success"; + } + if (event.type === "message_end") { + const index = rows.findIndex((row) => row.id === event.message.id); + if (index < 0) rows.push(event.message); else rows[index] = event.message; + } + }, + }); + return { runtime, rows, executed, errors, prompt: async (id = "user-1") => { + rows.push({ id, role: "user", content: "Run the synthetic probes", createdAt: new Date().toISOString() }); + await runtime.prompt("Run the synthetic probes", id, `turn-${id}`); + } }; +} diff --git a/scripts/e2e-fixed-tool-declarations.mjs b/scripts/e2e-fixed-tool-declarations.mjs index c44d32ef58..9264a20bad 100644 --- a/scripts/e2e-fixed-tool-declarations.mjs +++ b/scripts/e2e-fixed-tool-declarations.mjs @@ -36,12 +36,15 @@ const server = createServer(async (req, res) => { }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const baseUrl = `http://127.0.0.1:${server.address().port}/v1`; -const model = Object.values(DEEPSEEK_MODELS).find((model) => model.id === "deepseek-flash"); -const provider = { id: "fixture", name: "Flash fixture", modelId: model.id, baseUrl: model.baseUrl, +const flash = Object.values(DEEPSEEK_MODELS).find((model) => model.id === "deepseek-flash"); +const model = process.env.PI_FIXED_TOOL_FIXTURE_ROUTE === "compatible" + ? { ...flash, id: "compatible-fixture", baseUrl: "https://fixture.invalid/v1", compat: undefined } + : flash; +const provider = { id: "fixture", name: "Fixed tools fixture", modelId: model.id, baseUrl: model.baseUrl, modelConfig: modelConfigFromPi(model), apiKey: "fixture", authKind: "api_key", supportsReasoning: false, supportedThinkingLevels: ["off"] }; const fetchHook = join(root, "fixture-fetch.mjs"); await writeFile(fetchHook, `const fetch = globalThis.fetch; globalThis.fetch = (url, init) => { - if (String(url) !== "https://api.deepseek.com/chat/completions") throw new Error("Unexpected fixture endpoint"); + if (String(url) !== ${JSON.stringify(`${model.baseUrl}/chat/completions`)}) throw new Error("Unexpected fixture endpoint"); return fetch(${JSON.stringify(baseUrl)}, init); };`); let pluginTools = ["plugin_alpha", "plugin_beta"].map((name) => ({ name, From 157b55567fa01bdbb17ff96576d865fb9f4a3161 Mon Sep 17 00:00:00 2001 From: zszz3 <91608029+zszz3@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:12:24 +0800 Subject: [PATCH 2/5] test(runtime): verify activation separately in system E2E The unflagged fixture now declares its catalog upfront. Assert persisted activation and stable request schemas instead of expecting a later declaration delta. --- scripts/e2e-system-transcript.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/e2e-system-transcript.mjs b/scripts/e2e-system-transcript.mjs index 6d4139152a..a30350f0ef 100644 --- a/scripts/e2e-system-transcript.mjs +++ b/scripts/e2e-system-transcript.mjs @@ -121,7 +121,8 @@ try { const saved = await detail(); const states = saved.messages.filter((row) => row.modelSystem); assert.equal(states.length, 2); - assert(JSON.parse(states[1].modelSystem.messageJson).toolsAdded.some((tool) => tool.name === "BrowserPreview")); + assert(JSON.parse(JSON.parse(states[1].modelSystem.messageJson).sections.tool_activation).active.includes("BrowserPreview")); + assert.deepEqual(requests[1].tools, requests[0].tools); assert(requests[1].tools.some((tool) => tool.function.name === "BrowserPreview")); console.log("PASS: sidecar ToolSearch and acknowledged Host persistence"); await sidecar.dispose(); From 1cbd42017578205a8a4b7d4d1cbe7db6aaed5821 Mon Sep 17 00:00:00 2001 From: zszz3 <91608029+zszz3@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:24:28 +0800 Subject: [PATCH 3/5] test(runtime): publish process readiness atomically CI observed the readiness file between creation and its JSON write. Rename a completed sibling file into place so process cancellation assertions begin only after the PID list is readable. --- packages/agent-runtime/src/extensions/managed-exec.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/agent-runtime/src/extensions/managed-exec.test.ts b/packages/agent-runtime/src/extensions/managed-exec.test.ts index b41a6beeb9..5b869d663a 100644 --- a/packages/agent-runtime/src/extensions/managed-exec.test.ts +++ b/packages/agent-runtime/src/extensions/managed-exec.test.ts @@ -8,7 +8,8 @@ it("cancels an owned process tree after an explicit readiness signal", async () const root = mkdtempSync(join(tmpdir(), "pi-owned-exec-")); const ready = join(root, "ready.json"); const owner = new AbortController(); - const childSource = `require('node:fs').writeFileSync(${JSON.stringify(ready)}, JSON.stringify([process.ppid,process.pid])); setInterval(()=>{},1000);`; + // Publish readiness only after the complete PID list is visible to the reader. + const childSource = `const fs=require('node:fs'); fs.writeFileSync(${JSON.stringify(`${ready}.tmp`)}, JSON.stringify([process.ppid,process.pid])); fs.renameSync(${JSON.stringify(`${ready}.tmp`)}, ${JSON.stringify(ready)}); setInterval(()=>{},1000);`; const parentSource = `require('node:child_process').spawn(process.execPath,['-e',${JSON.stringify(childSource)}],{stdio:'inherit'}); setInterval(()=>{},1000);`; const pending = managedExec(process.execPath, ["-e", parentSource], root, owner.signal); try { From 1982cc0de9e2594792b68f17abe13587a870e4eb Mon Sep 17 00:00:00 2001 From: zszz3 <91608029+zszz3@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:32:34 +0800 Subject: [PATCH 4/5] docs(runtime): clarify pinned Anthropic tool transport Distinguish the pinned adapter behavior from the provider inline-tool beta. System-message and tool-reference capabilities cannot enable an unsupported wire format. --- docs/adr/chronological-system-transcript.md | 6 ++++-- docs/spec/03-runtime/02-agent-runtime.md | 6 ++++-- docs/zh-CN/spec/03-runtime/02-agent-runtime.md | 8 +++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/adr/chronological-system-transcript.md b/docs/adr/chronological-system-transcript.md index 1177064d94..cb8ef5f990 100644 --- a/docs/adr/chronological-system-transcript.md +++ b/docs/adr/chronological-system-transcript.md @@ -60,8 +60,10 @@ model-name allowlist. Responses (OpenAI/Codex) with verified system support and `additional_tools` or client tool search, Chat Completions with verified system/tool additions, and Pi's transcript transport retain native chronological additions. Other transports declare the complete current catalog in deterministic -name order from the first request. In particular Anthropic's native transition -blocks still grow its request-level schemas, so they use fixed declarations. +name order from the first request. In particular the pinned Pi 1.0 Anthropic +adapter's native transition blocks still grow its request-level schemas, so they use fixed declarations. +Anthropic's separate inline-definition beta is not wired by that adapter; +mid-conversation system/tool-reference support alone must not enable it. This policy also covers compatible relays without enabling unsupported native message roles or switching their configured API. diff --git a/docs/spec/03-runtime/02-agent-runtime.md b/docs/spec/03-runtime/02-agent-runtime.md index a0269ea9b0..d663446b35 100644 --- a/docs/spec/03-runtime/02-agent-runtime.md +++ b/docs/spec/03-runtime/02-agent-runtime.md @@ -1458,8 +1458,10 @@ support plus `supportsAdditionalTools` or `supportsToolSearch`, Chat Completions with verified system/tool additions, and Pi Messages retain native additions. Other bindings, including Anthropic Messages, Gemini, ordinary Chat Completions, older Responses/Codex models and compatible relays, declare the complete catalog -in deterministic name order on the first request. Anthropic's native tool-change -blocks still grow request-level schemas, so do not exempt them. ToolSearch changes +in deterministic name order on the first request. The pinned Pi 1.0 Anthropic +adapter's native tool-change blocks still grow request-level schemas, so do not exempt them. The separate +inline-definition beta is not wired by this adapter; do not infer it from +system-message or tool-reference capability flags. ToolSearch changes activation without changing the declared schemas. A visible schema does not permit execution: inactive deferred calls are rejected before extension hooks and the Host; activated calls still require the existing mode and Host checks. diff --git a/docs/zh-CN/spec/03-runtime/02-agent-runtime.md b/docs/zh-CN/spec/03-runtime/02-agent-runtime.md index 485f191b10..6e5571171a 100644 --- a/docs/zh-CN/spec/03-runtime/02-agent-runtime.md +++ b/docs/zh-CN/spec/03-runtime/02-agent-runtime.md @@ -992,9 +992,11 @@ epochs fail closed. Inactive declared tools are blocked before extension/Host execution, and activation never bypasses mode or approval checks. The full catalog is deterministic from the first request. More than 128 tools or an insufficient context budget falls back to on-demand declarations with a -diagnostic, without truncation. Responses and Chat Completions bindings with verified anchored additions, and -Pi Messages, retain native incremental publication. Anthropic native tool changes -still grow request schemas and therefore use fixed declarations. Strip only the +diagnostic, without truncation. Responses and Chat Completions bindings with +verified anchored additions, and Pi Messages, retain native incremental +publication. The pinned Pi 1.0 adapter's Anthropic native tool changes still grow +request schemas and therefore use fixed declarations. Its separate inline-definition +beta is not wired; system-message/tool-reference flags do not enable it. Strip only the private activation section from provider projection, preserving canonical state and all other instructions. This also stabilizes ToolSearch on folding APIs and compatible relays without enabling new transport capabilities. From 6492b63aed71562d4fbc5a680d55c693130bf92e Mon Sep 17 00:00:00 2001 From: zszz3 <91608029+zszz3@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:08:57 +0800 Subject: [PATCH 5/5] fix(runtime): retain Pi 1.0.1 inline Anthropic tools Keep verified Anthropic tool changes on their native append-only path now that Pi sends full definitions inline. System-only bindings still need a fixed catalog, and the request matrix covers both capability boundaries. --- docs/adr/chronological-system-transcript.md | 11 ++++++----- docs/spec/03-runtime/02-agent-runtime.md | 10 +++++----- docs/spec/06-delivery/04-e2e-test-plan.md | 8 ++++++-- docs/zh-CN/spec/03-runtime/02-agent-runtime.md | 7 ++++--- docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md | 8 ++++++-- packages/agent-runtime/src/fixed-tool-declarations.ts | 5 ++++- .../agent-runtime/src/fixed-tool-providers.test.ts | 9 +++++++-- 7 files changed, 38 insertions(+), 20 deletions(-) diff --git a/docs/adr/chronological-system-transcript.md b/docs/adr/chronological-system-transcript.md index cb8ef5f990..aac9932869 100644 --- a/docs/adr/chronological-system-transcript.md +++ b/docs/adr/chronological-system-transcript.md @@ -45,7 +45,7 @@ transport support from a models.dev metadata match or an account endpoint override. Both Pi and models.dev metadata projections can carry that binding; unverified routes and generic records retain the conservative fallback. -The Pi 1.0.0 dependency patch adds the missing mid-conversation system +The Pi 1.0.1 dependency patch adds the missing mid-conversation system capability to its `deepseek-flash` catalog entry. Authorized official-endpoint experiments confirmed both preserved cache reuse and effective updated instructions. Keep this correction in the single Pi catalog, not a parallel @@ -60,10 +60,11 @@ model-name allowlist. Responses (OpenAI/Codex) with verified system support and `additional_tools` or client tool search, Chat Completions with verified system/tool additions, and Pi's transcript transport retain native chronological additions. Other transports declare the complete current catalog in deterministic -name order from the first request. In particular the pinned Pi 1.0 Anthropic -adapter's native transition blocks still grow its request-level schemas, so they use fixed declarations. -Anthropic's separate inline-definition beta is not wired by that adapter; -mid-conversation system/tool-reference support alone must not enable it. +name order from the first request. The pinned Pi 1.0.1 Anthropic adapter now +supports inline tool definitions: verified system and tool-change support retain +native `tool_addition` messages without growing the initial schema list. Claude +bindings with only system-message support still use fixed declarations. Do not +infer tool-change support from a model name or unverified relay. This policy also covers compatible relays without enabling unsupported native message roles or switching their configured API. diff --git a/docs/spec/03-runtime/02-agent-runtime.md b/docs/spec/03-runtime/02-agent-runtime.md index d663446b35..56a86c6226 100644 --- a/docs/spec/03-runtime/02-agent-runtime.md +++ b/docs/spec/03-runtime/02-agent-runtime.md @@ -1456,12 +1456,12 @@ Only names in the current mode's deferred catalog are eligible. Select by the bound transport, not a model name. Responses with verified system support plus `supportsAdditionalTools` or `supportsToolSearch`, Chat Completions with verified system/tool additions, and Pi Messages retain native additions. -Other bindings, including Anthropic Messages, Gemini, ordinary Chat Completions, +The Pi 1.0.1 Anthropic adapter with verified system and tool-change support also +retains native additions, carrying full definitions in later `tool_addition` +blocks. System-message support alone does not enable this path. Other bindings, +including Claude without native tool changes, Gemini, ordinary Chat Completions, older Responses/Codex models and compatible relays, declare the complete catalog -in deterministic name order on the first request. The pinned Pi 1.0 Anthropic -adapter's native tool-change blocks still grow request-level schemas, so do not exempt them. The separate -inline-definition beta is not wired by this adapter; do not infer it from -system-message or tool-reference capability flags. ToolSearch changes +in deterministic name order on the first request. ToolSearch changes activation without changing the declared schemas. A visible schema does not permit execution: inactive deferred calls are rejected before extension hooks and the Host; activated calls still require the existing mode and Host checks. diff --git a/docs/spec/06-delivery/04-e2e-test-plan.md b/docs/spec/06-delivery/04-e2e-test-plan.md index 0faa5a6ef7..6d0016788c 100644 --- a/docs/spec/06-delivery/04-e2e-test-plan.md +++ b/docs/spec/06-delivery/04-e2e-test-plan.md @@ -16523,13 +16523,17 @@ renderer's durable transcript reads. No real model or provider is contacted. - Cross-provider acceptance: `fixed-tool-providers.test.ts` enters the real runtime prompt/ToolSearch/execution path and captures each Desktop-selectable Pi adapter's actual serialized payload at `onPayload`, before network dispatch. - Cover Chat Completions, Anthropic (native system on/off), Responses and Codex + Cover Chat Completions, Anthropic (no native updates, system-only, inline tools), Responses and Codex (fallback, additional tools, client tool search), Gemini and Pi Messages. Search A, execute A, search B, execute B, then finish: all five requests retain their top-level schema state and prior semantic message prefix. Native routes retain deferred schema additions. Activation JSON never reaches the provider. + Anthropic inline definitions must remain absent from the first request and + appear as `tool_definition` blocks after successful ToolSearch. This proves request construction, not server cache hits or paid API acceptance. -- The local HTTP/SSE fixture additionally covers official Flash, unflagged Chat +- Run `node scripts/e2e-fixed-tool-declarations.mjs` for the official Flash + binding and `PI_FIXED_TOOL_FIXTURE_ROUTE=compatible node scripts/e2e-fixed-tool-declarations.mjs` + for the compatible binding. The local HTTP/SSE fixture covers official Flash, unflagged Chat Completions and a compatible relay. Canonical activation restoration, denial before Host execution, mode/account/catalog invalidation and compaction remain required. Fixed declarations increase first-request size; oversized catalogs diff --git a/docs/zh-CN/spec/03-runtime/02-agent-runtime.md b/docs/zh-CN/spec/03-runtime/02-agent-runtime.md index 6e5571171a..8135691007 100644 --- a/docs/zh-CN/spec/03-runtime/02-agent-runtime.md +++ b/docs/zh-CN/spec/03-runtime/02-agent-runtime.md @@ -994,9 +994,10 @@ catalog is deterministic from the first request. More than 128 tools or an insufficient context budget falls back to on-demand declarations with a diagnostic, without truncation. Responses and Chat Completions bindings with verified anchored additions, and Pi Messages, retain native incremental -publication. The pinned Pi 1.0 adapter's Anthropic native tool changes still grow -request schemas and therefore use fixed declarations. Its separate inline-definition -beta is not wired; system-message/tool-reference flags do not enable it. Strip only the +publication. The pinned Pi 1.0.1 Anthropic adapter also retains native additions +with verified system and tool-change support: later messages carry full tool +definitions. System-message support alone is insufficient; other Claude bindings +use fixed declarations. Strip only the private activation section from provider projection, preserving canonical state and all other instructions. This also stabilizes ToolSearch on folding APIs and compatible relays without enabling new transport capabilities. diff --git a/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md b/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md index ae32e27420..cd77431506 100644 --- a/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md +++ b/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md @@ -9326,13 +9326,17 @@ the latest destination. These assertions measure work counts, not device FPS. - Cross-provider acceptance: `fixed-tool-providers.test.ts` enters the real runtime prompt/ToolSearch/execution path and captures each Desktop-selectable Pi adapter's actual serialized payload at `onPayload`, before network dispatch. - Cover Chat Completions, Anthropic (native system on/off), Responses and Codex + Cover Chat Completions, Anthropic (no native updates, system-only, inline tools), Responses and Codex (fallback, additional tools, client tool search), Gemini and Pi Messages. Search A, execute A, search B, execute B, then finish: all five requests retain their top-level schema state and prior semantic message prefix. Native routes retain deferred schema additions. Activation JSON never reaches the provider. + Anthropic inline definitions must remain absent from the first request and + appear as `tool_definition` blocks after successful ToolSearch. This proves request construction, not server cache hits or paid API acceptance. -- The local HTTP/SSE fixture additionally covers official Flash, unflagged Chat +- Run `node scripts/e2e-fixed-tool-declarations.mjs` for the official Flash + binding and `PI_FIXED_TOOL_FIXTURE_ROUTE=compatible node scripts/e2e-fixed-tool-declarations.mjs` + for the compatible binding. The local HTTP/SSE fixture covers official Flash, unflagged Chat Completions and a compatible relay. Canonical activation restoration, denial before Host execution, mode/account/catalog invalidation and compaction remain required. Fixed declarations increase first-request size; oversized catalogs diff --git a/packages/agent-runtime/src/fixed-tool-declarations.ts b/packages/agent-runtime/src/fixed-tool-declarations.ts index cf3a450fd1..3a8c4fc6c0 100644 --- a/packages/agent-runtime/src/fixed-tool-declarations.ts +++ b/packages/agent-runtime/src/fixed-tool-declarations.ts @@ -26,11 +26,14 @@ function hasAnchoredToolAdditions(model: Model): boolean { if (model.api === "openai-completions") { return "supportsMidConvoToolAdditions" in compat && compat.supportsMidConvoToolAdditions === true; } + if (model.api === "anthropic-messages") { + // Pi 1.0.1 sends later schemas inline rather than growing top-level tools. + return "supportsMidConvoToolChanges" in compat && compat.supportsMidConvoToolChanges === true; + } if (["openai-responses", "openai-codex-responses"].includes(model.api)) { return ("supportsAdditionalTools" in compat && compat.supportsAdditionalTools === true) || ("supportsToolSearch" in compat && compat.supportsToolSearch === true); } - // Anthropic's native tool-change blocks still grow its request-level schemas. return false; } diff --git a/packages/agent-runtime/src/fixed-tool-providers.test.ts b/packages/agent-runtime/src/fixed-tool-providers.test.ts index 12f8b24395..9c63b6f569 100644 --- a/packages/agent-runtime/src/fixed-tool-providers.test.ts +++ b/packages/agent-runtime/src/fixed-tool-providers.test.ts @@ -9,7 +9,8 @@ const base = Object.values(DEEPSEEK_MODELS).find((model) => model.id === "deepse const routes: { api: Api; compat?: Model["compat"]; native?: boolean; label: string }[] = [ { label: "Chat Completions", api: "openai-completions" }, { label: "Claude without native transitions", api: "anthropic-messages" }, - { label: "Claude with native transitions", api: "anthropic-messages", compat: { supportsMidConvoSystemMessages: true, supportsMidConvoToolChanges: true } }, + { label: "Claude with native transitions", api: "anthropic-messages", native: true, compat: { supportsMidConvoSystemMessages: true, supportsMidConvoToolChanges: true } }, + { label: "Claude with system updates only", api: "anthropic-messages", compat: { supportsMidConvoSystemMessages: true, supportsMidConvoToolChanges: false } }, { label: "OpenAI Responses fallback", api: "openai-responses" }, { label: "Codex fallback", api: "openai-codex-responses" }, { label: "Gemini", api: "google-generative-ai" }, @@ -19,7 +20,8 @@ const routes: { api: Api; compat?: Model["compat"]; native?: boolean; label { label: "Codex native additions", api: "openai-codex-responses", native: true, compat: { supportsMidConvoSystemMessages: true, supportsAdditionalTools: true } }, { label: "Pi transcript", api: "pi-messages", native: true }, ]; -// Cache markers move with the last message; they are not model input text. +// Compare semantic prefixes; cache breakpoints move with the last message. +// This intentionally does not assert server cache placement or hit rates. function semantic(value: unknown): unknown { if (Array.isArray(value)) return value.map(semantic); if (value && typeof value === "object") return Object.fromEntries(Object.entries(value) @@ -86,6 +88,9 @@ describe("ToolSearch user path through every Desktop-selectable Pi request adapt else { expect(JSON.stringify(requests[0].tools) ?? "").not.toContain("plugin_beta"); expect(JSON.stringify(requests.at(-1)?.messages)).toContain("plugin_beta"); + if (api === "anthropic-messages") { + expect(JSON.stringify(requests.at(-1)?.messages)).toContain('"type":"tool_definition"'); + } } for (let index = 1; index < requests.length; index++) { expect(requests[index].tools).toEqual(requests[0].tools);