From 20c9e8134f2c32afd3d96335e8e50efd49e5d4a1 Mon Sep 17 00:00:00 2001 From: James Sterling Date: Thu, 24 Sep 2026 16:00:04 +0000 Subject: [PATCH] fix(bench): keep effect/Context out of config schemas and record the effective search request Move stripVariantSuffix into an Effect-free module so BenchmarkRunConfigSchema can load inside the Temporal workflow sandbox, which rejects the frozen-global write effect/Context performs on import. Persist the switchyard plugin in the search solver's recorded requestBody so the stored request matches what was sent. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- package.json | 1 + src/benchmarks/benchmark-config.ts | 2 +- src/benchmarks/search/core/solver.test.ts | 7 ++++-- src/benchmarks/search/core/solver.ts | 5 ++++- src/harness/model-slug.test.ts | 27 +++++++++++++++++++++++ src/harness/model-slug.ts | 4 ++++ src/harness/model.ts | 5 +---- 7 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 src/harness/model-slug.test.ts create mode 100644 src/harness/model-slug.ts diff --git a/package.json b/package.json index f37ea59..9128cb1 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "./dataset": "./src/harness/dataset.ts", "./datasets/huggingface": "./src/datasets/huggingface.ts", "./model": "./src/harness/model.ts", + "./model-slug": "./src/harness/model-slug.ts", "./metric": "./src/harness/metric.ts", "./providers/openrouter-model": "./src/providers/openrouter-model.ts", "./run": "./src/harness/run.ts", diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index 4727c98..f63469f 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -5,7 +5,7 @@ import { SWITCHYARD_ALGORITHMS, VIDEO_PROCESSING_MODES, } from "../harness/constants"; -import { stripVariantSuffix } from "../harness/model"; +import { stripVariantSuffix } from "../harness/model-slug"; import { ProviderSort } from "../internal/enums"; import type { ValueOf } from "../internal/guards"; import { z, zDefaultedText, zInt } from "../internal/zod"; diff --git a/src/benchmarks/search/core/solver.test.ts b/src/benchmarks/search/core/solver.test.ts index df9fc6c..49cbb0e 100644 --- a/src/benchmarks/search/core/solver.test.ts +++ b/src/benchmarks/search/core/solver.test.ts @@ -318,9 +318,11 @@ describe("searchSolver", () => { }); it("appends the switchyard-router plugin after the serialized web plugin", async () => { let sentOptions: ResponsesSendOptions | undefined; + let sent: ResponsesRequest | undefined; const solver = searchSolver( { - send: (_body, options) => { + send: (body, options) => { + sent = body; sentOptions = options; return effectSucceed(fixtureResult({ text: "x" })); }, @@ -332,7 +334,7 @@ describe("searchSolver", () => { switchyardAlgorithm: "stage", } ); - await runSolver( + const state = await runSolver( solver(initialTaskState({ id: "s", input: "q", target: { text: "t" } })) ); expect(sentOptions?.extraBody).toEqual({ @@ -341,6 +343,7 @@ describe("searchSolver", () => { { id: "switchyard-router", algorithm: "stage" }, ], }); + expect(state.requestBody).toEqual({ ...sent, ...sentOptions?.extraBody }); }); it("sends only the switchyard-router plugin with server tools, ignoring the variant suffix", async () => { let sentOptions: ResponsesSendOptions | undefined; diff --git a/src/benchmarks/search/core/solver.ts b/src/benchmarks/search/core/solver.ts index c863321..df157a3 100644 --- a/src/benchmarks/search/core/solver.ts +++ b/src/benchmarks/search/core/solver.ts @@ -169,6 +169,7 @@ export function searchSolver( return completedState({ state, request: body, + extraBody, result, attemptResults, text: result.text.trim(), @@ -262,12 +263,14 @@ function makeStreamEventReporter( function completedState({ state, request, + extraBody, result, attemptResults, text, }: { readonly state: TaskState; readonly request: ResponsesRequest; + readonly extraBody: Readonly> | undefined; readonly result: ResponsesResult; readonly attemptResults: readonly ResponsesResult[]; readonly text: string; @@ -304,7 +307,7 @@ function completedState({ : []), ], responseItems: responseItemsForCall(request, result), - requestBody: { ...request }, + requestBody: { ...request, ...extraBody }, output: definedValues({ completion: text, message: { role: MessageRole.Assistant, content: text }, diff --git a/src/harness/model-slug.test.ts b/src/harness/model-slug.test.ts new file mode 100644 index 0000000..072ece4 --- /dev/null +++ b/src/harness/model-slug.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "bun:test"; + +import { stripVariantSuffix } from "./model-slug"; + +describe("stripVariantSuffix", () => { + it.each([ + ["nvidia/switchyard:online", "nvidia/switchyard"], + ["openai/gpt-5:free:online", "openai/gpt-5"], + ["openai/gpt-5", "openai/gpt-5"], + [":free", ":free"], + ] as const)("strips the variant from %s", (model, expected) => { + expect(stripVariantSuffix(model)).toBe(expected); + }); + + it("keeps effect/Context out of the benchmark config module graph", () => { + const script = [ + `await import(${JSON.stringify(`${import.meta.dir}/../benchmarks/benchmark-config.ts`)});`, + "const loaded = Object.keys(require.cache).filter((k) => k.endsWith('/effect/dist/esm/Context.js'));", + "console.log(loaded.length);", + ].join("\n"); + + const result = Bun.spawnSync({ cmd: ["bun", "-e", script] }); + + expect(result.stderr.toString()).toBe(""); + expect(result.stdout.toString().trim()).toBe("0"); + }); +}); diff --git a/src/harness/model-slug.ts b/src/harness/model-slug.ts new file mode 100644 index 0000000..c77327c --- /dev/null +++ b/src/harness/model-slug.ts @@ -0,0 +1,4 @@ +export function stripVariantSuffix(model: string): string { + const idx = model.indexOf(":"); + return idx <= 0 ? model : model.slice(0, idx); +} diff --git a/src/harness/model.ts b/src/harness/model.ts index b540a83..0711e21 100644 --- a/src/harness/model.ts +++ b/src/harness/model.ts @@ -14,10 +14,7 @@ import type { ToolDefinition, } from "./core"; -export function stripVariantSuffix(model: string): string { - const idx = model.indexOf(":"); - return idx <= 0 ? model : model.slice(0, idx); -} +export { stripVariantSuffix } from "./model-slug"; export interface GenerateConfig { readonly temperature?: number;