From fe5bb2d201b4ceac82a9707b2501e1c62f3c50d1 Mon Sep 17 00:00:00 2001 From: Abhinav Pola Date: Sat, 26 Sep 2026 00:15:24 +0000 Subject: [PATCH 1/2] refactor: rename the native benchmark set to Mendel --- package.json | 2 +- src/benchmarks/benchmark-config.test.ts | 10 ++++---- src/benchmarks/benchmark-config.ts | 32 ++++++++++++------------- src/benchmarks/types.ts | 6 ++--- src/runner/run-by-id.test.ts | 6 ++--- src/runner/run-by-id.ts | 14 +++++------ 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/package.json b/package.json index b5a887f..81d4319 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@openrouter/bench-harness", "version": "0.0.0", - "description": "Native TypeScript benchmark harness (inspect-ai-style primitives, Effect-based)", + "description": "Mendel, the OpenRouter TypeScript benchmark harness (inspect-ai-style primitives, Effect-based)", "packageManager": "bun@1.3.14", "type": "module", "exports": { diff --git a/src/benchmarks/benchmark-config.test.ts b/src/benchmarks/benchmark-config.test.ts index b88b874..71cd84e 100644 --- a/src/benchmarks/benchmark-config.test.ts +++ b/src/benchmarks/benchmark-config.test.ts @@ -9,7 +9,7 @@ import { isInjectedBenchmarkConfig, isModelBenchmarkConfig, isSearchBenchmarkConfig, - NativeBenchmarkRunConfigSchema, + MendelBenchmarkRunConfigSchema, } from "./benchmark-config"; describe("benchmark config", () => { @@ -23,8 +23,8 @@ describe("benchmark config", () => { expect(result.status).toBe(0); }); - it("keeps native configs precise while parsing them through the native schema", () => { - const result = parseSchema(NativeBenchmarkRunConfigSchema, { + it("keeps Mendel configs precise while parsing them through the Mendel schema", () => { + const result = parseSchema(MendelBenchmarkRunConfigSchema, { benchmarkId: "search_hle", model: "openai/gpt-5.4", reasoningEffort: "high", @@ -40,7 +40,7 @@ describe("benchmark config", () => { expect(isModelBenchmarkConfig(result.right)).toBe(true); }); - it("does not let malformed native configs fall through to the injected variant", () => { + it("does not let malformed Mendel configs fall through to the injected variant", () => { const result = parseSchema(BenchmarkRunConfigSchema, { benchmarkId: "gpqa_diamond", model: 42, @@ -140,7 +140,7 @@ describe("benchmark config", () => { expect(result.right.options).toEqual({}); }); - it("rejects an injected config that reuses a native benchmark id", () => { + it("rejects an injected config that reuses a Mendel benchmark id", () => { const result = parseSchema(InjectedBenchmarkRunConfigSchema, { benchmarkId: "gpqa_diamond", model: "injected/model", diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index 953021c..b419218 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -329,7 +329,7 @@ export type SearchBenchmarkConfig = | DsqaBenchmarkConfig | WideSearchBenchmarkConfig; -export const NativeBenchmarkRunConfigSchema = z.discriminatedUnion( +export const MendelBenchmarkRunConfigSchema = z.discriminatedUnion( "benchmarkId", [ GpqaBenchmarkConfigSchema, @@ -353,18 +353,18 @@ export const NativeBenchmarkRunConfigSchema = z.discriminatedUnion( ] ); -export type NativeBenchmarkRunConfig = z.infer< - typeof NativeBenchmarkRunConfigSchema +export type MendelBenchmarkRunConfig = z.infer< + typeof MendelBenchmarkRunConfigSchema >; -type NativeModelBenchmarkConfig = Extract< - NativeBenchmarkRunConfig, +type MendelModelBenchmarkConfig = Extract< + MendelBenchmarkRunConfig, { model: string; } >; -export type ModelBenchmarkId = NativeModelBenchmarkConfig["benchmarkId"]; +export type ModelBenchmarkId = MendelModelBenchmarkConfig["benchmarkId"]; export const BENCHMARK_OPTIONS_SCHEMAS = { gpqa_diamond: GpqaOptionsSchema, @@ -386,8 +386,8 @@ export const BENCHMARK_OPTIONS_SCHEMAS = { vgi_bench: VgiBenchOptionsSchema, } as const satisfies Record>; -const NATIVE_BENCHMARK_ID_SET: ReadonlySet = new Set( - NativeBenchmarkRunConfigSchema.options.flatMap((schema) => { +const MENDEL_BENCHMARK_ID_SET: ReadonlySet = new Set( + MendelBenchmarkRunConfigSchema.options.flatMap((schema) => { const benchmarkId = schema.shape.benchmarkId; return benchmarkId instanceof z.ZodLiteral ? [benchmarkId.value] : []; }) @@ -397,8 +397,8 @@ const InjectedBenchmarkIdSchema = z .string() .min(1) .refine( - (benchmarkId) => !NATIVE_BENCHMARK_ID_SET.has(benchmarkId), - "Injected benchmark ids must not reuse native benchmark ids" + (benchmarkId) => !MENDEL_BENCHMARK_ID_SET.has(benchmarkId), + "Injected benchmark ids must not reuse Mendel benchmark ids" ); export const InjectedBenchmarkRunConfigSchema = z.object({ @@ -412,7 +412,7 @@ export type InjectedBenchmarkRunConfig = z.infer< >; export const BenchmarkRunConfigSchema = z - .union([NativeBenchmarkRunConfigSchema, InjectedBenchmarkRunConfigSchema]) + .union([MendelBenchmarkRunConfigSchema, InjectedBenchmarkRunConfigSchema]) .refine( (config) => !("model" in config) || @@ -427,7 +427,7 @@ export const BenchmarkRunConfigSchema = z export type BenchmarkRunConfig = z.infer; export type ModelBenchmarkConfig = - | NativeModelBenchmarkConfig + | MendelModelBenchmarkConfig | InjectedBenchmarkRunConfig; export function isModelBenchmarkConfig( @@ -436,16 +436,16 @@ export function isModelBenchmarkConfig( return "model" in config; } -export function isNativeBenchmarkConfig( +export function isMendelBenchmarkConfig( config: BenchmarkRunConfig -): config is NativeBenchmarkRunConfig { - return NATIVE_BENCHMARK_ID_SET.has(config.benchmarkId); +): config is MendelBenchmarkRunConfig { + return MENDEL_BENCHMARK_ID_SET.has(config.benchmarkId); } export function isInjectedBenchmarkConfig( config: BenchmarkRunConfig ): config is InjectedBenchmarkRunConfig { - return !isNativeBenchmarkConfig(config); + return !isMendelBenchmarkConfig(config); } const SEARCH_BENCHMARK_ID_SET: ReadonlySet = new Set([ diff --git a/src/benchmarks/types.ts b/src/benchmarks/types.ts index 1316eb8..9bd0407 100644 --- a/src/benchmarks/types.ts +++ b/src/benchmarks/types.ts @@ -10,11 +10,11 @@ import type { ResponsesModel } from "../providers/responses-model"; import type { RetryConfig } from "../runtime/retry"; import type { BenchmarkRunConfig, - NativeBenchmarkRunConfig, + MendelBenchmarkRunConfig, } from "./benchmark-config"; export interface BenchmarkRunInput< - Config extends BenchmarkRunConfig = NativeBenchmarkRunConfig, + Config extends BenchmarkRunConfig = MendelBenchmarkRunConfig, > { readonly apiKey: string; readonly baseUrl?: string; @@ -38,7 +38,7 @@ export interface BenchmarkPrimaryScore { } export interface Benchmark< - Config extends BenchmarkRunConfig = NativeBenchmarkRunConfig, + Config extends BenchmarkRunConfig = MendelBenchmarkRunConfig, > { readonly id: string; readonly makeDatasetLayer: (retryConfig?: RetryConfig) => Layer; diff --git a/src/runner/run-by-id.test.ts b/src/runner/run-by-id.test.ts index 2954477..052e51e 100644 --- a/src/runner/run-by-id.test.ts +++ b/src/runner/run-by-id.test.ts @@ -37,7 +37,7 @@ const INJECTED_CONFIG = { } as const; describe("benchmark runner by id", () => { - it("runs an injected benchmark instead of the native registry", async () => { + it("runs an injected benchmark instead of the Mendel registry", async () => { const result = await runBenchmarkById({ benchmarkId: INJECTED_BENCHMARK.id, injectedBenchmark: INJECTED_BENCHMARK, @@ -66,7 +66,7 @@ describe("benchmark runner by id", () => { expect(result.left).toContain("injected benchmark is required"); }); - it("rejects an injected benchmark for a native config", async () => { + it("rejects an injected benchmark for a Mendel config", async () => { const result = await runBenchmarkById({ benchmarkId: "search_hle", injectedBenchmark: INJECTED_BENCHMARK, @@ -81,7 +81,7 @@ describe("benchmark runner by id", () => { }); assertLeft(result); - expect(result.left).toContain("cannot be supplied for native benchmark"); + expect(result.left).toContain("cannot be supplied for Mendel benchmark"); }); it("resolves dataset size from an injected benchmark", async () => { diff --git a/src/runner/run-by-id.ts b/src/runner/run-by-id.ts index eaba83c..6de82e8 100644 --- a/src/runner/run-by-id.ts +++ b/src/runner/run-by-id.ts @@ -12,7 +12,7 @@ import type { InjectedBenchmarkRunConfig, } from "../benchmarks/benchmark-config"; import { - isNativeBenchmarkConfig, + isMendelBenchmarkConfig, modelFromConfig, } from "../benchmarks/benchmark-config"; import { getBenchmark } from "../benchmarks/registry"; @@ -241,20 +241,20 @@ function resolveRunBenchmark(input: RunBenchmarkInput): Either.Either< }, string > { - if (isNativeBenchmarkConfig(input.benchmarkConfig)) { + if (isMendelBenchmarkConfig(input.benchmarkConfig)) { if (input.injectedBenchmark !== undefined) { return Either.left( - `An injected benchmark cannot be supplied for native benchmark "${input.benchmarkId}"` + `An injected benchmark cannot be supplied for Mendel benchmark "${input.benchmarkId}"` ); } - const nativeBenchmark = getBenchmark(input.benchmarkId); - if (nativeBenchmark === undefined) { + const mendelBenchmark = getBenchmark(input.benchmarkId); + if (mendelBenchmark === undefined) { return Either.left(`Unknown benchmark "${input.benchmarkId}"`); } return Either.right({ - benchmark: nativeBenchmark, + benchmark: mendelBenchmark, benchmarkLayer: makeBenchmarkLayer( - nativeBenchmark, + mendelBenchmark, input, input.benchmarkConfig ), From ef32fa66ec9b1dbdccd0bb6cf3d5d8b791455479 Mon Sep 17 00:00:00 2001 From: Abhinav Pola Date: Sat, 26 Sep 2026 00:30:27 +0000 Subject: [PATCH 2/2] refactor: call the TypeScript benchmark harness Kepler --- package.json | 2 +- src/benchmarks/benchmark-config.test.ts | 10 ++++---- src/benchmarks/benchmark-config.ts | 32 ++++++++++++------------- src/benchmarks/types.ts | 6 ++--- src/runner/run-by-id.test.ts | 6 ++--- src/runner/run-by-id.ts | 14 +++++------ 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/package.json b/package.json index 81d4319..8bf385d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@openrouter/bench-harness", "version": "0.0.0", - "description": "Mendel, the OpenRouter TypeScript benchmark harness (inspect-ai-style primitives, Effect-based)", + "description": "Kepler, the OpenRouter TypeScript benchmark harness (inspect-ai-style primitives, Effect-based)", "packageManager": "bun@1.3.14", "type": "module", "exports": { diff --git a/src/benchmarks/benchmark-config.test.ts b/src/benchmarks/benchmark-config.test.ts index 71cd84e..fc5afc4 100644 --- a/src/benchmarks/benchmark-config.test.ts +++ b/src/benchmarks/benchmark-config.test.ts @@ -9,7 +9,7 @@ import { isInjectedBenchmarkConfig, isModelBenchmarkConfig, isSearchBenchmarkConfig, - MendelBenchmarkRunConfigSchema, + KeplerBenchmarkRunConfigSchema, } from "./benchmark-config"; describe("benchmark config", () => { @@ -23,8 +23,8 @@ describe("benchmark config", () => { expect(result.status).toBe(0); }); - it("keeps Mendel configs precise while parsing them through the Mendel schema", () => { - const result = parseSchema(MendelBenchmarkRunConfigSchema, { + it("keeps Kepler configs precise while parsing them through the Kepler schema", () => { + const result = parseSchema(KeplerBenchmarkRunConfigSchema, { benchmarkId: "search_hle", model: "openai/gpt-5.4", reasoningEffort: "high", @@ -40,7 +40,7 @@ describe("benchmark config", () => { expect(isModelBenchmarkConfig(result.right)).toBe(true); }); - it("does not let malformed Mendel configs fall through to the injected variant", () => { + it("does not let malformed Kepler configs fall through to the injected variant", () => { const result = parseSchema(BenchmarkRunConfigSchema, { benchmarkId: "gpqa_diamond", model: 42, @@ -140,7 +140,7 @@ describe("benchmark config", () => { expect(result.right.options).toEqual({}); }); - it("rejects an injected config that reuses a Mendel benchmark id", () => { + it("rejects an injected config that reuses a Kepler benchmark id", () => { const result = parseSchema(InjectedBenchmarkRunConfigSchema, { benchmarkId: "gpqa_diamond", model: "injected/model", diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index b419218..e3cde56 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -329,7 +329,7 @@ export type SearchBenchmarkConfig = | DsqaBenchmarkConfig | WideSearchBenchmarkConfig; -export const MendelBenchmarkRunConfigSchema = z.discriminatedUnion( +export const KeplerBenchmarkRunConfigSchema = z.discriminatedUnion( "benchmarkId", [ GpqaBenchmarkConfigSchema, @@ -353,18 +353,18 @@ export const MendelBenchmarkRunConfigSchema = z.discriminatedUnion( ] ); -export type MendelBenchmarkRunConfig = z.infer< - typeof MendelBenchmarkRunConfigSchema +export type KeplerBenchmarkRunConfig = z.infer< + typeof KeplerBenchmarkRunConfigSchema >; -type MendelModelBenchmarkConfig = Extract< - MendelBenchmarkRunConfig, +type KeplerModelBenchmarkConfig = Extract< + KeplerBenchmarkRunConfig, { model: string; } >; -export type ModelBenchmarkId = MendelModelBenchmarkConfig["benchmarkId"]; +export type ModelBenchmarkId = KeplerModelBenchmarkConfig["benchmarkId"]; export const BENCHMARK_OPTIONS_SCHEMAS = { gpqa_diamond: GpqaOptionsSchema, @@ -386,8 +386,8 @@ export const BENCHMARK_OPTIONS_SCHEMAS = { vgi_bench: VgiBenchOptionsSchema, } as const satisfies Record>; -const MENDEL_BENCHMARK_ID_SET: ReadonlySet = new Set( - MendelBenchmarkRunConfigSchema.options.flatMap((schema) => { +const KEPLER_BENCHMARK_ID_SET: ReadonlySet = new Set( + KeplerBenchmarkRunConfigSchema.options.flatMap((schema) => { const benchmarkId = schema.shape.benchmarkId; return benchmarkId instanceof z.ZodLiteral ? [benchmarkId.value] : []; }) @@ -397,8 +397,8 @@ const InjectedBenchmarkIdSchema = z .string() .min(1) .refine( - (benchmarkId) => !MENDEL_BENCHMARK_ID_SET.has(benchmarkId), - "Injected benchmark ids must not reuse Mendel benchmark ids" + (benchmarkId) => !KEPLER_BENCHMARK_ID_SET.has(benchmarkId), + "Injected benchmark ids must not reuse Kepler benchmark ids" ); export const InjectedBenchmarkRunConfigSchema = z.object({ @@ -412,7 +412,7 @@ export type InjectedBenchmarkRunConfig = z.infer< >; export const BenchmarkRunConfigSchema = z - .union([MendelBenchmarkRunConfigSchema, InjectedBenchmarkRunConfigSchema]) + .union([KeplerBenchmarkRunConfigSchema, InjectedBenchmarkRunConfigSchema]) .refine( (config) => !("model" in config) || @@ -427,7 +427,7 @@ export const BenchmarkRunConfigSchema = z export type BenchmarkRunConfig = z.infer; export type ModelBenchmarkConfig = - | MendelModelBenchmarkConfig + | KeplerModelBenchmarkConfig | InjectedBenchmarkRunConfig; export function isModelBenchmarkConfig( @@ -436,16 +436,16 @@ export function isModelBenchmarkConfig( return "model" in config; } -export function isMendelBenchmarkConfig( +export function isKeplerBenchmarkConfig( config: BenchmarkRunConfig -): config is MendelBenchmarkRunConfig { - return MENDEL_BENCHMARK_ID_SET.has(config.benchmarkId); +): config is KeplerBenchmarkRunConfig { + return KEPLER_BENCHMARK_ID_SET.has(config.benchmarkId); } export function isInjectedBenchmarkConfig( config: BenchmarkRunConfig ): config is InjectedBenchmarkRunConfig { - return !isMendelBenchmarkConfig(config); + return !isKeplerBenchmarkConfig(config); } const SEARCH_BENCHMARK_ID_SET: ReadonlySet = new Set([ diff --git a/src/benchmarks/types.ts b/src/benchmarks/types.ts index 9bd0407..2ac2064 100644 --- a/src/benchmarks/types.ts +++ b/src/benchmarks/types.ts @@ -10,11 +10,11 @@ import type { ResponsesModel } from "../providers/responses-model"; import type { RetryConfig } from "../runtime/retry"; import type { BenchmarkRunConfig, - MendelBenchmarkRunConfig, + KeplerBenchmarkRunConfig, } from "./benchmark-config"; export interface BenchmarkRunInput< - Config extends BenchmarkRunConfig = MendelBenchmarkRunConfig, + Config extends BenchmarkRunConfig = KeplerBenchmarkRunConfig, > { readonly apiKey: string; readonly baseUrl?: string; @@ -38,7 +38,7 @@ export interface BenchmarkPrimaryScore { } export interface Benchmark< - Config extends BenchmarkRunConfig = MendelBenchmarkRunConfig, + Config extends BenchmarkRunConfig = KeplerBenchmarkRunConfig, > { readonly id: string; readonly makeDatasetLayer: (retryConfig?: RetryConfig) => Layer; diff --git a/src/runner/run-by-id.test.ts b/src/runner/run-by-id.test.ts index 052e51e..36ce04b 100644 --- a/src/runner/run-by-id.test.ts +++ b/src/runner/run-by-id.test.ts @@ -37,7 +37,7 @@ const INJECTED_CONFIG = { } as const; describe("benchmark runner by id", () => { - it("runs an injected benchmark instead of the Mendel registry", async () => { + it("runs an injected benchmark instead of the Kepler registry", async () => { const result = await runBenchmarkById({ benchmarkId: INJECTED_BENCHMARK.id, injectedBenchmark: INJECTED_BENCHMARK, @@ -66,7 +66,7 @@ describe("benchmark runner by id", () => { expect(result.left).toContain("injected benchmark is required"); }); - it("rejects an injected benchmark for a Mendel config", async () => { + it("rejects an injected benchmark for a Kepler config", async () => { const result = await runBenchmarkById({ benchmarkId: "search_hle", injectedBenchmark: INJECTED_BENCHMARK, @@ -81,7 +81,7 @@ describe("benchmark runner by id", () => { }); assertLeft(result); - expect(result.left).toContain("cannot be supplied for Mendel benchmark"); + expect(result.left).toContain("cannot be supplied for Kepler benchmark"); }); it("resolves dataset size from an injected benchmark", async () => { diff --git a/src/runner/run-by-id.ts b/src/runner/run-by-id.ts index 6de82e8..e7e5c56 100644 --- a/src/runner/run-by-id.ts +++ b/src/runner/run-by-id.ts @@ -12,7 +12,7 @@ import type { InjectedBenchmarkRunConfig, } from "../benchmarks/benchmark-config"; import { - isMendelBenchmarkConfig, + isKeplerBenchmarkConfig, modelFromConfig, } from "../benchmarks/benchmark-config"; import { getBenchmark } from "../benchmarks/registry"; @@ -241,20 +241,20 @@ function resolveRunBenchmark(input: RunBenchmarkInput): Either.Either< }, string > { - if (isMendelBenchmarkConfig(input.benchmarkConfig)) { + if (isKeplerBenchmarkConfig(input.benchmarkConfig)) { if (input.injectedBenchmark !== undefined) { return Either.left( - `An injected benchmark cannot be supplied for Mendel benchmark "${input.benchmarkId}"` + `An injected benchmark cannot be supplied for Kepler benchmark "${input.benchmarkId}"` ); } - const mendelBenchmark = getBenchmark(input.benchmarkId); - if (mendelBenchmark === undefined) { + const keplerBenchmark = getBenchmark(input.benchmarkId); + if (keplerBenchmark === undefined) { return Either.left(`Unknown benchmark "${input.benchmarkId}"`); } return Either.right({ - benchmark: mendelBenchmark, + benchmark: keplerBenchmark, benchmarkLayer: makeBenchmarkLayer( - mendelBenchmark, + keplerBenchmark, input, input.benchmarkConfig ),