diff --git a/.changeset/subagent-model-routing.md b/.changeset/subagent-model-routing.md new file mode 100644 index 000000000..0b4be525b --- /dev/null +++ b/.changeset/subagent-model-routing.md @@ -0,0 +1,6 @@ +--- +'@truefoundry/trueforge-core': patch +'@truefoundry/trueforge': patch +--- + +Add optional subagent model routing. In TrueFoundry mode, `TRUEFOUNDRY_MODEL_ROUTING` turns it on per tenant: before a subagent starts, a decision model rates each of the caller's scored models for the task, and the highest-rated one is picked. `shadow` logs the pick and keeps the parent's model; `on` runs the pick. Off by default. diff --git a/packages/trueforge-core/src/agent-session/ITurnResourceResolver.ts b/packages/trueforge-core/src/agent-session/ITurnResourceResolver.ts index 63d5d56c2..96330cabc 100644 --- a/packages/trueforge-core/src/agent-session/ITurnResourceResolver.ts +++ b/packages/trueforge-core/src/agent-session/ITurnResourceResolver.ts @@ -14,6 +14,26 @@ export interface ResolvedAgentDefinition { extraCapabilities?: AgentCapability[] | undefined; } +/** Input to {@link ITurnResourceResolver.routeModel}: a subagent that's about to start, and the task it was given. */ +export interface RouteModelInput { + threadId: string; + /** Model the subagent runs on when routing doesn't pick one (`provider/model`): the parent's model. */ + fallbackModel: string; + task: { + name?: string | undefined; + instructions: string; + /** MCP server names, plus `web_search` and `sandbox` when available. */ + tools: string[]; + }; + /** The subagent's abort signal; cancels routing when the turn is cancelled. */ + signal: AbortSignal; +} + +export interface RouteModelResult { + /** Model to run instead of the fallback. Undefined keeps the fallback. */ + model?: string | undefined; +} + /** * Per-run wiring contract consumed by SessionHandle.createTurn(). Implementations hold * per-request state (secrets, providers, caches) freely. @@ -63,6 +83,11 @@ export interface ITurnResourceResolver; + /** + * Chooses the model for a subagent before it starts. Omit to keep every subagent on its parent's model. + * Must not throw: on any failure, resolve without a model. + */ + routeModel?(input: RouteModelInput): Promise; /** * Release per-run resources. Called by the harness at the end of the run: * by TurnHandle.stream() in its finally (AFTER the terminal state write), or by diff --git a/packages/trueforge-core/src/agent-session/SessionHandle.ts b/packages/trueforge-core/src/agent-session/SessionHandle.ts index e709a5846..99a994c63 100644 --- a/packages/trueforge-core/src/agent-session/SessionHandle.ts +++ b/packages/trueforge-core/src/agent-session/SessionHandle.ts @@ -600,9 +600,29 @@ export class SessionHandle< tracing: AgentTracing; }): CreateDynamicSubAgentThread { return async params => { + let request = params.request; + if (request.model === undefined && input.resolver.routeModel !== undefined) { + const routed = await input.resolver.routeModel({ + threadId: params.threadId, + fallbackModel: params.parentDefinition.modelName, + task: { + name: request.name, + instructions: request.input, + tools: [ + ...(input.spec.mcp_servers ?? []).map(server => server.name), + ...(input.spec.config.web_search.enabled && input.resolver.webSearchProvider ? ['web_search'] : []), + ...(input.sandbox ? ['sandbox'] : []), + ], + }, + signal: params.signal, + }); + if (routed.model !== undefined) { + request = { ...request, model: routed.model }; + } + } const { definition, extraCapabilities } = await input.resolver.resolveAgentDefinition({ spec: input.spec, - agent_info: params.request, + agent_info: request, previousTurn: input.previous, signal: params.signal, tracing: input.tracing, @@ -615,7 +635,7 @@ export class SessionHandle< // The delegated task goes in as the initial user message; the sub-agent // system prompt is SUB_AGENT_IDENTITY (added by AgentThread.buildInstruction). instruction: undefined, - messages: [{ role: 'user', content: params.request.input }], + messages: [{ role: 'user', content: request.input }], // Sub-agents should return free-form summaries to the parent, not the user-facing structured response. responseFormat: undefined, }; @@ -634,10 +654,10 @@ export class SessionHandle< return new AgentThread({ definition: childDefinition, threadId: params.threadId, - title: params.request.name, + title: request.name, sandbox: input.sandbox, parent: params.parent, - agentInfo: params.request, + agentInfo: request, capabilities, tracing: input.tracing, logger: input.resolver.logger, diff --git a/packages/trueforge-core/src/agent-session/TurnResourceResolver.ts b/packages/trueforge-core/src/agent-session/TurnResourceResolver.ts index 25bc1e3b9..359fa2a61 100644 --- a/packages/trueforge-core/src/agent-session/TurnResourceResolver.ts +++ b/packages/trueforge-core/src/agent-session/TurnResourceResolver.ts @@ -14,7 +14,12 @@ import type { HarnessSandbox, SandboxInfo } from '../core/sandbox/Sandbox'; import type { AgentTracing } from '../core/tracing/AgentTracing'; import { NOOP_AGENT_TRACING } from '../core/tracing/NoopAgentTracing'; import type { IWebSearchProvider } from '../core/web-search/WebSearchProvider'; -import type { ITurnResourceResolver, ResolvedAgentDefinition } from './ITurnResourceResolver'; +import type { + ITurnResourceResolver, + ResolvedAgentDefinition, + RouteModelInput, + RouteModelResult, +} from './ITurnResourceResolver'; import type { TurnRecord } from './models/TurnRecord'; import type { AgentSpec } from './schemas/agentSpec'; @@ -80,6 +85,7 @@ export class TurnResourceResolver< readonly #models = new Map>(); #sandbox?: HarnessSandbox | undefined; readonly maxParallelSubAgents?: number; + readonly routeModel?: (input: RouteModelInput) => Promise; constructor( protected readonly deps: { @@ -110,6 +116,8 @@ export class TurnResourceResolver< * `resolveAgentDefinition` without calling `super` does not mount them. */ rootCapabilities?: (() => readonly AgentCapability[]) | undefined; + /** Chooses a subagent's model before it starts; omit to keep subagents on their parent's model. */ + routeModel?: ((input: RouteModelInput) => Promise) | undefined; /** Forwarded to RemoteMCP / AgentThread (required by their constructors). */ logger: Logger; }, @@ -117,6 +125,9 @@ export class TurnResourceResolver< if (deps.maxParallelSubAgents !== undefined) { this.maxParallelSubAgents = deps.maxParallelSubAgents; } + if (deps.routeModel !== undefined) { + this.routeModel = deps.routeModel; + } } get webSearchProvider(): IWebSearchProvider | undefined { diff --git a/packages/trueforge-core/src/agent-session/index.ts b/packages/trueforge-core/src/agent-session/index.ts index f420f70f0..4d9550964 100644 --- a/packages/trueforge-core/src/agent-session/index.ts +++ b/packages/trueforge-core/src/agent-session/index.ts @@ -133,7 +133,7 @@ export { TurnOwnershipLostError, } from './store/SessionStoreErrors'; -export type { ITurnResourceResolver } from './ITurnResourceResolver'; +export type { ITurnResourceResolver, RouteModelInput, RouteModelResult } from './ITurnResourceResolver'; export { TRUEFORGE_MCP_ID, TurnResourceResolver } from './TurnResourceResolver'; export type { TurnSandboxFactory } from './TurnResourceResolver'; diff --git a/packages/trueforge/.env.example b/packages/trueforge/.env.example index 87731cfdf..d38b8fad2 100644 --- a/packages/trueforge/.env.example +++ b/packages/trueforge/.env.example @@ -307,6 +307,11 @@ POSTGRES_PORT=5432 ## Tenants omitted are unfiltered. Example: {"internal":["openai-main","anthropic-main"]} # TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS={"internal":["openai-main"]} +## Optional subagent model routing (JSON object). A decision model picks each subagent's model from the +## caller's scored models. Per tenant: mode off (default) | shadow (log the pick only) | on, and the tenant's +## own decision model on its gateway (account/model). +# TRUEFOUNDRY_MODEL_ROUTING={"tenants":{"internal":{"subagent":"shadow","decision_model":"typesafe/jev-1.13.0"}}} + ## Internal mutual TLS for calls to the ServiceFoundry server. When true, trueforge presents its client ## certificate (and upgrades a mesh-direct http:// peer URL to https://). Only meaningful in-cluster, ## where TRUEFOUNDRY_SERVICEFOUNDRY_SERVER_URL points at the ServiceFoundry server Service directly. diff --git a/packages/trueforge/src/apis/turns.ts b/packages/trueforge/src/apis/turns.ts index 26c71d79d..f7a163415 100644 --- a/packages/trueforge/src/apis/turns.ts +++ b/packages/trueforge/src/apis/turns.ts @@ -75,7 +75,7 @@ import { type RequestContext, type ResolveRequestContext, } from '../auth/identity'; -import configuration from '../config'; +import configuration, { isTrueFoundryModeEnabled } from '../config'; import type { IAccessControlStore } from '../db/accessControlStore'; import type { AgentRecord, IAgentStore } from '../db/agentStore'; import { EmptyCapabilitiesStore, type ICapabilitiesStore } from '../db/capabilitiesStore'; @@ -91,6 +91,7 @@ import { currentLogContext, runWithLogContext, updateLogContext } from '../logge import { TRUEFORGE_SESSION_HEADER, X_TFG_TRUEFORGE_MCP } from '../mcp/server/trueforge/headers'; import type { TenantMemoryDeps } from '../memory/providers'; import { resolveSessionMemory } from '../memory/sessionMemory'; +import { createModelRouter } from '../modelRouting/router'; import { createAndExecuteTurnRoute, createInternalSessionTurnRoute, @@ -123,6 +124,7 @@ import { restoreTurnEventLog } from '../runtime/turnEventLogRestore'; import { MAX_SESSION_TITLE_LENGTH } from '../schemas/session'; import { assertGatewayMetadataRequestHeaders, X_TFY_METADATA } from '../truefoundry/gatewayMetadata'; import { X_TFG_MCP, X_TFG_MODELS, X_TFG_SKILLS, X_TFG_TOOLS } from '../truefoundry/inlineResources'; +import { createJevDecisionClient } from '../truefoundry/jevDecisionClient'; import { X_TFG_MCP_HEADERS } from '../truefoundry/perServerMcpHeaders'; import { newId } from '../utils/id'; import { resolveWebSearchProvider } from '../websearch/providers'; @@ -363,6 +365,44 @@ interface BeginTurnExecutionParams { deps: BeginTurnExecutionDeps; } +/** This turn's subagent model router, or undefined when routing is off for the tenant. */ +function subagentModelRouter(input: { + tenant_id: string; + modelProviderStore: IModelProviderStore; + turnMetadata: TurnMetadata; + logger: Logger; +}): ReturnType | undefined { + const { tenant_id, modelProviderStore, turnMetadata, logger } = input; + const routing = isTrueFoundryModeEnabled(configuration) ? configuration.TRUEFOUNDRY_MODEL_ROUTING : undefined; + const tenantRouting = routing?.tenants[tenant_id]; + if ( + routing === undefined || + tenantRouting === undefined || + tenantRouting.subagent === 'off' || + modelProviderStore.listRoutingCandidates === undefined + ) { + return undefined; + } + return createModelRouter({ + mode: tenantRouting.subagent, + listCandidates: () => modelProviderStore.listRoutingCandidates?.({ tenant_id }) ?? Promise.resolve([]), + decision: createJevDecisionClient({ + resolveProviderConfig: async () => + ( + await getModelDetails({ + tenant_id, + name: tenantRouting.decision_model, + store: modelProviderStore, + turnMetadata, + }) + ).providerConfig, + }), + timeoutMs: routing.timeout_ms, + maxCandidates: routing.max_candidates, + logger, + }); +} + /** * Builds the per-turn resolver. Agent / MCP / sandbox / LLM lookups are wired * the same way: async factories over the corresponding stores. @@ -404,8 +444,10 @@ function createTurnResolver(deps: { } = deps; const tenant_id = session.tenant_id; const sessionId = session.session_id; + const routeModel = subagentModelRouter({ tenant_id, modelProviderStore, turnMetadata, logger }); return new TurnResourceResolver({ + ...(routeModel === undefined ? {} : { routeModel }), llm: async name => { const resolved = await getModelDetails({ tenant_id, diff --git a/packages/trueforge/src/config.ts b/packages/trueforge/src/config.ts index 18d893a6b..91c9107ac 100644 --- a/packages/trueforge/src/config.ts +++ b/packages/trueforge/src/config.ts @@ -20,6 +20,7 @@ import { DEFAULT_MAX_PARALLEL_SUB_AGENTS } from '@truefoundry/trueforge-core/cor import envPaths from 'env-paths'; import { z } from 'zod'; +import { MODEL_ROUTING_MODES } from './modelRouting/types'; import { parseRedisSentinelNodes, type RedisConnection } from './runtime/redis'; export type { RedisConnection }; @@ -156,6 +157,33 @@ export function parseOidcAllowedEmails(raw: string | undefined): string[] { return parseCommaSeparatedEnvList(raw); } +const TrueFoundryModelRoutingSchema = z.object({ + timeout_ms: z.number().int().positive().default(2_000), + max_candidates: z.number().int().positive().default(60), + /** Per tenant: the decision model is the tenant's own gateway model (`account/model`). */ + tenants: z.record( + z.string(), + z.object({ subagent: z.enum(MODEL_ROUTING_MODES).default('off'), decision_model: z.string().min(1) }), + ), +}); + +export type TrueFoundryModelRouting = z.infer; + +/** Parses `TRUEFOUNDRY_MODEL_ROUTING` JSON. Empty / unset → undefined (routing off everywhere). */ +export function parseTrueFoundryModelRouting(raw: string | undefined): TrueFoundryModelRouting | undefined { + if (raw === undefined || raw.trim() === '') { + return undefined; + } + try { + return TrueFoundryModelRoutingSchema.parse(JSON.parse(raw)); + } catch (error) { + throw new Error( + 'Environment variable TRUEFOUNDRY_MODEL_ROUTING must be a JSON object with per-tenant "tenants" entries (e.g. {"tenants":{"":{"subagent":"shadow","decision_model":"typesafe/jev-1.13.0"}}})', + { cause: error }, + ); + } +} + /** * Parses `TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS` JSON. * Empty / unset → `{}` (no filtering). @@ -1102,6 +1130,12 @@ export type DistributedServerConfiguration = SharedServerConfiguration & { * Env: `TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS`. */ TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS: Record; + /** + * Optional subagent model routing (TrueFoundry mode only). JSON object with `timeout_ms`, + * `max_candidates`, and per-tenant entries: mode (`off` | `shadow` | `on`) and the tenant's decision + * model. Unset, or a tenant not listed → off. Env: `TRUEFOUNDRY_MODEL_ROUTING`. + */ + TRUEFOUNDRY_MODEL_ROUTING: TrueFoundryModelRouting | undefined; /** * Optional built-in web search provider (TrueFoundry mode only). JSON object * `Record` with `name` (`parallel`) and `api_key`. @@ -1474,6 +1508,7 @@ const configuration: ServerConfiguration = standalone TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS: parseTenantIdToAllowedModelProviderAccounts( getEnv('TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS', { required: false }), ), + TRUEFOUNDRY_MODEL_ROUTING: parseTrueFoundryModelRouting(getEnv('TRUEFOUNDRY_MODEL_ROUTING', { required: false })), TRUEFOUNDRY_WEB_SEARCH_PROVIDER: parseTrueFoundryWebSearchProvider( getEnv('TRUEFOUNDRY_WEB_SEARCH_PROVIDER', { required: false }), ), diff --git a/packages/trueforge/src/db/modelProviderStore.ts b/packages/trueforge/src/db/modelProviderStore.ts index 314f99319..0a6cea417 100644 --- a/packages/trueforge/src/db/modelProviderStore.ts +++ b/packages/trueforge/src/db/modelProviderStore.ts @@ -3,6 +3,7 @@ * `ModelProviderManifest` document. Implementations: Postgres, Sqlite, and * TrueFoundry (read-only ServiceFoundry server listing). */ +import type { RoutingCandidate } from '../modelRouting/types'; import type { ResourceName } from '../schemas/common'; import type { AvailableModel, ModelProperties, ModelProviderManifest } from '../schemas/modelProvider'; import type { TurnMetadata } from './turnMetadata'; @@ -85,6 +86,11 @@ export interface IModelProviderStore { record: ModelProviderRecord; turnMetadata?: TurnMetadata; }): Promise>; + /** + * The caller's models with the catalog fields model routing needs. Only the TrueFoundry-mode store + * implements it for now; stores that omit it never get routing. + */ + listRoutingCandidates?(input: ListModelProvidersInput): Promise; } export function modelPropertiesWarning(input: { properties: ModelProperties }): string | undefined { diff --git a/packages/trueforge/src/modelRouting/benchmarks.ts b/packages/trueforge/src/modelRouting/benchmarks.ts new file mode 100644 index 000000000..3d6300904 --- /dev/null +++ b/packages/trueforge/src/modelRouting/benchmarks.ts @@ -0,0 +1,55 @@ +// Descriptions are quoted verbatim, retrieved 2026-11-16; don't paraphrase them. Sources: +// https://artificialanalysis.ai/methodology/intelligence-benchmarking (all but HLE), https://lastexam.ai/ (HLE). + +/** + * The nine evaluations in the Artificial Analysis Intelligence Index v5, the score routing uses. + * Jev is asked which ones a task resembles; the answers are logged, never used to pick a model. + */ +export const RESEMBLANCE_BENCHMARKS: Readonly> = + { + briefcase: { + name: 'AA-Briefcase', + description: + 'AA-Briefcase is a new benchmark for testing models on realistic knowledge work tasks in complex projects built by industry experts. Models are evaluated on multi-week knowledge work projects, each with many linked tasks and thousands of input source files.', + }, + automationbench: { + name: 'AutomationBench', + description: + 'It tests whether models can complete realistic SaaS workflows that span multiple simulated business apps, using REST APIs as the tool interface.', + }, + gdp_pdf: { + name: 'GDP.pdf', + description: + 'A benchmark that tests whether models can reason over long, real-world professional documents and satisfy task-specific criteria.', + }, + frontiercode: { + name: 'FrontierCode', + description: + 'Each task asks an agent to make a change to a real open-source repository, and the benchmark measures whether the change is good enough for a maintainer to merge.', + }, + terminal_bench: { + name: 'Terminal-Bench', + description: + 'Covers software engineering, system administration, data processing, model training, and security, with each task graded by its own verification suite.', + }, + terminal_bench_science: { + name: 'Terminal-Bench-Science', + description: + 'Its research workflow tasks are drawn from real scientific practice, and experts curate and review each one. Each task has its own set of tests that the agent must pass by working in a terminal.', + }, + critpt: { + name: 'CritPt', + description: + 'Research-level physics reasoning benchmark with unpublished, frontier physics problems spanning a wide range of subfields.', + }, + omniscience: { + name: 'AA-Omniscience', + description: + 'AA-Omniscience is a knowledge and hallucination benchmark that measures factual reliability, rewards precise knowledge, and penalizes incorrect guesses or hallucinations.', + }, + hle: { + name: "Humanity's Last Exam", + description: + "Humanity's Last Exam, a multi-modal benchmark at the frontier of human knowledge, designed to be the final closed-ended academic benchmark of its kind with broad subject coverage.", + }, + }; diff --git a/packages/trueforge/src/modelRouting/modelRoutingScores.ts b/packages/trueforge/src/modelRouting/modelRoutingScores.ts new file mode 100644 index 000000000..3501fd2e8 --- /dev/null +++ b/packages/trueforge/src/modelRouting/modelRoutingScores.ts @@ -0,0 +1,137 @@ +// Generated from the Artificial Analysis free API; regenerate rather than hand-editing. + +/** Intelligence Index v5 per Artificial Analysis slug. Effort variants are separate slugs (`-high`). */ +export const MODEL_ROUTING_SCORES: { + readonly retrievedAt: string; + readonly scores: Readonly>; +} = { + retrievedAt: '2026-11-13', + scores: { + 'agnes-2-5-pro-theta': { name: 'Agnes 2.5 Pro Theta', score: 24 }, + 'apodex-1-1': { name: 'Apodex 1.1', score: 13.7 }, + 'claude-4-5-haiku-reasoning': { name: 'Claude 4.5 Haiku', score: 7.4 }, + 'claude-fable-5-1': { name: 'Claude Fable 5.1', score: 49.2 }, + 'claude-fable-5-1-high': { name: 'Claude Fable 5.1', score: 47.2 }, + 'claude-fable-5-1-low': { name: 'Claude Fable 5.1', score: 38.7 }, + 'claude-fable-5-1-medium': { name: 'Claude Fable 5.1', score: 43.2 }, + 'claude-fable-5-1-xhigh': { name: 'Claude Fable 5.1', score: 50 }, + 'claude-fable-5-2': { name: 'Claude Fable 5.2', score: 64.6 }, + 'claude-fable-5-2-high': { name: 'Claude Fable 5.2', score: 61.8 }, + 'claude-fable-5-2-low': { name: 'Claude Fable 5.2', score: 53.9 }, + 'claude-fable-5-2-medium': { name: 'Claude Fable 5.2', score: 56.4 }, + 'claude-fable-5-2-xhigh': { name: 'Claude Fable 5.2', score: 62.2 }, + 'claude-haiku-5-5': { name: 'Claude Haiku 5.5', score: 36.5 }, + 'claude-haiku-5-5-high': { name: 'Claude Haiku 5.5', score: 29.9 }, + 'claude-haiku-5-5-low': { name: 'Claude Haiku 5.5', score: 18.6 }, + 'claude-haiku-5-5-medium': { name: 'Claude Haiku 5.5', score: 24.6 }, + 'claude-haiku-5-5-xhigh': { name: 'Claude Haiku 5.5', score: 34.6 }, + 'claude-opus-5': { name: 'Claude Opus 5', score: 44.2 }, + 'claude-opus-5-5': { name: 'Claude Opus 5.5', score: 59.1 }, + 'claude-opus-5-5-high': { name: 'Claude Opus 5.5', score: 51.8 }, + 'claude-opus-5-5-low': { name: 'Claude Opus 5.5', score: 34.8 }, + 'claude-opus-5-5-medium': { name: 'Claude Opus 5.5', score: 48.2 }, + 'claude-opus-5-5-xhigh': { name: 'Claude Opus 5.5', score: 57.4 }, + 'claude-sonnet-5-5': { name: 'Claude Sonnet 5.5', score: 54.5 }, + 'claude-sonnet-5-5-high': { name: 'Claude Sonnet 5.5', score: 41.5 }, + 'claude-sonnet-5-5-low': { name: 'Claude Sonnet 5.5', score: 25.6 }, + 'claude-sonnet-5-5-medium': { name: 'Claude Sonnet 5.5', score: 30.8 }, + 'claude-sonnet-5-5-xhigh': { name: 'Claude Sonnet 5.5', score: 49.9 }, + 'command-a-plus': { name: 'Command A+', score: 6.2 }, + 'deepseek-v4-1-flash-1017': { name: 'DeepSeek V4.1 Flash 1017', score: 32.6 }, + 'deepseek-v4-1-flash-1017-high': { name: 'DeepSeek V4.1 Flash 1017', score: 28.6 }, + 'deepseek-v4-1-flash-1017-non-reasoning': { name: 'DeepSeek V4.1 Flash 1017', score: 18.5 }, + 'deepseek-v4-pro': { name: 'DeepSeek V4 Pro 0813', score: 23.1 }, + 'deepseek-v4-pro-non-reasoning': { name: 'DeepSeek V4 Pro 0813', score: 12.6 }, + 'gemini-3-1-flash-lite-preview': { name: 'Gemini 3.1 Flash-Lite', score: 6.2 }, + 'gemini-3-5-flash-lite': { name: 'Gemini 3.5 Flash-Lite', score: 11.7 }, + 'gemini-3-8-flash': { name: 'Gemini 3.8 Flash', score: 30.8 }, + 'gemini-3-8-flash-low': { name: 'Gemini 3.8 Flash', score: 22.5 }, + 'gemini-3-8-flash-medium': { name: 'Gemini 3.8 Flash', score: 30.1 }, + 'gemini-4-argon': { name: 'Gemini 4 Argon', score: 50.4 }, + 'gemini-4-carbon': { name: 'Gemini 4 Carbon', score: 60 }, + 'gemini-4-carbon-low': { name: 'Gemini 4 Carbon', score: 54.2 }, + 'gemini-4-carbon-medium': { name: 'Gemini 4 Carbon', score: 58.3 }, + 'gemini-4-hydrogen': { name: 'Gemini 4 Hydrogen', score: 53.9 }, + 'gemini-4-hydrogen-high': { name: 'Gemini 4 Hydrogen', score: 50.8 }, + 'gemini-4-hydrogen-low': { name: 'Gemini 4 Hydrogen', score: 26.9 }, + 'gemini-4-hydrogen-medium': { name: 'Gemini 4 Hydrogen', score: 41.2 }, + 'gemma-4-31b': { name: 'Gemma 4 31B', score: 4.4 }, + 'glm-5-3': { name: 'GLM-5.3', score: 31.7 }, + 'glm-5-3-flash': { name: 'GLM-5.3-Flash', score: 27.8 }, + 'glm-5-3-low': { name: 'GLM-5.3', score: 23.1 }, + 'gpt-5-6-luna': { name: 'GPT-5.6 Luna', score: 23.8 }, + 'gpt-5-6-terra': { name: 'GPT-5.6 Terra', score: 32.3 }, + 'gpt-6-1-sol': { name: 'GPT-6.1 Sol', score: 54.6 }, + 'gpt-6-1-sol-high': { name: 'GPT-6.1 Sol', score: 50.3 }, + 'gpt-6-1-sol-low': { name: 'GPT-6.1 Sol', score: 34.8 }, + 'gpt-6-1-sol-medium': { name: 'GPT-6.1 Sol', score: 46.3 }, + 'gpt-6-1-sol-xhigh': { name: 'GPT-6.1 Sol', score: 52.5 }, + 'gpt-6-1-terra': { name: 'GPT-6.1 Terra', score: 51.3 }, + 'gpt-6-1-terra-high': { name: 'GPT-6.1 Terra', score: 42.4 }, + 'gpt-6-1-terra-low': { name: 'GPT-6.1 Terra', score: 29.4 }, + 'gpt-6-1-terra-medium': { name: 'GPT-6.1 Terra', score: 38.2 }, + 'gpt-6-1-terra-non-reasoning': { name: 'GPT-6.1 Terra', score: 30.5 }, + 'gpt-6-1-terra-xhigh': { name: 'GPT-6.1 Terra', score: 45.3 }, + 'gpt-6-astra': { name: 'GPT-6 Astra', score: 56.9 }, + 'gpt-6-astra-high': { name: 'GPT-6 Astra', score: 54.8 }, + 'gpt-6-astra-low': { name: 'GPT-6 Astra', score: 43.3 }, + 'gpt-6-astra-medium': { name: 'GPT-6 Astra', score: 51.6 }, + 'gpt-6-astra-xhigh': { name: 'GPT-6 Astra', score: 55.3 }, + 'gpt-6-luna': { name: 'GPT-6 Luna', score: 27.9 }, + 'gpt-6-luna-high': { name: 'GPT-6 Luna', score: 21.4 }, + 'gpt-6-luna-low': { name: 'GPT-6 Luna', score: 9.7 }, + 'gpt-6-luna-medium': { name: 'GPT-6 Luna', score: 18.5 }, + 'gpt-6-luna-non-reasoning': { name: 'GPT-6 Luna', score: 9.4 }, + 'gpt-6-luna-xhigh': { name: 'GPT-6 Luna', score: 23.5 }, + 'gpt-6-sol': { name: 'GPT-6 Sol', score: 44.2 }, + 'gpt-oss-120b': { name: 'gpt-oss-120b', score: 3.8 }, + 'granite-4-2-3b': { name: 'Granite 4.2 3B', score: 4.7 }, + 'granite-4-2-8b': { name: 'Granite 4.2 8B', score: 5.1 }, + 'grok-4-6-xhigh': { name: 'Grok 4.6', score: 31 }, + 'grok-4-7': { name: 'Grok 4.7', score: 35.5 }, + 'grok-4-7-high': { name: 'Grok 4.7', score: 35.2 }, + 'grok-4-7-low': { name: 'Grok 4.7', score: 31.1 }, + inkling: { name: 'Inkling', score: 12.5 }, + 'inkling-2-small': { name: 'Inkling 2 Small', score: 30.6 }, + 'inkling-small': { name: 'Inkling Small', score: 12.9 }, + 'jt-4-1-flash-236b-a21b': { name: 'JT-4.1 Flash 236B A21B', score: 19 }, + 'k-exaone-2-0': { name: 'K-EXAONE 2.0', score: 7.6 }, + 'k2-horizon-375b-a23b': { name: 'K2 Horizon 375B A23B', score: 17.8 }, + 'kimi-k3-1-pro': { name: 'Kimi K3.1 Pro', score: 46.5 }, + 'kimi-k3-low': { name: 'Kimi K3', score: 17.4 }, + 'ling-3-1-flash-1019': { name: 'Ling 3.1 Flash 1019', score: 29.6 }, + 'mimo-v2-6-flash': { name: 'MiMo-V2.6-Flash', score: 26.7 }, + 'mimo-v2-6-pro': { name: 'MiMo-V2.6-Pro', score: 29.9 }, + 'minimax-m3': { name: 'MiniMax-M3', score: 18.9 }, + 'minimax-m3-1-flash': { name: 'MiniMax-M3.1-Flash', score: 29.1 }, + 'mistral-large-4': { name: 'Mistral Large 4', score: 26.1 }, + 'mistral-large-4-medium': { name: 'Mistral Large 4', score: 23.1 }, + 'mistral-large-4-preview': { name: 'Mistral Large 4 Preview', score: 25.3 }, + 'mistral-medium-3-5': { name: 'Mistral Medium 3.5', score: 4.6 }, + 'mistral-small-4': { name: 'Mistral Small 4', score: 4.3 }, + 'muse-ember': { name: 'Muse Ember', score: 36 }, + 'muse-ember-xhigh': { name: 'Muse Ember', score: 34.1 }, + 'muse-glimmer': { name: 'Muse Glimmer', score: 6.3 }, + 'muse-spark-1-3': { name: 'Muse Spark 1.3', score: 36.6 }, + 'muse-spark-open': { name: 'Muse Spark Open', score: 24.9 }, + 'muse-wonder': { name: 'Muse Wonder', score: 43.6 }, + 'muse-wonder-full': { name: 'Muse Wonder Full', score: 50.1 }, + 'muse-wonder-high': { name: 'Muse Wonder', score: 40.5 }, + 'nanbeige4-3-33b': { name: 'Nanbeige4.3-33B', score: 20.4 }, + 'nemotron-3-5-lightning': { name: 'Nemotron 3.5 Lightning', score: 5.5 }, + 'nemotron-3-5-super-vl': { name: 'Nemotron 3.5 Super VL', score: 13.6 }, + 'north-mini-code': { name: 'North Mini Code', score: 3.4 }, + 'nvidia-nemotron-3-nano-4b': { name: 'NVIDIA Nemotron 3 Nano 4B', score: 1.8 }, + 'nvidia-nemotron-3-ultra-550b-a55b': { name: 'Nemotron 3 Ultra 550B A55B', score: 11.8 }, + 'qwen3-8-27b': { name: 'Qwen3.8 27B', score: 20.1 }, + 'qwen3-8-27b-low': { name: 'Qwen3.8 27B', score: 15.7 }, + 'qwen3-8-27b-medium': { name: 'Qwen3.8 27B', score: 17.5 }, + 'qwen3-8-27b-non-reasoning': { name: 'Qwen3.8 27B', score: 10 }, + 'qwen3-8-flash-next': { name: 'Qwen3.8-Flash-Next', score: 28.6 }, + 'qwen3-8-max': { name: 'Qwen3.8 Max', score: 33.7 }, + 'reflection-beam': { name: 'Beam 1.0', score: 17.7 }, + 'solar-pro4': { name: 'Solar Pro 4', score: 14.2 }, + 'step-5': { name: 'Step 5 Preview', score: 28.5 }, + 'tencent-hy4': { name: 'Tencent Hy4', score: 34.6 }, + }, +}; diff --git a/packages/trueforge/src/modelRouting/pick.ts b/packages/trueforge/src/modelRouting/pick.ts new file mode 100644 index 000000000..8825f2748 --- /dev/null +++ b/packages/trueforge/src/modelRouting/pick.ts @@ -0,0 +1,16 @@ +export interface ScoredAnswer { + candidate: T; + pYes: number; + price?: number | undefined; +} + +/** Higher yes-probability first; among equal probabilities, cheaper first, and an unpriced model last. */ +function byPYesThenPrice(a: ScoredAnswer, b: ScoredAnswer): number { + const price = (answer: ScoredAnswer) => answer.price ?? Number.MAX_VALUE; + return b.pYes - a.pYes || price(a) - price(b); +} + +/** The answer with the highest yes-probability; ties go to the cheapest model. Undefined when there are no answers. */ +export function pickCandidate(answers: readonly ScoredAnswer[]): ScoredAnswer | undefined { + return [...answers].sort(byPYesThenPrice)[0]; +} diff --git a/packages/trueforge/src/modelRouting/router.ts b/packages/trueforge/src/modelRouting/router.ts new file mode 100644 index 000000000..aa613f3bb --- /dev/null +++ b/packages/trueforge/src/modelRouting/router.ts @@ -0,0 +1,200 @@ +import type { RouteModelInput, RouteModelResult } from '@truefoundry/trueforge-core/agent-session'; +import { describeUnknownError } from '@truefoundry/trueforge-core/core'; +import type { Logger } from 'winston'; +import { RESEMBLANCE_BENCHMARKS } from './benchmarks'; +import { pickCandidate } from './pick'; +import { lookupScore } from './scores'; +import type { DecisionClient, DecisionQuestion, RoutingCandidate } from './types'; + +const TASK_CHARS = 8_000; + +const SCORE_NOTE = + 'Artificial Analysis Intelligence Index (0-100, higher is better) averages agentic knowledge work, coding, science and general reasoning evaluations.'; + +const CAN_COMPLETE: DecisionQuestion['criteria'] = { + true: 'It would produce a result that can be used as-is: complete, accurate, and with the needed tool use done.', + false: 'It would likely miss parts, make errors, or stop before the task is done.', +}; + +const RESEMBLES: DecisionQuestion['criteria'] = { + true: 'Same kind of work as the benchmark.', + false: 'A different kind of work.', +}; + +interface ScoredCandidate extends RoutingCandidate { + name: string; + score: number; +} + +type FallbackReason = 'no_candidates' | 'decision_error' | 'decision_timeout'; + +function clip(text: string): string { + if (text.length <= TASK_CHARS) { + return text; + } + const half = TASK_CHARS / 2; + return `${text.slice(0, half)}\n…\n${text.slice(-half)}`; +} + +function accountOf(model: string): string { + return model.split('/', 1)[0] ?? model; +} + +/** One candidate per scored slug, preferring the fallback model's account, capped by score. */ +function scoredCandidates(input: { + models: readonly RoutingCandidate[]; + fallbackModel: string; + maxCandidates: number; +}): ScoredCandidate[] { + const fallbackAccount = accountOf(input.fallbackModel); + const bySlug = new Map(); + const ordered = [...input.models].sort((a, b) => { + const preferred = Number(accountOf(b.model) === fallbackAccount) - Number(accountOf(a.model) === fallbackAccount); + return preferred !== 0 ? preferred : a.model.localeCompare(b.model); + }); + for (const model of ordered) { + if (model.slug === undefined || bySlug.has(model.slug)) { + continue; + } + const scored = lookupScore({ slug: model.slug, effort: model.defaultEffort }); + if (scored !== undefined) { + bySlug.set(model.slug, { ...model, ...scored }); + } + } + return [...bySlug.values()].sort((a, b) => b.score - a.score).slice(0, input.maxCandidates); +} + +function questions(candidates: readonly ScoredCandidate[]): Record { + const out: Record = {}; + candidates.forEach((candidate, index) => { + const effort = candidate.defaultEffort ?? 'its default'; + out[`can_${String(index)}`] = { + instructions: `Can ${candidate.name} at ${effort} effort (Intelligence Index ${String(candidate.score)}) complete this task fully and correctly?`, + criteria: CAN_COMPLETE, + }; + }); + for (const [key, benchmark] of Object.entries(RESEMBLANCE_BENCHMARKS)) { + out[`resembles_${key}`] = { + instructions: `Does this task resemble the benchmark ${benchmark.name}? ${benchmark.description}`, + criteria: RESEMBLES, + }; + } + return out; +} + +function priceDirection(input: { picked?: number | undefined; fallback?: number | undefined }) { + if (input.picked === undefined || input.fallback === undefined) { + return undefined; + } + return input.picked > input.fallback ? 'up' : input.picked < input.fallback ? 'down' : 'same'; +} + +/** + * `routeModel` for one turn. Candidates are listed once and shared by the turn's subagents. + * Shadow mode logs the pick and keeps the fallback. Never throws. + */ +export function createModelRouter(input: { + mode: 'shadow' | 'on'; + /** The caller's models; called once per turn. */ + listCandidates: () => Promise; + decision: DecisionClient; + timeoutMs: number; + maxCandidates: number; + logger: Logger; +}): (route: RouteModelInput) => Promise { + let models: Promise | undefined; + + return async route => { + const startedMs = Date.now(); + const log = (fields: Record) => { + input.logger.info('model routing', { + thread_id: route.threadId, + mode: input.mode, + fallback_model: route.fallbackModel, + latency_ms: Date.now() - startedMs, + ...fields, + }); + }; + const fallback = (reason: FallbackReason, extra: Record = {}): RouteModelResult => { + log({ reason, chosen_model: route.fallbackModel, ...extra }); + return {}; + }; + + const decide = async (timeout: AbortSignal): Promise => { + models ??= input.listCandidates().catch((error: unknown) => { + models = undefined; + throw error; + }); + const all = await models; + timeout.throwIfAborted(); + const candidates = scoredCandidates({ + models: all, + fallbackModel: route.fallbackModel, + maxCandidates: input.maxCandidates, + }); + if (candidates.length === 0) { + return fallback('no_candidates'); + } + const answers = await input.decision.ask({ + threadId: route.threadId, + state: { + name: route.task.name, + task: clip(route.task.instructions), + tools: route.task.tools, + benchmark_note: SCORE_NOTE, + }, + questions: questions(candidates), + signal: AbortSignal.any([route.signal, timeout]), + }); + timeout.throwIfAborted(); + const picked = pickCandidate( + candidates.flatMap((candidate, index) => { + const pYes = answers[`can_${String(index)}`]; + return pYes === undefined ? [] : [{ candidate, pYes, price: candidate.price }]; + }), + ); + if (picked === undefined) { + return fallback('decision_error', { candidate_count: candidates.length }); + } + const fallbackPrice = all.find(model => model.model === route.fallbackModel)?.price; + const chosen = input.mode === 'on' ? picked.candidate.model : route.fallbackModel; + log({ + reason: 'picked', + picked_model: picked.candidate.model, + chosen_model: chosen, + picked_p_yes: picked.pYes, + picked_price: picked.candidate.price, + fallback_price: fallbackPrice, + price_direction: priceDirection({ picked: picked.candidate.price, fallback: fallbackPrice }), + picked_image_input: picked.candidate.imageInput, + candidate_count: candidates.length, + // A list, not an object: the JSON log format sorts object keys, which would lose the ranking. + candidates: candidates + .map((candidate, index) => ({ model: candidate.model, p_yes: answers[`can_${String(index)}`] })) + .sort((a, b) => (b.p_yes ?? -1) - (a.p_yes ?? -1)), + }); + return input.mode === 'on' ? { model: picked.candidate.model } : {}; + }; + + // The budget covers the whole step, including candidate listing and gateway lookups, not just the Jev call. + const timeout = AbortSignal.timeout(input.timeoutMs); + const timedOut = new Promise((_, reject) => { + timeout.addEventListener( + 'abort', + () => { + reject(new Error(`Routing timed out after ${String(input.timeoutMs)} ms`, { cause: timeout.reason })); + }, + { once: true }, + ); + }); + const work = decide(timeout); + // Whichever promise loses the race settles later; its outcome is already reported. + work.catch(() => undefined); + timedOut.catch(() => undefined); + try { + return await Promise.race([work, timedOut]); + } catch (error) { + return fallback(timeout.aborted ? 'decision_timeout' : 'decision_error', { err: describeUnknownError(error) }); + } + }; +} diff --git a/packages/trueforge/src/modelRouting/scores.ts b/packages/trueforge/src/modelRouting/scores.ts new file mode 100644 index 000000000..afaa32675 --- /dev/null +++ b/packages/trueforge/src/modelRouting/scores.ts @@ -0,0 +1,58 @@ +import { SUPPORTED_REASONING_EFFORTS } from '@truefoundry/trueforge-core/core'; +import { MODEL_ROUTING_SCORES } from './modelRoutingScores'; + +type Effort = (typeof SUPPORTED_REASONING_EFFORTS)[number]; + +type ScoreTable = Readonly>; + +const EFFORTS: readonly string[] = SUPPORTED_REASONING_EFFORTS; + +function isEffort(value: string | undefined): value is Effort { + return value !== undefined && EFFORTS.includes(value); +} + +/** Artificial Analysis publishes each effort as its own slug: `-high`, and `-non-reasoning` for `none`. */ +function variantSlug({ slug, effort }: { slug: string; effort: Effort }): string { + return effort === 'none' ? `${slug}-non-reasoning` : `${slug}-${effort}`; +} + +/** Efforts ordered by closeness to `target`; of two equally close, the higher effort comes first. */ +function effortsNearest(target: Effort): Effort[] { + const rank = (effort: Effort) => SUPPORTED_REASONING_EFFORTS.indexOf(effort); + const distance = (effort: Effort) => Math.abs(rank(effort) - rank(target)); + return [...SUPPORTED_REASONING_EFFORTS].sort((a, b) => distance(a) - distance(b) || rank(b) - rank(a)); +} + +/** + * The Intelligence Index for a model running at `effort`. Tries, in order: the exact effort variant + * (`-high`); the model's unlabelled score, which Artificial Analysis publishes at max effort; and the + * closest effort that has a score (from medium when `effort` is unknown). + */ +export function lookupScore({ + slug, + effort, + scores = MODEL_ROUTING_SCORES.scores, +}: { + slug: string; + effort: string | undefined; + scores?: ScoreTable; +}): { name: string; score: number } | undefined { + const target = isEffort(effort) ? effort : undefined; + if (target !== undefined) { + const exact = scores[variantSlug({ slug, effort: target })]; + if (exact !== undefined) { + return exact; + } + } + const unlabelled = scores[slug]; + if (unlabelled !== undefined) { + return unlabelled; + } + for (const nearby of effortsNearest(target ?? 'medium')) { + const entry = scores[variantSlug({ slug, effort: nearby })]; + if (entry !== undefined) { + return entry; + } + } + return undefined; +} diff --git a/packages/trueforge/src/modelRouting/types.ts b/packages/trueforge/src/modelRouting/types.ts new file mode 100644 index 000000000..f2853abe4 --- /dev/null +++ b/packages/trueforge/src/modelRouting/types.ts @@ -0,0 +1,32 @@ +export const MODEL_ROUTING_MODES = ['off', 'shadow', 'on'] as const; + +export type ModelRoutingMode = (typeof MODEL_ROUTING_MODES)[number]; + +/** One of the caller's models. Only models whose slug has a score become routing candidates. */ +export interface RoutingCandidate { + /** `provider/model`, as the harness resolves it. */ + model: string; + /** Artificial Analysis slug; the score lookup key. */ + slug?: string | undefined; + /** Reasoning effort the model runs at when none is set. */ + defaultEffort?: string | undefined; + /** Blended USD per million tokens: 0.8 × input + 0.2 × output. */ + price?: number | undefined; + /** Logged with the pick only; never used to choose a model. */ + imageInput?: boolean | undefined; +} + +export interface DecisionQuestion { + instructions: string; + criteria: { true: string; false: string }; +} + +export interface DecisionClient { + /** Yes-probability per question key; keys the decision model didn't answer are omitted. */ + ask(input: { + threadId: string; + state: Record; + questions: Record; + signal: AbortSignal; + }): Promise>; +} diff --git a/packages/trueforge/src/truefoundry/InlineModelProviderStore.ts b/packages/trueforge/src/truefoundry/InlineModelProviderStore.ts index f3d1181ac..ff6c5877d 100644 --- a/packages/trueforge/src/truefoundry/InlineModelProviderStore.ts +++ b/packages/trueforge/src/truefoundry/InlineModelProviderStore.ts @@ -9,6 +9,7 @@ import type { UpsertModelProviderInput, } from '../db/modelProviderStore'; import type { TurnMetadata } from '../db/turnMetadata'; +import type { RoutingCandidate } from '../modelRouting/types'; import type { AvailableModel } from '../schemas/modelProvider'; import { gatewayMetadataHeadersForTurn } from './gatewayMetadata'; import type { InlineModelProviders } from './inlineResources'; @@ -64,6 +65,11 @@ export class InlineModelProviderStore implements IModelPro return this.#inner.listModels(input, transaction); } + /** Request-scoped providers carry no catalog metadata, so only the inner store's models are routable. */ + listRoutingCandidates(input: ListModelProvidersInput): Promise { + return this.#inner.listRoutingCandidates?.(input) ?? Promise.resolve([]); + } + /** * The manifest carries its own `api_key`, which the adapter sends as the bearer, so no caller * token is resolved here. Mid-turn, still stamp `x-tfy-metadata` the same way registry diff --git a/packages/trueforge/src/truefoundry/TrueFoundryModelProviderStore.ts b/packages/trueforge/src/truefoundry/TrueFoundryModelProviderStore.ts index a7c18abe5..e274e87e2 100644 --- a/packages/trueforge/src/truefoundry/TrueFoundryModelProviderStore.ts +++ b/packages/trueforge/src/truefoundry/TrueFoundryModelProviderStore.ts @@ -13,6 +13,7 @@ import { type UpsertModelProviderInput, } from '../db/modelProviderStore'; import type { TurnMetadata } from '../db/turnMetadata'; +import type { RoutingCandidate } from '../modelRouting/types'; import type { AvailableModel, ModelProviderManifest } from '../schemas/modelProvider'; import { accessTokenForRequest, @@ -101,6 +102,16 @@ export class TrueFoundryModelProviderStore implements IMod return flattenProviderModels(await this.listProviders(input, transaction)); } + async listRoutingCandidates(input: ListModelProvidersInput): Promise { + const integrations = await this.#client.listProviderIntegrations({ + accessToken: await this.#resolveServiceFoundryAuthorization(), + }); + return filterEnvModels({ tenant_id: input.tenant_id, models: mapEnabledModels({ integrations }) }).map(model => ({ + model: `${model.accountName}/${model.modelName}`, + ...model.routing, + })); + } + async resolveInvokeHeaders(input: { record: ModelProviderRecord; turnMetadata?: TurnMetadata; @@ -129,9 +140,13 @@ export class TrueFoundryModelProviderStore implements IMod this.#client.listGatewayInstallations(serviceFoundryAccessToken), ]); const gatewayUrl = resolveDefaultGatewayUrl(installations); + // A by-name lookup also resolves decision models (the model router's Jev); listings stay chat-only. const models = filterEnvModels({ tenant_id: input.tenant_id, - models: mapEnabledModels({ integrations }), + models: mapEnabledModels({ + integrations, + ...(input.filter !== undefined ? { modelTypes: ['chat', 'decision'] } : {}), + }), }); return toRecords({ tenant_id: input.tenant_id, diff --git a/packages/trueforge/src/truefoundry/jevDecisionClient.ts b/packages/trueforge/src/truefoundry/jevDecisionClient.ts new file mode 100644 index 000000000..a3417fc6e --- /dev/null +++ b/packages/trueforge/src/truefoundry/jevDecisionClient.ts @@ -0,0 +1,57 @@ +import { modelSsrfFetch, type VercelAIProviderConfig } from '@truefoundry/trueforge-core/core'; +import { z } from 'zod'; +import type { DecisionClient } from '../modelRouting/types'; +import { gatewayMetadataHeaders, headerValue, parseGatewayMetadataHeader, X_TFY_METADATA } from './gatewayMetadata'; + +const SystemOneResponseSchema = z.object({ + answers: z.record(z.string(), z.object({ noul: z.number().nullish() })), +}); + +/** + * Yes/no questions to a TrueFoundry decision model (Jev) through the gateway's `/v1/systemone`. + * `resolveProviderConfig` is the decision model's provider config, resolved on first use. + */ +export function createJevDecisionClient(input: { + resolveProviderConfig: () => Promise; +}): DecisionClient { + return { + async ask({ threadId, state, questions, signal }) { + const config = await input.resolveProviderConfig(); + if (config.provider.type !== 'truefoundry' || config.baseUrl === undefined) { + throw new Error(`Decision model "${config.name}" is not served by the TrueFoundry gateway`); + } + const turnMetadataHeader = headerValue(config.headers, X_TFY_METADATA); + const headers = { + ...Object.fromEntries(Object.entries(config.headers).filter(([key]) => key.toLowerCase() !== X_TFY_METADATA)), + ...gatewayMetadataHeaders({ + ...(turnMetadataHeader === undefined ? {} : parseGatewayMetadataHeader(turnMetadataHeader)), + 'tfg.thread_id': threadId, + }), + Authorization: `Bearer ${config.apiKey}`, + 'content-type': 'application/json', + }; + const root = config.baseUrl.replace(/\/+$/, '').replace(/\/v1$/, ''); + const response = await modelSsrfFetch(`${root}/v1/systemone`, { + method: 'POST', + headers, + body: JSON.stringify({ + model: config.model.id, + state, + questions: Object.fromEntries( + Object.entries(questions).map(([key, question]) => [key, { type: 'noul', ...question }]), + ), + }), + signal, + }); + if (!response.ok) { + throw new Error(`Decision model "${config.name}" returned HTTP ${String(response.status)}`); + } + const { answers } = SystemOneResponseSchema.parse(await response.json()); + return Object.fromEntries( + Object.entries(answers).flatMap(([key, answer]) => + typeof answer.noul === 'number' ? [[key, answer.noul]] : [], + ), + ); + }, + }; +} diff --git a/packages/trueforge/src/truefoundry/mapEnabledModels.ts b/packages/trueforge/src/truefoundry/mapEnabledModels.ts index 32e0cfa2e..eeb9c0167 100644 --- a/packages/trueforge/src/truefoundry/mapEnabledModels.ts +++ b/packages/trueforge/src/truefoundry/mapEnabledModels.ts @@ -2,13 +2,21 @@ import { SUPPORTED_REASONING_EFFORTS } from '@truefoundry/trueforge-core/core'; import { HTTPException } from 'hono/http-exception'; import { z } from 'zod'; import configuration, { isTrueFoundryModeEnabled } from '../config'; +import type { RoutingCandidate } from '../modelRouting/types'; import type { ModelProperties, ReasoningEffort } from '../schemas/modelProvider'; const ModelParamSchema = z.object({ key: z.string(), supportedValues: z.array(z.string()).optional(), + defaultValue: z.unknown().optional(), }); +const TokenCostSchema = z.object({ + input_cost_per_token: z.number().nullish(), + output_cost_per_token: z.number().nullish(), +}); + +// Routing-only fields fall back to undefined on an unexpected shape, so they can never break listing. const MetadataSchema = z.object({ limits: z .object({ @@ -19,6 +27,16 @@ const MetadataSchema = z.object({ }) .optional(), params: z.array(ModelParamSchema).optional(), + external_ids: z.object({ artificial_analysis_slug: z.string().nullish() }).optional().catch(undefined), + public_cost: TokenCostSchema.optional().catch(undefined), + costs: z + .array(TokenCostSchema.extend({ region: z.string().nullish() })) + .optional() + .catch(undefined), + modalities: z + .object({ input: z.array(z.string()).optional() }) + .optional() + .catch(undefined), }); const IntegrationSchema = z.object({ @@ -61,23 +79,57 @@ function toProperties(metadata: IntegrationMetadata | undefined): ModelPropertie }; } +/** + * Blended USD per million tokens, weighted 0.8 × input + 0.2 × output since agent loops read far more than + * they write. Uses the public price if set, else the global-region price, else the first region with a price. + */ +function toBlendedPrice(metadata: IntegrationMetadata | undefined): number | undefined { + const regionCosts = metadata?.costs ?? []; + const globalCosts = regionCosts.filter(cost => cost.region === '*'); + const inPreferenceOrder = [metadata?.public_cost, ...globalCosts, ...regionCosts]; + for (const cost of inPreferenceOrder) { + if (typeof cost?.input_cost_per_token === 'number' && typeof cost.output_cost_per_token === 'number') { + return (0.8 * cost.input_cost_per_token + 0.2 * cost.output_cost_per_token) * 1e6; + } + } + return undefined; +} + +function toRoutingMetadata(metadata: IntegrationMetadata | undefined): TrueFoundryEnabledModel['routing'] { + const defaultEffort = metadata?.params?.find(param => param.key === 'reasoning_effort')?.defaultValue; + return { + slug: metadata?.external_ids?.artificial_analysis_slug ?? undefined, + defaultEffort: typeof defaultEffort === 'string' ? defaultEffort : undefined, + price: toBlendedPrice(metadata), + imageInput: metadata?.modalities?.input?.includes('image'), + }; +} + export interface TrueFoundryEnabledModel { accountName: string; modelName: string; properties: ModelProperties; + /** Catalog fields the model router reads; never part of the public model listing. */ + routing: Omit; } -export function mapEnabledModels(input: { integrations: readonly unknown[] }): TrueFoundryEnabledModel[] { +/** `modelTypes` defaults to chat only, so listings never show models agents can't run on. */ +export function mapEnabledModels(input: { + integrations: readonly unknown[]; + modelTypes?: readonly string[]; +}): TrueFoundryEnabledModel[] { + const modelTypes = input.modelTypes ?? ['chat']; const models: TrueFoundryEnabledModel[] = []; for (const row of input.integrations) { const integration = IntegrationSchema.parse(row); - if (!integration.manifest.model_types?.includes('chat')) { + if (!integration.manifest.model_types?.some(type => modelTypes.includes(type))) { continue; } models.push({ accountName: integration.providerAccount.name, modelName: integration.name, properties: toProperties(integration.metadata), + routing: toRoutingMetadata(integration.metadata), }); } return models; diff --git a/packages/trueforge/tests/unit/modelRouting/router.test.ts b/packages/trueforge/tests/unit/modelRouting/router.test.ts new file mode 100644 index 000000000..c69f053f4 --- /dev/null +++ b/packages/trueforge/tests/unit/modelRouting/router.test.ts @@ -0,0 +1,75 @@ +import { createLogger } from 'winston'; +import { createModelRouter } from '../../../src/modelRouting/router'; +import type { DecisionClient } from '../../../src/modelRouting/types'; + +const logger = createLogger({ silent: true }); + +// Haiku is listed first and is cheaper, so a tie would pick it; only Jev's answer can pick Sonnet. +const candidates = [ + { model: 'acct/claude-haiku-5-5', slug: 'claude-haiku-5-5', defaultEffort: 'medium', price: 0.2 }, + { model: 'acct/claude-sonnet-5-5', slug: 'claude-sonnet-5-5', defaultEffort: 'high', price: 3.6 }, +]; + +function route( + decision: DecisionClient, + mode: 'shadow' | 'on', + listCandidates: () => Promise = () => Promise.resolve(candidates), +) { + const routeModel = createModelRouter({ + mode, + listCandidates, + decision, + timeoutMs: 1_000, + maxCandidates: 60, + logger, + }); + return routeModel({ + threadId: 'child', + fallbackModel: 'acct/claude-opus-5-5', + task: { name: 'research', instructions: 'Compare pricing of three CI tools.', tools: [] }, + signal: new AbortController().signal, + }); +} + +function prefersSonnet(): DecisionClient & { ask: jest.Mock } { + return { + ask: jest.fn(({ questions }: Parameters[0]) => + Promise.resolve( + Object.fromEntries( + Object.entries(questions).map(([key, question]) => [ + key, + question.instructions.includes('Sonnet') ? 0.6 : 0.3, + ]), + ), + ), + ), + }; +} + +describe('createModelRouter', () => { + it('runs the highest yes-probability candidate in on mode', async () => { + await expect(route(prefersSonnet(), 'on')).resolves.toEqual({ model: 'acct/claude-sonnet-5-5' }); + }); + + it('asks the decision model but keeps the fallback model in shadow mode', async () => { + const decision = prefersSonnet(); + await expect(route(decision, 'shadow')).resolves.toEqual({}); + expect(decision.ask).toHaveBeenCalledTimes(1); + }); + + it('keeps the fallback model when listing candidates outlasts the timeout', async () => { + jest.useFakeTimers(); + try { + const pending = route(prefersSonnet(), 'on', () => new Promise(() => undefined)); + await jest.advanceTimersByTimeAsync(1_000); + await expect(pending).resolves.toEqual({}); + } finally { + jest.useRealTimers(); + } + }); + + it('keeps the fallback model when the decision model fails', async () => { + const failing: DecisionClient = { ask: () => Promise.reject(new Error('HTTP 403')) }; + await expect(route(failing, 'on')).resolves.toEqual({}); + }); +}); diff --git a/packages/trueforge/tests/unit/truefoundry/TrueFoundryModelProviderStore.test.ts b/packages/trueforge/tests/unit/truefoundry/TrueFoundryModelProviderStore.test.ts index 197db0ce8..83efec5b4 100644 --- a/packages/trueforge/tests/unit/truefoundry/TrueFoundryModelProviderStore.test.ts +++ b/packages/trueforge/tests/unit/truefoundry/TrueFoundryModelProviderStore.test.ts @@ -120,3 +120,35 @@ describe('TrueFoundryModelProviderStore authorization', () => { expect(client.vendToken).toHaveBeenCalledTimes(1); }); }); + +describe('TrueFoundryModelProviderStore decision models', () => { + it('resolves a decision model by name but never lists it', async () => { + const client = { + listProviderIntegrations: jest.fn().mockResolvedValue([ + { name: 'gpt-4', manifest: { model_types: ['chat'] }, providerAccount: { name: 'openai-main' } }, + { name: 'jev-1.13.0', manifest: { model_types: ['decision'] }, providerAccount: { name: 'typesafe' } }, + ]), + listGatewayInstallations: jest + .fn() + .mockResolvedValue([{ isDefault: true, manifest: { url: 'https://gateway.example' } }]), + vendToken: jest.fn().mockResolvedValue({ subjectToken: SUBJECT_TOKEN, actorToken: ACTOR_TOKEN }), + } as unknown as TrueFoundryServiceFoundryServerClient; + const store = new TrueFoundryModelProviderStore({ + client, + requestContext: createTrueFoundryRequestContext({ + tenant_id: TENANT, + subject: { id: 'user-1', type: 'user', display_name: 'User' }, + roles: [], + user_credential: CALLER_TOKEN, + }), + agent: undefined, + logger: createLogger({ silent: true }), + }); + + const listed = await store.listModels({ tenant_id: TENANT }); + expect(listed.map(model => model.name)).toEqual(['openai-main/gpt-4']); + await expect( + store.getProvider({ tenant_id: TENANT, name: 'typesafe', model_name: 'jev-1.13.0' }), + ).resolves.toMatchObject({ name: 'typesafe' }); + }); +}); diff --git a/packages/trueforge/tests/unit/truefoundry/trueFoundryNaming.test.ts b/packages/trueforge/tests/unit/truefoundry/trueFoundryNaming.test.ts index 2b7400e19..b0cb1d486 100644 --- a/packages/trueforge/tests/unit/truefoundry/trueFoundryNaming.test.ts +++ b/packages/trueforge/tests/unit/truefoundry/trueFoundryNaming.test.ts @@ -313,7 +313,9 @@ describe('TrueFoundry naming vs NameSchema', () => { }, ], }); - expect(models).toEqual([{ accountName: PROVIDER_ACCOUNT, modelName: MODEL_NAME, properties: {} }]); + expect(models).toEqual([ + expect.objectContaining({ accountName: PROVIDER_ACCOUNT, modelName: MODEL_NAME, properties: {} }), + ]); const fqn = `${models[0]?.accountName}/${models[0]?.modelName}`; expect(fqn).toBe(`${PROVIDER_ACCOUNT}/${MODEL_NAME}`); expect(NameSchema.safeParse(fqn).success).toBe(false);