Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/subagent-model-routing.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@truefoundry/trueforge-core': patch
'@truefoundry/trueforge': patch
---

Add optional subagent model routing. In TrueFoundry mode, `TRUEFOUNDRY_MODEL_ROUTING` turns it on per tenant: before a subagent starts, a decision model rates each of the caller's scored models for the task, and the highest-rated one is picked. `shadow` logs the pick and keeps the parent's model; `on` runs the pick. Off by default.
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,26 @@ export interface ResolvedAgentDefinition {
extraCapabilities?: AgentCapability[] | undefined;
}

/** Input to {@link ITurnResourceResolver.routeModel}: a subagent that's about to start, and the task it was given. */
export interface RouteModelInput {
threadId: string;
/** Model the subagent runs on when routing doesn't pick one (`provider/model`): the parent's model. */
fallbackModel: string;
task: {
name?: string | undefined;
instructions: string;
/** MCP server names, plus `web_search` and `sandbox` when available. */
tools: string[];
};
/** The subagent's abort signal; cancels routing when the turn is cancelled. */
signal: AbortSignal;
}

export interface RouteModelResult {
/** Model to run instead of the fallback. Undefined keeps the fallback. */
model?: string | undefined;
}

/**
* Per-run wiring contract consumed by SessionHandle.createTurn(). Implementations hold
* per-request state (secrets, providers, caches) freely.
Expand Down Expand Up @@ -63,6 +83,11 @@ export interface ITurnResourceResolver<TTurnCustom extends object = Record<strin
signal: AbortSignal;
tracing: AgentTracing;
}): Promise<ResolvedAgentDefinition>;
/**
* Chooses the model for a subagent before it starts. Omit to keep every subagent on its parent's model.
* Must not throw: on any failure, resolve without a model.
*/
routeModel?(input: RouteModelInput): Promise<RouteModelResult>;
/**
* Release per-run resources. Called by the harness at the end of the run:
* by TurnHandle.stream() in its finally (AFTER the terminal state write), or by
Expand Down
28 changes: 24 additions & 4 deletions packages/trueforge-core/src/agent-session/SessionHandle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -600,9 +600,29 @@ export class SessionHandle<
tracing: AgentTracing;
}): CreateDynamicSubAgentThread {
return async params => {
let request = params.request;
if (request.model === undefined && input.resolver.routeModel !== undefined) {
const routed = await input.resolver.routeModel({
threadId: params.threadId,
fallbackModel: params.parentDefinition.modelName,
task: {
name: request.name,
instructions: request.input,
tools: [
...(input.spec.mcp_servers ?? []).map(server => server.name),
...(input.spec.config.web_search.enabled && input.resolver.webSearchProvider ? ['web_search'] : []),
...(input.sandbox ? ['sandbox'] : []),
],
},
signal: params.signal,
});
if (routed.model !== undefined) {
request = { ...request, model: routed.model };
}
}
const { definition, extraCapabilities } = await input.resolver.resolveAgentDefinition({
spec: input.spec,
agent_info: params.request,
agent_info: request,
previousTurn: input.previous,
signal: params.signal,
tracing: input.tracing,
Expand All @@ -615,7 +635,7 @@ export class SessionHandle<
// The delegated task goes in as the initial user message; the sub-agent
// system prompt is SUB_AGENT_IDENTITY (added by AgentThread.buildInstruction).
instruction: undefined,
messages: [{ role: 'user', content: params.request.input }],
messages: [{ role: 'user', content: request.input }],
// Sub-agents should return free-form summaries to the parent, not the user-facing structured response.
responseFormat: undefined,
};
Expand All @@ -634,10 +654,10 @@ export class SessionHandle<
return new AgentThread({
definition: childDefinition,
threadId: params.threadId,
title: params.request.name,
title: request.name,
sandbox: input.sandbox,
parent: params.parent,
agentInfo: params.request,
agentInfo: request,
capabilities,
tracing: input.tracing,
logger: input.resolver.logger,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ import type { HarnessSandbox, SandboxInfo } from '../core/sandbox/Sandbox';
import type { AgentTracing } from '../core/tracing/AgentTracing';
import { NOOP_AGENT_TRACING } from '../core/tracing/NoopAgentTracing';
import type { IWebSearchProvider } from '../core/web-search/WebSearchProvider';
import type { ITurnResourceResolver, ResolvedAgentDefinition } from './ITurnResourceResolver';
import type {
ITurnResourceResolver,
ResolvedAgentDefinition,
RouteModelInput,
RouteModelResult,
} from './ITurnResourceResolver';
import type { TurnRecord } from './models/TurnRecord';
import type { AgentSpec } from './schemas/agentSpec';

Expand Down Expand Up @@ -80,6 +85,7 @@ export class TurnResourceResolver<
readonly #models = new Map<string, Promise<ResolvedModel>>();
#sandbox?: HarnessSandbox | undefined;
readonly maxParallelSubAgents?: number;
readonly routeModel?: (input: RouteModelInput) => Promise<RouteModelResult>;

constructor(
protected readonly deps: {
Expand Down Expand Up @@ -110,13 +116,18 @@ export class TurnResourceResolver<
* `resolveAgentDefinition` without calling `super` does not mount them.
*/
rootCapabilities?: (() => readonly AgentCapability[]) | undefined;
/** Chooses a subagent's model before it starts; omit to keep subagents on their parent's model. */
routeModel?: ((input: RouteModelInput) => Promise<RouteModelResult>) | undefined;
/** Forwarded to RemoteMCP / AgentThread (required by their constructors). */
logger: Logger;
},
) {
if (deps.maxParallelSubAgents !== undefined) {
this.maxParallelSubAgents = deps.maxParallelSubAgents;
}
if (deps.routeModel !== undefined) {
this.routeModel = deps.routeModel;
}
}

get webSearchProvider(): IWebSearchProvider | undefined {
Expand Down
2 changes: 1 addition & 1 deletion packages/trueforge-core/src/agent-session/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ export {
TurnOwnershipLostError,
} from './store/SessionStoreErrors';

export type { ITurnResourceResolver } from './ITurnResourceResolver';
export type { ITurnResourceResolver, RouteModelInput, RouteModelResult } from './ITurnResourceResolver';
export { TRUEFORGE_MCP_ID, TurnResourceResolver } from './TurnResourceResolver';
export type { TurnSandboxFactory } from './TurnResourceResolver';

Expand Down
5 changes: 5 additions & 0 deletions packages/trueforge/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,11 @@ POSTGRES_PORT=5432
## Tenants omitted are unfiltered. Example: {"internal":["openai-main","anthropic-main"]}
# TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS={"internal":["openai-main"]}

## Optional subagent model routing (JSON object). A decision model picks each subagent's model from the
## caller's scored models. Per tenant: mode off (default) | shadow (log the pick only) | on, and the tenant's
## own decision model on its gateway (account/model).
# TRUEFOUNDRY_MODEL_ROUTING={"tenants":{"internal":{"subagent":"shadow","decision_model":"typesafe/jev-1.13.0"}}}

## Internal mutual TLS for calls to the ServiceFoundry server. When true, trueforge presents its client
## certificate (and upgrades a mesh-direct http:// peer URL to https://). Only meaningful in-cluster,
## where TRUEFOUNDRY_SERVICEFOUNDRY_SERVER_URL points at the ServiceFoundry server Service directly.
Expand Down
44 changes: 43 additions & 1 deletion packages/trueforge/src/apis/turns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ import {
type RequestContext,
type ResolveRequestContext,
} from '../auth/identity';
import configuration from '../config';
import configuration, { isTrueFoundryModeEnabled } from '../config';
import type { IAccessControlStore } from '../db/accessControlStore';
import type { AgentRecord, IAgentStore } from '../db/agentStore';
import { EmptyCapabilitiesStore, type ICapabilitiesStore } from '../db/capabilitiesStore';
Expand All @@ -91,6 +91,7 @@ import { currentLogContext, runWithLogContext, updateLogContext } from '../logge
import { TRUEFORGE_SESSION_HEADER, X_TFG_TRUEFORGE_MCP } from '../mcp/server/trueforge/headers';
import type { TenantMemoryDeps } from '../memory/providers';
import { resolveSessionMemory } from '../memory/sessionMemory';
import { createModelRouter } from '../modelRouting/router';
import {
createAndExecuteTurnRoute,
createInternalSessionTurnRoute,
Expand Down Expand Up @@ -123,6 +124,7 @@ import { restoreTurnEventLog } from '../runtime/turnEventLogRestore';
import { MAX_SESSION_TITLE_LENGTH } from '../schemas/session';
import { assertGatewayMetadataRequestHeaders, X_TFY_METADATA } from '../truefoundry/gatewayMetadata';
import { X_TFG_MCP, X_TFG_MODELS, X_TFG_SKILLS, X_TFG_TOOLS } from '../truefoundry/inlineResources';
import { createJevDecisionClient } from '../truefoundry/jevDecisionClient';
import { X_TFG_MCP_HEADERS } from '../truefoundry/perServerMcpHeaders';
import { newId } from '../utils/id';
import { resolveWebSearchProvider } from '../websearch/providers';
Expand Down Expand Up @@ -363,6 +365,44 @@ interface BeginTurnExecutionParams<TTransaction> {
deps: BeginTurnExecutionDeps<TTransaction>;
}

/** This turn's subagent model router, or undefined when routing is off for the tenant. */
function subagentModelRouter(input: {
tenant_id: string;
modelProviderStore: IModelProviderStore;
turnMetadata: TurnMetadata;
logger: Logger;
}): ReturnType<typeof createModelRouter> | undefined {
const { tenant_id, modelProviderStore, turnMetadata, logger } = input;
const routing = isTrueFoundryModeEnabled(configuration) ? configuration.TRUEFOUNDRY_MODEL_ROUTING : undefined;
const tenantRouting = routing?.tenants[tenant_id];
if (
routing === undefined ||
tenantRouting === undefined ||
tenantRouting.subagent === 'off' ||
modelProviderStore.listRoutingCandidates === undefined
) {
return undefined;
}
return createModelRouter({
mode: tenantRouting.subagent,
listCandidates: () => modelProviderStore.listRoutingCandidates?.({ tenant_id }) ?? Promise.resolve([]),
decision: createJevDecisionClient({
resolveProviderConfig: async () =>
(
await getModelDetails({
tenant_id,
name: tenantRouting.decision_model,
store: modelProviderStore,
turnMetadata,
})
).providerConfig,
}),
timeoutMs: routing.timeout_ms,
maxCandidates: routing.max_candidates,
logger,
});
}

/**
* Builds the per-turn resolver. Agent / MCP / sandbox / LLM lookups are wired
* the same way: async factories over the corresponding stores.
Expand Down Expand Up @@ -404,8 +444,10 @@ function createTurnResolver(deps: {
} = deps;
const tenant_id = session.tenant_id;
const sessionId = session.session_id;
const routeModel = subagentModelRouter({ tenant_id, modelProviderStore, turnMetadata, logger });

return new TurnResourceResolver({
...(routeModel === undefined ? {} : { routeModel }),
llm: async name => {
const resolved = await getModelDetails({
tenant_id,
Expand Down
35 changes: 35 additions & 0 deletions packages/trueforge/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { DEFAULT_MAX_PARALLEL_SUB_AGENTS } from '@truefoundry/trueforge-core/cor
import envPaths from 'env-paths';
import { z } from 'zod';

import { MODEL_ROUTING_MODES } from './modelRouting/types';
import { parseRedisSentinelNodes, type RedisConnection } from './runtime/redis';

export type { RedisConnection };
Expand Down Expand Up @@ -156,6 +157,33 @@ export function parseOidcAllowedEmails(raw: string | undefined): string[] {
return parseCommaSeparatedEnvList(raw);
}

const TrueFoundryModelRoutingSchema = z.object({
timeout_ms: z.number().int().positive().default(2_000),
max_candidates: z.number().int().positive().default(60),
/** Per tenant: the decision model is the tenant's own gateway model (`account/model`). */
tenants: z.record(
z.string(),
z.object({ subagent: z.enum(MODEL_ROUTING_MODES).default('off'), decision_model: z.string().min(1) }),
),
});

export type TrueFoundryModelRouting = z.infer<typeof TrueFoundryModelRoutingSchema>;

/** Parses `TRUEFOUNDRY_MODEL_ROUTING` JSON. Empty / unset → undefined (routing off everywhere). */
export function parseTrueFoundryModelRouting(raw: string | undefined): TrueFoundryModelRouting | undefined {
if (raw === undefined || raw.trim() === '') {
return undefined;
}
try {
return TrueFoundryModelRoutingSchema.parse(JSON.parse(raw));
} catch (error) {
throw new Error(
'Environment variable TRUEFOUNDRY_MODEL_ROUTING must be a JSON object with per-tenant "tenants" entries (e.g. {"tenants":{"<tenant_id>":{"subagent":"shadow","decision_model":"typesafe/jev-1.13.0"}}})',
{ cause: error },
);
}
}

/**
* Parses `TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS` JSON.
* Empty / unset → `{}` (no filtering).
Expand Down Expand Up @@ -1102,6 +1130,12 @@ export type DistributedServerConfiguration = SharedServerConfiguration & {
* Env: `TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS`.
*/
TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS: Record<string, string[]>;
/**
* Optional subagent model routing (TrueFoundry mode only). JSON object with `timeout_ms`,
* `max_candidates`, and per-tenant entries: mode (`off` | `shadow` | `on`) and the tenant's decision
* model. Unset, or a tenant not listed → off. Env: `TRUEFOUNDRY_MODEL_ROUTING`.
*/
TRUEFOUNDRY_MODEL_ROUTING: TrueFoundryModelRouting | undefined;
/**
* Optional built-in web search provider (TrueFoundry mode only). JSON object
* `Record<string, string>` with `name` (`parallel`) and `api_key`.
Expand Down Expand Up @@ -1474,6 +1508,7 @@ const configuration: ServerConfiguration = standalone
TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS: parseTenantIdToAllowedModelProviderAccounts(
getEnv('TRUEFOUNDRY_TENANT_ID_TO_ALLOWED_MODEL_PROVIDER_ACCOUNTS', { required: false }),
),
TRUEFOUNDRY_MODEL_ROUTING: parseTrueFoundryModelRouting(getEnv('TRUEFOUNDRY_MODEL_ROUTING', { required: false })),
TRUEFOUNDRY_WEB_SEARCH_PROVIDER: parseTrueFoundryWebSearchProvider(
getEnv('TRUEFOUNDRY_WEB_SEARCH_PROVIDER', { required: false }),
),
Expand Down
6 changes: 6 additions & 0 deletions packages/trueforge/src/db/modelProviderStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
* `ModelProviderManifest` document. Implementations: Postgres, Sqlite, and
* TrueFoundry (read-only ServiceFoundry server listing).
*/
import type { RoutingCandidate } from '../modelRouting/types';
import type { ResourceName } from '../schemas/common';
import type { AvailableModel, ModelProperties, ModelProviderManifest } from '../schemas/modelProvider';
import type { TurnMetadata } from './turnMetadata';
Expand Down Expand Up @@ -85,6 +86,11 @@ export interface IModelProviderStore<TTransaction = never> {
record: ModelProviderRecord;
turnMetadata?: TurnMetadata;
}): Promise<Record<string, string>>;
/**
* The caller's models with the catalog fields model routing needs. Only the TrueFoundry-mode store
* implements it for now; stores that omit it never get routing.
*/
listRoutingCandidates?(input: ListModelProvidersInput): Promise<RoutingCandidate[]>;
}

export function modelPropertiesWarning(input: { properties: ModelProperties }): string | undefined {
Expand Down
55 changes: 55 additions & 0 deletions packages/trueforge/src/modelRouting/benchmarks.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// Descriptions are quoted verbatim, retrieved 2026-11-16; don't paraphrase them. Sources:
// https://artificialanalysis.ai/methodology/intelligence-benchmarking (all but HLE), https://lastexam.ai/ (HLE).

/**
* The nine evaluations in the Artificial Analysis Intelligence Index v5, the score routing uses.
* Jev is asked which ones a task resembles; the answers are logged, never used to pick a model.
*/
export const RESEMBLANCE_BENCHMARKS: Readonly<Record<string, { readonly name: string; readonly description: string }>> =
{
briefcase: {
name: 'AA-Briefcase',
description:
'AA-Briefcase is a new benchmark for testing models on realistic knowledge work tasks in complex projects built by industry experts. Models are evaluated on multi-week knowledge work projects, each with many linked tasks and thousands of input source files.',
},
automationbench: {
name: 'AutomationBench',
description:
'It tests whether models can complete realistic SaaS workflows that span multiple simulated business apps, using REST APIs as the tool interface.',
},
gdp_pdf: {
name: 'GDP.pdf',
description:
'A benchmark that tests whether models can reason over long, real-world professional documents and satisfy task-specific criteria.',
},
frontiercode: {
name: 'FrontierCode',
description:
'Each task asks an agent to make a change to a real open-source repository, and the benchmark measures whether the change is good enough for a maintainer to merge.',
},
terminal_bench: {
name: 'Terminal-Bench',
description:
'Covers software engineering, system administration, data processing, model training, and security, with each task graded by its own verification suite.',
},
terminal_bench_science: {
name: 'Terminal-Bench-Science',
description:
'Its research workflow tasks are drawn from real scientific practice, and experts curate and review each one. Each task has its own set of tests that the agent must pass by working in a terminal.',
},
critpt: {
name: 'CritPt',
description:
'Research-level physics reasoning benchmark with unpublished, frontier physics problems spanning a wide range of subfields.',
},
omniscience: {
name: 'AA-Omniscience',
description:
'AA-Omniscience is a knowledge and hallucination benchmark that measures factual reliability, rewards precise knowledge, and penalizes incorrect guesses or hallucinations.',
},
hle: {
name: "Humanity's Last Exam",
description:
"Humanity's Last Exam, a multi-modal benchmark at the frontier of human knowledge, designed to be the final closed-ended academic benchmark of its kind with broad subject coverage.",
},
};
Loading
Loading