From 6f6e5940f1a391a8413b31cef1aacc5c30cbe341 Mon Sep 17 00:00:00 2001 From: Nyvo <75425811+Nyvo-io@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:48:24 +0800 Subject: [PATCH] feat(core,storage): add user-overridable model facts Generated-by: Codex --- packages/core/package.json | 1 + .../core/src/__tests__/model-catalog.test.ts | 67 ++++ .../core/src/__tests__/model-facts.test.ts | 187 ++++++++++ .../__tests__/runtime-policy-codec.test.ts | 44 +++ packages/core/src/llm-connections.ts | 20 +- packages/core/src/model-catalog.ts | 43 ++- packages/core/src/model-facts.ts | 321 ++++++++++++++++++ packages/core/src/runtime-policy.ts | 2 + .../connection-catalog-codec.ts | 76 ++++- .../src/__tests__/protocol.test.ts | 4 + .../runtime-policy-coordinator.test.ts | 68 ++++ packages/runtime-host/src/protocol/index.ts | 3 + .../src/protocol/runtime-policy.ts | 55 ++- .../server/connection-effect-coordinator.ts | 1 + .../src/server/runtime-policy-coordinator.ts | 11 +- .../context-budget-model-facts.test.ts | 19 ++ packages/runtime/src/context-budget-policy.ts | 9 +- .../src/__tests__/model-facts-store.test.ts | 78 +++++ .../runtime-policy-model-facts.test.ts | 310 +++++++++++++++++ .../__tests__/runtime-policy-stores.test.ts | 25 +- packages/storage/src/model-facts-store.ts | 99 ++++++ .../connection-catalog-document.ts | 18 +- .../storage/src/runtime-policy/coordinator.ts | 171 ++++++++-- .../storage/src/runtime-policy/document-io.ts | 39 ++- packages/storage/src/runtime-policy/errors.ts | 1 + 25 files changed, 1619 insertions(+), 53 deletions(-) create mode 100644 packages/core/src/__tests__/model-facts.test.ts create mode 100644 packages/core/src/model-facts.ts create mode 100644 packages/storage/src/__tests__/model-facts-store.test.ts create mode 100644 packages/storage/src/__tests__/runtime-policy-model-facts.test.ts create mode 100644 packages/storage/src/model-facts-store.ts diff --git a/packages/core/package.json b/packages/core/package.json index 8e979f2e18..7c756d3b99 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -84,6 +84,7 @@ "./llm-connections": "./dist/llm-connections.js", "./provider-registry": "./dist/provider-registry.js", "./model-catalog": "./dist/model-catalog.js", + "./model-facts": "./dist/model-facts.js", "./model-metadata": "./dist/model-metadata.js", "./model-web-search": "./dist/model-web-search.js", "./model-thinking": "./dist/model-thinking.js", diff --git a/packages/core/src/__tests__/model-catalog.test.ts b/packages/core/src/__tests__/model-catalog.test.ts index 81b36bbfff..1e4afe101d 100644 --- a/packages/core/src/__tests__/model-catalog.test.ts +++ b/packages/core/src/__tests__/model-catalog.test.ts @@ -285,6 +285,72 @@ test('every picker sees a model the user enabled but no catalog describes', () = assert.deepEqual(declared?.provenance.sources?.userChoice, ['saved_model']); }); +test('catalog provenance follows the projected model facts marker used in production', () => { + const [entry] = buildConnectionModelCatalogEntries({ + connection: { + slug: 'facts', + providerType: 'openai', + defaultModel: 'custom-model', + models: [ + { + id: 'custom-model', + contextWindow: 200_000, + capabilities: { chat: true }, + factOverriddenFields: ['contextWindow', 'capabilities'], + }, + ], + modelSource: 'fetched', + }, + }); + assert.equal(entry?.capabilitySource, 'user_override'); + assert.equal(entry?.contextWindow, 200_000); +}); + +test('fallback provider catalogs include projected facts-backed models', () => { + const entries = buildConnectionModelCatalogEntries({ + connection: { + slug: 'opencode-free-facts', + providerType: 'opencode-free', + defaultModel: 'custom-free-model', + models: [ + { + id: 'custom-free-model', + contextWindow: 128_000, + factOverriddenFields: ['contextWindow', 'capabilities'], + }, + ], + modelSource: 'fallback', + }, + }); + const entry = entries.find((candidate) => candidate.id === 'custom-free-model'); + assert.equal(entry?.contextWindow, 128_000); + assert.equal(entry?.capabilitySource, 'user_override'); +}); + +test('fallback provider catalogs apply facts to known fallback models', () => { + const entries = buildConnectionModelCatalogEntries({ + connection: { + slug: 'opencode-free-known-facts', + providerType: 'opencode-free', + defaultModel: 'nemotron-3-ultra-free', + models: [ + { + id: 'nemotron-3-ultra-free', + contextWindow: 200_000, + inputLimit: 200_000, + capabilities: { chat: true }, + factOverriddenFields: ['contextWindow', 'inputLimit', 'capabilities'], + }, + ], + modelSource: 'fallback', + }, + }); + const entry = entries.find((candidate) => candidate.id === 'nemotron-3-ultra-free'); + assert.equal(entry?.contextWindow, 200_000); + assert.equal(entry?.inputLimit, 200_000); + assert.equal(entry?.capabilitySource, 'user_override'); +}); + test('unknown persisted provider ids return an empty catalog', () => { assert.deepEqual( buildConnectionModelCatalogEntries({ @@ -378,6 +444,7 @@ test('DeepSeek catalogs the V4 vision model display metadata from a bare provide reasoning: true, functionCalling: true, vision: true, + webSearch: true, }); assert.deepEqual(model?.modalities, { input: ['text', 'image'], output: ['text'] }); assert.equal(model?.canUseAsChatDefault, true); diff --git a/packages/core/src/__tests__/model-facts.test.ts b/packages/core/src/__tests__/model-facts.test.ts new file mode 100644 index 0000000000..930cff7012 --- /dev/null +++ b/packages/core/src/__tests__/model-facts.test.ts @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + applyModelFactOverride, + applyModelFactOverridesToConnection, + decodeModelFactsDocument, + modelFactKey, +} from '../model-facts.js'; +import { CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION } from '../runtime-policy.js'; + +test('model facts use provider:model keys and merge fields without replacing provider facts', () => { + const key = modelFactKey('openai', 'o4-mini'); + const document = decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { [key]: { contextWindow: 200_000, capabilities: { vision: false } } }, + }); + const model = applyModelFactOverride( + { + id: 'o4-mini', + displayName: 'Provider name', + maxOutputTokens: 4_000, + capabilities: { chat: true, vision: true }, + }, + document.overrides[key], + ); + assert.equal(model.displayName, 'Provider name'); + assert.equal(model.contextWindow, 200_000); + assert.deepEqual(model.capabilities, { chat: true, vision: false }); +}); + +test('malformed and unknown model fact fields are rejected', () => { + assert.throws(() => + decodeModelFactsDocument({ schemaVersion: 1, overrides: { 'openai:o4-mini': {} } }), + ); + assert.throws(() => + decodeModelFactsDocument({ schemaVersion: 1, overrides: { 'openai:o4-mini': { nope: true } } }), + ); + assert.throws(() => + decodeModelFactsDocument({ schemaVersion: 1, overrides: { 'o4-mini': { contextWindow: 1 } } }), + ); + assert.throws(() => + decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { 'openai:o4-mini': { contextWindow: 0 } }, + }), + ); + assert.throws(() => + decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { 'openai:o4-mini': { capabilities: { toString: true } } }, + }), + ); + assert.throws(() => + decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { 'toString:model': { contextWindow: 1 } }, + }), + ); +}); + +test('model fact keys preserve colons in provider model ids', () => { + const key = modelFactKey('ollama-cloud', 'gpt-oss:120b'); + assert.equal(key, 'ollama-cloud:gpt-oss:120b'); + const document = decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { [key]: { contextWindow: 131_072 } }, + }); + assert.equal(document.overrides[key]?.contextWindow, 131_072); +}); + +test('override-only models are projected only when enabled', () => { + const connection = { + slug: 'openai', + providerType: 'openai' as const, + defaultModel: 'custom', + enabledModelIds: ['custom'], + models: [{ id: 'provider-model' }], + }; + const result = applyModelFactOverridesToConnection(connection, { + 'openai:custom': { contextWindow: 64_000 }, + 'openai:hidden': { contextWindow: 1_000 }, + }); + assert.deepEqual(result.models, [ + { id: 'provider-model' }, + { + id: 'custom', + contextWindow: 64_000, + inputLimit: 64_000, + factOverriddenFields: ['contextWindow', 'inputLimit'], + }, + ]); +}); + +test('context window facts cannot be truncated by an older input limit', () => { + const result = applyModelFactOverride( + { id: 'model', contextWindow: 8_192, inputLimit: 8_192 }, + { contextWindow: 200_000 }, + ); + assert.equal(result.contextWindow, 200_000); + assert.equal(result.inputLimit, 200_000); +}); + +test('overrides replace fields on discovered models while preserving untouched provider facts', () => { + const result = applyModelFactOverridesToConnection( + { + providerType: 'openai', + defaultModel: 'provider-model', + enabledModelIds: ['provider-model'], + models: [ + { id: 'provider-model', contextWindow: 8_000, capabilities: { chat: true, vision: true } }, + ], + }, + { 'openai:provider-model': { contextWindow: 64_000, capabilities: { vision: false } } }, + ); + assert.equal(result.models?.[0]?.contextWindow, 64_000); + assert.deepEqual(result.models?.[0]?.capabilities, { chat: true, vision: false }); +}); + +test('model-fact projection preserves the catalog model bound', () => { + const projected = applyModelFactOverridesToConnection( + { + providerType: 'openai', + enabledModelIds: ['custom-model'], + models: Array.from({ length: CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION }, (_, index) => ({ + id: `provider-model-${index}`, + })), + }, + { 'openai:custom-model': { contextWindow: 64_000 } }, + ); + + assert.equal(projected.models?.length, CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION); + assert.equal( + projected.models?.some((model) => model.id === 'custom-model'), + false, + ); +}); + +test('modality overrides merge directions independently', () => { + const result = applyModelFactOverride( + { + id: 'multimodal', + modalities: { input: ['text', 'image', 'pdf'], output: ['text', 'audio'] }, + }, + { modalities: { input: ['text'] } }, + ); + assert.deepEqual(result.modalities, { + input: ['text'], + output: ['text', 'audio'], + }); +}); + +test('model capabilities preserve web search facts from metadata and overrides', () => { + const result = applyModelFactOverride( + { id: 'web-model', capabilities: { webSearch: true } }, + { capabilities: { chat: true } }, + ); + assert.deepEqual(result.capabilities, { webSearch: true, chat: true }); +}); + +test('model fact overrides accept parallel tool-call capability metadata', () => { + const document = decodeModelFactsDocument({ + schemaVersion: 1, + overrides: { 'openai:tool-model': { capabilities: { parallelToolCalls: false } } }, + }); + assert.deepEqual(document.overrides['openai:tool-model']?.capabilities, { + parallelToolCalls: false, + }); +}); diff --git a/packages/core/src/__tests__/runtime-policy-codec.test.ts b/packages/core/src/__tests__/runtime-policy-codec.test.ts index f21350a4db..06bdd1328c 100644 --- a/packages/core/src/__tests__/runtime-policy-codec.test.ts +++ b/packages/core/src/__tests__/runtime-policy-codec.test.ts @@ -484,6 +484,50 @@ test('normalizes exact bounded model discovery results', () => { } }); +test('normalizes extended model facts used by the runtime host catalog', () => { + const result = normalizeConnectionModelDiscoveryResult({ + models: [ + { + id: 'custom-model', + description: 'A custom model', + inputLimit: 120_000, + knowledgeCutoff: '2025-01', + structuredOutput: true, + lastUpdated: '2026-01-01', + modalities: { input: ['text', 'image'], output: ['text'] }, + }, + ], + source: 'fetched', + fetchedAt: 42, + }); + assert.deepEqual(result.models[0], { + id: 'custom-model', + description: 'A custom model', + inputLimit: 120_000, + knowledgeCutoff: '2025-01', + structuredOutput: true, + lastUpdated: '2026-01-01', + modalities: { input: ['text', 'image'], output: ['text'] }, + }); +}); + +test('rejects sparse model modality arrays', () => { + assert.throws( + () => + normalizeConnectionModelDiscoveryResult({ + models: [ + { + id: 'custom-model', + modalities: { input: Array(1), output: ['text'] }, + }, + ], + source: 'fetched', + fetchedAt: 42, + }), + RuntimePolicyDomainDecodeError, + ); +}); + test('credential domain validation requires material but leaves capacity to callers', () => { const input = normalizeSetCredentialInput({ locator: { diff --git a/packages/core/src/llm-connections.ts b/packages/core/src/llm-connections.ts index 472a90b508..0b65b23697 100644 --- a/packages/core/src/llm-connections.ts +++ b/packages/core/src/llm-connections.ts @@ -115,8 +115,27 @@ export interface ModelInfo { input: Array<'text' | 'image' | 'audio' | 'pdf'>; output: Array<'text' | 'image' | 'audio'>; }; + /** + * Read-time provenance for values overlaid from model-facts.json. This is + * never persisted in a provider inventory; it lets catalog consumers show + * where a projected value came from. + */ + factOverriddenFields?: readonly ModelFactField[]; } +export type ModelFactField = + | 'displayName' + | 'description' + | 'apiProtocol' + | 'contextWindow' + | 'inputLimit' + | 'maxOutputTokens' + | 'knowledgeCutoff' + | 'structuredOutput' + | 'lastUpdated' + | 'capabilities' + | 'modalities'; + export type ModelDiscoverySource = 'fetched' | 'fallback'; export interface ModelDiscoveryResult { @@ -411,7 +430,6 @@ export function reconcileConnectionAfterModelFetch( ), ), ]; - // Seed a first choice only for a connection that has never had a list to // pick from: four providers ship no `fallbackModels`, so for them discovery // is the only place a first default can come from. diff --git a/packages/core/src/model-catalog.ts b/packages/core/src/model-catalog.ts index 72a56c67a4..0cb56e4568 100644 --- a/packages/core/src/model-catalog.ts +++ b/packages/core/src/model-catalog.ts @@ -63,6 +63,7 @@ export interface KnownModelCapabilities { functionCalling?: true; parallelToolCalls?: true; imageGeneration?: true; + webSearch?: true; } export interface ModelCatalogPricing { @@ -264,13 +265,35 @@ export function buildConnectionModelCatalogEntries( const fallbackModels = [...(catalogFallbackModels ?? defaults.fallbackModels)].filter( (id) => !broken.has(id), ); + const fallbackModelIds = new Set(fallbackModels); + const projectedModelsById = new Map( + (connection.models ?? []).filter(({ id }) => !broken.has(id)).map((model) => [model.id, model]), + ); + // Fallback providers have no live inventory, but a projected connection can + // still carry enabled model-facts entries that are absent from the static + // list. Keep both sets in the catalog so those user-declared models retain + // their metadata and provenance. + const models = supportsModelDiscovery + ? connection.models?.filter(({ id }) => !broken.has(id)) + : [ + ...fallbackModels.map( + (id) => + projectedModelsById.get(id) ?? { + id, + ...displayNameForKnownModel(connection.providerType, id), + }, + ), + ...(connection.models ?? []).filter( + (model) => !broken.has(model.id) && !fallbackModelIds.has(model.id), + ), + ]; return buildModelCatalogEntries({ providerType: connection.providerType, connectionSlug: connection.slug, defaultModel: connection.defaultModel, - models: connection.models?.filter(({ id }) => !broken.has(id)), - modelSource: connection.modelSource, - modelsFetchedAt: connection.modelsFetchedAt, + models, + modelSource: supportsModelDiscovery ? connection.modelSource : 'fallback', + modelsFetchedAt: supportsModelDiscovery ? connection.modelsFetchedAt : undefined, fallbackModels: supportsModelDiscovery ? (input.fallbackModels ?? fallbackModels) : fallbackModels, @@ -356,11 +379,13 @@ function makeEntry( providerType: input.providerType, ...(input.connectionSlug ? { connectionSlug: input.connectionSlug } : {}), source, - capabilitySource: normalizedModel.capabilities - ? source - : metadata.capabilities - ? 'static_catalog' - : 'unknown', + capabilitySource: normalizedModel.factOverriddenFields?.includes('capabilities') + ? 'user_override' + : normalizedModel.capabilities + ? source + : metadata.capabilities + ? 'static_catalog' + : 'unknown', unavailableReason, availability: availabilityOf(unavailableReason), canUseAsChatDefault: canUseUnavailableReasonAsDefault(unavailableReason), @@ -408,6 +433,7 @@ function mergeCapabilities( parallelToolCalls: providerCapabilities.parallelToolCalls ?? metadataCapabilities.parallelToolCalls, imageGeneration: providerCapabilities.imageGeneration ?? metadataCapabilities.imageGeneration, + webSearch: providerCapabilities.webSearch ?? metadataCapabilities.webSearch, }; } @@ -647,6 +673,7 @@ function normalizeCapabilities(caps: ModelInfo['capabilities']): KnownModelCapab ...(caps.functionCalling === true ? { functionCalling: true as const } : {}), ...(caps.parallelToolCalls === true ? { parallelToolCalls: true as const } : {}), ...(caps.imageGeneration === true ? { imageGeneration: true as const } : {}), + ...(caps.webSearch === true ? { webSearch: true as const } : {}), }; } diff --git a/packages/core/src/model-facts.ts b/packages/core/src/model-facts.ts new file mode 100644 index 0000000000..ecd03223cf --- /dev/null +++ b/packages/core/src/model-facts.ts @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { PROVIDER_REGISTRY, type ProviderType } from './provider-registry.js'; +import type { ModelFactField, ModelInfo } from './llm-connections.js'; +import { + CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION, + type ConnectionCatalogEntry, + type ConnectionCatalogSnapshot, +} from './runtime-policy.js'; + +export const MODEL_FACTS_SCHEMA_VERSION = 1 as const; +export const MODEL_FACT_KEY_MAX_LENGTH = 512; +export const MODEL_FACTS_MAX_OVERRIDES = 512; + +export type ModelFactOverride = Readonly< + Omit>, 'modalities'> & { + readonly modalities?: Readonly>; + } +>; +export type ModelFactOverrides = Readonly>; + +export interface ModelFactsDocument { + readonly schemaVersion: typeof MODEL_FACTS_SCHEMA_VERSION; + readonly overrides: ModelFactOverrides; +} + +export class UnsupportedModelFactsSchemaError extends Error { + constructor(readonly schemaVersion: number) { + super(`model-facts.json schema version ${schemaVersion} is not supported`); + this.name = 'UnsupportedModelFactsSchemaError'; + } +} + +const PROVIDER_ID_PATTERN = /^[^:\s]{1,128}$/; +// Model ids may contain colons (for example, Ollama's `gpt-oss:120b`). The +// provider is the only component that is constrained to the first separator. +const MODEL_ID_PATTERN = /^[^\s]{1,256}$/; +const PROVIDER_MODEL_KEY_PATTERN = /^([^:\s]{1,128}):([^\s]{1,256})$/; +const MAX_FACT_NUMBER = 10_000_000_000; + +export function modelFactKey(providerType: ProviderType | string, modelId: string): string { + const provider = providerType.trim(); + const model = modelId.trim(); + if (!provider || !model || !PROVIDER_ID_PATTERN.test(provider) || !MODEL_ID_PATTERN.test(model)) { + throw new Error('Model fact keys must use a non-empty provider:model identifier'); + } + if (!Object.hasOwn(PROVIDER_REGISTRY, provider)) { + throw new Error(`Unknown model-facts provider: ${provider}`); + } + const key = `${provider}:${model}`; + if (key.length > MODEL_FACT_KEY_MAX_LENGTH) throw new Error('Model fact key is too long'); + return key; +} + +export function lookupModelFactOverride( + overrides: ModelFactOverrides | undefined, + providerType: ProviderType | string, + modelId: string, +): ModelFactOverride | undefined { + if (!overrides) return undefined; + try { + return overrides[modelFactKey(providerType, modelId)]; + } catch { + return undefined; + } +} + +/** Return model ids with facts for one provider without exposing other providers. */ +export function modelFactOverrideIdsForProvider( + overrides: ModelFactOverrides | undefined, + providerType: ProviderType | string, +): string[] { + if (!overrides) return []; + const prefix = `${providerType.trim()}:`; + return Object.keys(overrides) + .filter((key) => key.startsWith(prefix)) + .map((key) => key.slice(prefix.length)); +} + +export function decodeModelFactsDocument(value: unknown): ModelFactsDocument { + if (!isRecord(value)) throw new Error('model-facts.json must be an object'); + if (!Number.isSafeInteger(value.schemaVersion)) { + throw new Error('model-facts.json schemaVersion must be an integer'); + } + if (value.schemaVersion !== MODEL_FACTS_SCHEMA_VERSION) + throw new UnsupportedModelFactsSchemaError(value.schemaVersion); + if (!isRecord(value.overrides)) throw new Error('model-facts.json.overrides must be an object'); + const keys = Object.keys(value.overrides); + if (keys.length > MODEL_FACTS_MAX_OVERRIDES) + throw new Error('model-facts.json has too many overrides'); + const overrides: Record = {}; + for (const key of keys) { + const match = PROVIDER_MODEL_KEY_PATTERN.exec(key); + if (!match || key.length > MODEL_FACT_KEY_MAX_LENGTH) throw new Error('Invalid model fact key'); + modelFactKey(match[1]!, match[2]!); + overrides[key] = normalizeModelFactOverride(value.overrides[key]); + } + return { schemaVersion: MODEL_FACTS_SCHEMA_VERSION, overrides }; +} + +export function normalizeModelFactOverride(value: unknown): ModelFactOverride { + if (!isRecord(value)) throw new Error('Model fact override must be an object'); + const allowed = new Set([ + 'displayName', + 'description', + 'apiProtocol', + 'contextWindow', + 'inputLimit', + 'maxOutputTokens', + 'knowledgeCutoff', + 'structuredOutput', + 'lastUpdated', + 'capabilities', + 'modalities', + ]); + for (const key of Object.keys(value)) + if (!allowed.has(key)) throw new Error(`Unknown model fact field: ${key}`); + const result: Record = {}; + for (const key of ['displayName', 'description', 'knowledgeCutoff', 'lastUpdated'] as const) { + if (key in value) { + if (typeof value[key] !== 'string' || value[key].length > 2048) + throw new Error(`Invalid ${key}`); + result[key] = value[key]; + } + } + if ('apiProtocol' in value) { + if ( + value.apiProtocol !== 'openai-chat' && + value.apiProtocol !== 'openai-responses' && + value.apiProtocol !== 'anthropic-messages' + ) + throw new Error('Invalid apiProtocol'); + result.apiProtocol = value.apiProtocol; + } + for (const key of ['contextWindow', 'inputLimit', 'maxOutputTokens'] as const) { + if (key in value) { + const number = value[key]; + if (!isPositiveBoundedInteger(number)) throw new Error(`Invalid ${key}`); + result[key] = number; + } + } + for (const key of ['structuredOutput'] as const) { + if (key in value) { + if (typeof value[key] !== 'boolean') throw new Error(`Invalid ${key}`); + result[key] = value[key]; + } + } + if ('capabilities' in value) result.capabilities = normalizeCapabilities(value.capabilities); + if ('modalities' in value) result.modalities = normalizeModalities(value.modalities); + if (Object.keys(result).length === 0) throw new Error('Model fact override must not be empty'); + return result as ModelFactOverride; +} + +function normalizeCapabilities(value: unknown): NonNullable { + if (!isRecord(value)) throw new Error('Invalid capabilities'); + const result: Record = {}; + const allowed = [ + 'chat', + 'vision', + 'reasoning', + 'functionCalling', + 'parallelToolCalls', + 'imageGeneration', + 'webSearch', + ] as const; + for (const key of allowed) { + if (key in value) { + if (typeof value[key] !== 'boolean') throw new Error(`Invalid capability: ${key}`); + result[key] = value[key]; + } + } + for (const key of Object.keys(value)) + if (!allowed.includes(key as (typeof allowed)[number])) { + throw new Error(`Unknown capability: ${key}`); + } + return result; +} + +function normalizeModalities(value: unknown): NonNullable { + if (!isRecord(value)) throw new Error('Invalid modalities'); + if (value.input === undefined && value.output === undefined) + throw new Error('Invalid modalities'); + const input = normalizeModalityDirection(value.input, isModality); + const output = normalizeModalityDirection(value.output, isOutputModality); + return { + ...(input === undefined ? {} : { input }), + ...(output === undefined ? {} : { output }), + }; +} + +function normalizeModalityDirection( + value: unknown, + allowed: (value: unknown) => value is T, +): T[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) throw new Error('Invalid modality value'); + const entries = Array.from(value); + if (!entries.every(allowed)) throw new Error('Invalid modality value'); + return [...new Set(entries)]; +} + +function isModality(value: unknown): value is 'text' | 'image' | 'audio' | 'pdf' { + return value === 'text' || value === 'image' || value === 'audio' || value === 'pdf'; +} +function isOutputModality(value: unknown): value is 'text' | 'image' | 'audio' { + return value === 'text' || value === 'image' || value === 'audio'; +} +function isPositiveBoundedInteger(value: unknown): value is number { + return ( + typeof value === 'number' && + Number.isSafeInteger(value) && + value > 0 && + value <= MAX_FACT_NUMBER + ); +} +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function applyModelFactOverride( + model: ModelInfo, + override: ModelFactOverride | undefined, +): ModelInfo { + if (!override) return { ...model }; + const overriddenFields = new Set(model.factOverriddenFields); + for (const field of Object.keys(override) as ModelFactField[]) overriddenFields.add(field); + // An authoritative context-window correction must not leave a stale, + // narrower provider input limit to silently win in the runtime resolver. + if (override.contextWindow !== undefined && override.inputLimit === undefined) { + overriddenFields.add('inputLimit'); + } + // Modalities are merged per direction: a partial override changes only the + // direction it names and preserves the provider's other direction. + const modalities = override.modalities + ? { + input: override.modalities.input ?? model.modalities?.input ?? ['text'], + output: override.modalities.output ?? model.modalities?.output ?? ['text'], + } + : model.modalities; + const { modalities: _ignoredModalities, ...scalarOverride } = override; + return { + ...model, + ...scalarOverride, + id: model.id, + factOverriddenFields: [...overriddenFields], + ...(override.contextWindow === undefined || override.inputLimit !== undefined + ? {} + : { inputLimit: override.contextWindow }), + ...(override.capabilities === undefined + ? {} + : { capabilities: { ...model.capabilities, ...override.capabilities } }), + ...(modalities === undefined ? {} : { modalities }), + } satisfies ModelInfo; +} + +type ModelFactConnectionLike = { + readonly providerType: ProviderType; + readonly defaultModel?: string; + readonly models?: readonly ModelInfo[]; + readonly enabledModelIds?: readonly string[]; +}; + +export function applyModelFactOverridesToConnection( + connection: T, + overrides: ModelFactOverrides, +): T { + const models = (connection.models ?? []).map((model) => + applyModelFactOverride( + model, + lookupModelFactOverride(overrides, connection.providerType, model.id), + ), + ); + const existing = new Set(models.map((model) => model.id)); + const enabled = new Set( + connection.enabledModelIds ?? + (connection.defaultModel === undefined ? [] : [connection.defaultModel]), + ); + for (const modelId of enabled) { + if (models.length >= CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION) break; + if (existing.has(modelId)) continue; + const override = lookupModelFactOverride(overrides, connection.providerType, modelId); + if (override) { + models.push(applyModelFactOverride({ id: modelId }, override)); + existing.add(modelId); + } + } + return { ...connection, models } as T; +} + +export function applyModelFactOverridesToCatalogSnapshot( + snapshot: ConnectionCatalogSnapshot, + overrides: ModelFactOverrides, +): ConnectionCatalogSnapshot { + return { + ...snapshot, + connections: snapshot.connections.map( + (connection) => + applyModelFactOverridesToConnection( + connection, + overrides, + ) as unknown as ConnectionCatalogEntry, + ), + }; +} diff --git a/packages/core/src/runtime-policy.ts b/packages/core/src/runtime-policy.ts index a6da3087af..69061cf67d 100644 --- a/packages/core/src/runtime-policy.ts +++ b/packages/core/src/runtime-policy.ts @@ -244,6 +244,8 @@ export interface ConnectionCatalogEntry extends ConnectionConfiguration { readonly modelSource?: ConnectionModelDiscoveryResult['source']; readonly modelsFetchedAt?: ConnectionModelDiscoveryResult['fetchedAt']; readonly lastTest?: ConnectionTestSummary; + /** Digest of the model-facts subset used when `lastTest` was recorded. */ + readonly lastTestModelFactsFingerprint?: string; } export type ConnectionCatalogEntryDraft = ConnectionConfiguration; diff --git a/packages/core/src/runtime-policy/connection-catalog-codec.ts b/packages/core/src/runtime-policy/connection-catalog-codec.ts index d90639cee5..6535ee3c0f 100644 --- a/packages/core/src/runtime-policy/connection-catalog-codec.ts +++ b/packages/core/src/runtime-policy/connection-catalog-codec.ts @@ -372,6 +372,7 @@ export function decodeCanonicalConnectionCatalogEntry(value: unknown): Connectio 'modelSource', 'modelsFetchedAt', 'lastTest', + 'lastTestModelFactsFingerprint', ], [ 'connectionId', @@ -440,6 +441,15 @@ export function decodeCanonicalConnectionCatalogEntry(value: unknown): Connectio ...(item.lastTest === undefined ? {} : { lastTest: decodeConnectionTestSummary(item.lastTest) }), + ...(item.lastTestModelFactsFingerprint === undefined + ? {} + : { + lastTestModelFactsFingerprint: stringValue( + item.lastTestModelFactsFingerprint, + 'connection test model facts fingerprint', + 128, + ), + }), }; assertCanonicalValue(value, decoded, 'connection catalog entry'); return decoded; @@ -465,7 +475,20 @@ export function decodeConnectionModel(value: unknown): ConnectionModel { const item = exactRecord( value, 'connection model', - ['id', 'displayName', 'apiProtocol', 'contextWindow', 'maxOutputTokens', 'capabilities'], + [ + 'id', + 'displayName', + 'description', + 'apiProtocol', + 'contextWindow', + 'inputLimit', + 'maxOutputTokens', + 'knowledgeCutoff', + 'structuredOutput', + 'lastUpdated', + 'capabilities', + 'modalities', + ], ['id'], ); if ( @@ -500,11 +523,16 @@ export function decodeConnectionModel(value: unknown): ConnectionModel { ); } } + const modalities = + item.modalities === undefined ? undefined : decodeModelModalities(item.modalities); return { id: decodeConnectionModelId(item.id), ...(item.displayName === undefined ? {} : { displayName: stringValue(item.displayName, 'model display name', 512) }), + ...(item.description === undefined + ? {} + : { description: stringValue(item.description, 'model description', 2048) }), ...(item.apiProtocol === undefined ? {} : { apiProtocol: item.apiProtocol }), ...(item.contextWindow === undefined ? {} @@ -516,6 +544,16 @@ export function decodeConnectionModel(value: unknown): ConnectionModel { Number.MAX_SAFE_INTEGER, ), }), + ...(item.inputLimit === undefined + ? {} + : { + inputLimit: integerValue( + item.inputLimit, + 'model input limit', + 1, + Number.MAX_SAFE_INTEGER, + ), + }), ...(item.maxOutputTokens === undefined ? {} : { @@ -526,10 +564,46 @@ export function decodeConnectionModel(value: unknown): ConnectionModel { Number.MAX_SAFE_INTEGER, ), }), + ...(item.knowledgeCutoff === undefined + ? {} + : { knowledgeCutoff: stringValue(item.knowledgeCutoff, 'model knowledge cutoff', 2048) }), + ...(item.structuredOutput === undefined + ? {} + : { structuredOutput: booleanValue(item.structuredOutput, 'model structured output') }), + ...(item.lastUpdated === undefined + ? {} + : { lastUpdated: stringValue(item.lastUpdated, 'model last updated', 2048) }), ...(capabilities === undefined ? {} : { capabilities }), + ...(modalities === undefined ? {} : { modalities }), }; } +function decodeModelModalities(value: unknown): NonNullable { + const item = exactRecord(value, 'connection model modalities', ['input', 'output']); + if (!Array.isArray(item.input) || !Array.isArray(item.output)) { + throw domainError('connection model modalities must contain input and output arrays'); + } + const input = Array.from(item.input, (entry) => decodeModelInputModality(entry)); + const output = Array.from(item.output, (entry) => decodeModelOutputModality(entry)); + return { input, output }; +} + +function decodeModelInputModality(value: unknown): 'text' | 'image' | 'audio' | 'pdf' { + const modality = stringValue(value, 'connection model input modality', 16); + if (modality !== 'text' && modality !== 'image' && modality !== 'audio' && modality !== 'pdf') { + throw domainError('connection model input modality is invalid'); + } + return modality; +} + +function decodeModelOutputModality(value: unknown): 'text' | 'image' | 'audio' { + const modality = stringValue(value, 'connection model output modality', 16); + if (modality !== 'text' && modality !== 'image' && modality !== 'audio') { + throw domainError('connection model output modality is invalid'); + } + return modality; +} + export function decodeConnectionTestSummary(value: unknown): ConnectionTestSummary { const item = exactRecord( value, diff --git a/packages/runtime-host/src/__tests__/protocol.test.ts b/packages/runtime-host/src/__tests__/protocol.test.ts index 24c14a6f89..9fd833d3eb 100644 --- a/packages/runtime-host/src/__tests__/protocol.test.ts +++ b/packages/runtime-host/src/__tests__/protocol.test.ts @@ -342,6 +342,10 @@ describe('Runtime Host bootstrap protocol', () => { assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 56); }); + test('publishes a new compatibility epoch for catalog model-facts provenance', () => { + assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 65); + }); + test('selects the highest mutually supported protocol and rejects a gap', () => { assert.equal(negotiateProtocol({ min: 0, max: 0 }, { min: 0, max: 0 }), 0); assert.equal(negotiateProtocol({ min: 1, max: 3 }, { min: 2, max: 4 }), 3); diff --git a/packages/runtime-host/src/__tests__/runtime-policy-coordinator.test.ts b/packages/runtime-host/src/__tests__/runtime-policy-coordinator.test.ts index dc03b56c84..f98b04310f 100644 --- a/packages/runtime-host/src/__tests__/runtime-policy-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/runtime-policy-coordinator.test.ts @@ -877,6 +877,74 @@ test('a fully profiled relay catalog paginates with profiles riding per item', a }); }); +test('catalog pages preserve model-facts provenance from the projected snapshot', async () => { + await withCoordinator(async ({ coordinator, root, stores }) => { + const created = await stores.connectionCatalog.create({ + expectedCatalogRevision: 0, + connection: { + slug: 'facts-backed', + name: 'Facts backed', + providerType: 'openai', + enabled: true, + enabledModelIds: ['custom-model'], + }, + }); + assert.equal(created.kind, 'committed'); + if (created.kind !== 'committed') return; + const connection = created.snapshot.connections[0]; + assert.ok(connection); + if (!connection) return; + const credential = await stores.credentialVault.set({ + locator: { + scope: 'connection', + connectionId: connection.connectionId, + kind: 'api_key', + }, + expected: null, + secret: 'facts-backed-test-key', + }); + assert.equal(credential.kind, 'committed'); + if (credential.kind !== 'committed') return; + const fetch = await stores.operations.beginModelFetch(connection.connectionId); + assert.equal(fetch.kind, 'ready'); + if (fetch.kind !== 'ready') return; + const discovered = await stores.operations.completeModelFetch(fetch.ticket, { + models: [{ id: 'provider-model' }], + source: 'fetched', + fetchedAt: 1, + }); + assert.equal(discovered.kind, 'committed'); + if (discovered.kind !== 'committed') return; + await writeFile( + join(root, 'model-facts.json'), + JSON.stringify({ + schemaVersion: 1, + overrides: { 'openai:custom-model': { contextWindow: 200_000 } }, + }), + 'utf8', + ); + + const result = await coordinator.handlers['connection.catalog.query']( + { kind: 'start' }, + context, + ); + + assert.equal(result.ok, true); + if (!result.ok || result.result.kind !== 'page') return; + const decoded = RUNTIME_POLICY_OPERATION_SPECS['connection.catalog.query'].decodeOutput( + result.result, + ); + if (decoded.kind !== 'page') return; + assert.deepEqual( + decoded.items.find( + (item): item is Extract => + item.kind === 'model' && item.model.id === 'custom-model', + )?.model.factOverriddenFields, + ['contextWindow', 'inputLimit'], + ); + }); +}); + test('catalog protocol preserves an extra request body after a committed update', async () => { await withCoordinator(async ({ coordinator, stores }) => { const emptyBodyBytes = Buffer.byteLength(JSON.stringify({ padding: '' }), 'utf8'); diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index a359bcdefd..f6cd4767c2 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -98,6 +98,9 @@ export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 67 as const; // 67: Message lifecycle queries expose durable execution ownership and // cancellation. Older peers cannot decode or provide the closed proof list. // 66: Peer Mesh queries expose one canonical transit selection and runtime metrics. +// Runtime Policy catalog models also gained validated user-overridden fact +// provenance at epoch 66. Older peers reject this projected model shape or lose +// that provenance while resolving model facts. // 65: live `tool_start` frames may carry optional `intent` / `argsPreview` // keys. Older Clients decode the event with a strict allowed-key list and tear // the connection down on unknown keys, so the pair must be refused up front. diff --git a/packages/runtime-host/src/protocol/runtime-policy.ts b/packages/runtime-host/src/protocol/runtime-policy.ts index 191f683842..7d0359d2dc 100644 --- a/packages/runtime-host/src/protocol/runtime-policy.ts +++ b/packages/runtime-host/src/protocol/runtime-policy.ts @@ -78,6 +78,33 @@ export const CONNECTION_CATALOG_PAGE_MAX_BYTES = 48 * 1024; export const RUNTIME_POLICY_SNAPSHOT_MAX_BYTES = 48 * 1024; export const CREDENTIAL_SECRET_MAX_BYTES = 10 * 1024; +const CONNECTION_MODEL_FIELDS = [ + 'id', + 'displayName', + 'description', + 'apiProtocol', + 'contextWindow', + 'inputLimit', + 'maxOutputTokens', + 'knowledgeCutoff', + 'structuredOutput', + 'lastUpdated', + 'capabilities', + 'modalities', +] as const; +const MODEL_FACT_OVERRIDE_FIELDS = [ + 'displayName', + 'description', + 'apiProtocol', + 'contextWindow', + 'inputLimit', + 'maxOutputTokens', + 'knowledgeCutoff', + 'structuredOutput', + 'lastUpdated', + 'capabilities', + 'modalities', +] as const; const QUERY_ERRORS = [ 'host_not_ready', 'host_draining', @@ -588,7 +615,7 @@ function catalogPageItem(value: unknown): ConnectionCatalogPageItem { 0, CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION - 1, ), - model: decodeDomain(() => decodeConnectionModel(modelItem.model)), + model: decodeProjectedCatalogModel(modelItem.model), }; } if (item.kind !== 'connection') @@ -693,6 +720,32 @@ function catalogPageItem(value: unknown): ConnectionCatalogPageItem { }; } +/** Decode catalog-only read-time provenance without admitting it to persisted models. */ +function decodeProjectedCatalogModel(value: unknown): ConnectionModel { + const item = requireShapedRecord( + value, + 'projected connection model', + ['id'], + [...CONNECTION_MODEL_FIELDS.slice(1), 'factOverriddenFields'], + ); + const { factOverriddenFields: rawOverriddenFields, ...persistentModel } = item; + const model = decodeDomain(() => decodeConnectionModel(persistentModel)); + if (rawOverriddenFields === undefined) return model; + if (!Array.isArray(rawOverriddenFields) || rawOverriddenFields.length === 0) { + throw invalidProtocolFrame('Invalid model fact overridden fields'); + } + const factOverriddenFields = rawOverriddenFields.map((field) => { + if (!(MODEL_FACT_OVERRIDE_FIELDS as readonly unknown[]).includes(field)) { + throw invalidProtocolFrame('Invalid model fact overridden field'); + } + return field as (typeof MODEL_FACT_OVERRIDE_FIELDS)[number]; + }); + if (new Set(factOverriddenFields).size !== factOverriddenFields.length) { + throw invalidProtocolFrame('Duplicate model fact overridden field'); + } + return { ...model, factOverriddenFields }; +} + function decodeCreateConnectionInput(value: unknown): CreateCatalogConnectionInput { const input = decodeDomain(() => normalizeCreateCatalogConnectionInput(value)); assertMutationEnabledModelIds(input.connection.enabledModelIds); diff --git a/packages/runtime-host/src/server/connection-effect-coordinator.ts b/packages/runtime-host/src/server/connection-effect-coordinator.ts index 5133a7903d..667a80e1e0 100644 --- a/packages/runtime-host/src/server/connection-effect-coordinator.ts +++ b/packages/runtime-host/src/server/connection-effect-coordinator.ts @@ -578,6 +578,7 @@ function storeFailure< return operationFailure('persistence_failed', 'Connection effect persistence failed'); case 'invalid_policy_input': case 'invalid_connection_input': + case 'revision_conflict': return operationFailure('invalid_request', 'Connection effect request is invalid'); case 'invalid_credential_input': throw new Error('Connection effect admitted an invalid credential operation'); diff --git a/packages/runtime-host/src/server/runtime-policy-coordinator.ts b/packages/runtime-host/src/server/runtime-policy-coordinator.ts index 7f3e9dbcdf..40710e9d59 100644 --- a/packages/runtime-host/src/server/runtime-policy-coordinator.ts +++ b/packages/runtime-host/src/server/runtime-policy-coordinator.ts @@ -360,6 +360,7 @@ export class HostRuntimePolicyCoordinator { }; case 'invalid_policy_input': case 'invalid_connection_input': + case 'revision_conflict': if (mode !== 'mutation') { throw invariantFailure('A read operation admitted invalid runtime policy input'); } @@ -440,7 +441,15 @@ function projectCatalogItems(snapshot: ConnectionCatalogSnapshot): ConnectionCat // Profiles ride on their enabled_model_id item, never in one header // table: a header item is atomic to the paginator, so a long declaration // list would make the whole connection unreadable. - const { enabledModelIds, models, relayModelProfiles, ...header } = connection; + const { + enabledModelIds, + models, + relayModelProfiles, + // This marker is durable invalidation metadata, not part of the + // client-visible catalog protocol. + lastTestModelFactsFingerprint: _lastTestModelFactsFingerprint, + ...header + } = connection; items.push({ kind: 'connection', connectionIndex, diff --git a/packages/runtime/src/__tests__/context-budget-model-facts.test.ts b/packages/runtime/src/__tests__/context-budget-model-facts.test.ts index d472ad9354..46eeeb73f1 100644 --- a/packages/runtime/src/__tests__/context-budget-model-facts.test.ts +++ b/packages/runtime/src/__tests__/context-budget-model-facts.test.ts @@ -58,3 +58,22 @@ test('a relay user declaration remains ahead of runtime and static model facts', assert.equal(resolveSelectedModelContextWindow(connection, undefined), 32_000); }); + +test('a model-facts context window is the authoritative user declaration', () => { + const connection = { + slug: 'relay', + providerType: 'openai-compatible' as const, + defaultModel: 'relay-model', + models: [ + { + id: 'relay-model', + contextWindow: 200_000, + inputLimit: 200_000, + factOverriddenFields: ['contextWindow', 'inputLimit'] as const, + }, + ], + relayModelProfiles: { 'relay-model': { contextWindow: 8_192 } }, + }; + + assert.equal(resolveSelectedModelContextWindow(connection, undefined), 200_000); +}); diff --git a/packages/runtime/src/context-budget-policy.ts b/packages/runtime/src/context-budget-policy.ts index 2c210c7fdc..ef94524602 100644 --- a/packages/runtime/src/context-budget-policy.ts +++ b/packages/runtime/src/context-budget-policy.ts @@ -96,13 +96,20 @@ export function resolveSelectedModelContextWindow( ): number | undefined { const selectedModelId = modelId ?? connection.defaultModel; if (selectedModelId === undefined) return undefined; + const model = connection.models?.find((candidate) => candidate.id === selectedModelId); + // A model-facts pin is the cross-provider correction authority. It must win + // over the older relay-only declaration so catalog display and execution use + // the same window. Relay declarations retain their existing precedence when + // there is no facts pin for this field. + if (model?.factOverriddenFields?.includes('contextWindow')) { + return narrowestPositiveLimit(model.contextWindow, model.inputLimit); + } // A user declaration outranks both the provider's /models report and // generated metadata — mirrors the declared-vision precedence in // model-metadata.ts. A declared context window is legal on any provider: it // states a fact about the model, not a request shape (#1584). const declared = relayModelProfile(connection, selectedModelId)?.contextWindow; if (declared !== undefined) return declared; - const model = connection.models?.find((candidate) => candidate.id === selectedModelId); const metadata = lookupModelMetadata(connection.providerType, selectedModelId); // Provider/access-path facts outrank static metadata. Within one source, // use the narrowest positive bound: models.dev's input limit can be lower diff --git a/packages/storage/src/__tests__/model-facts-store.test.ts b/packages/storage/src/__tests__/model-facts-store.test.ts new file mode 100644 index 0000000000..afb600aa36 --- /dev/null +++ b/packages/storage/src/__tests__/model-facts-store.test.ts @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { mkdtemp, readdir, readFile, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { ModelFactsDocumentOwner } from '../model-facts-store.js'; +import { cleanupRuntimePolicyDocumentTemps } from '../runtime-policy/document-io.js'; + +test('model facts persist and malformed documents fail closed with a bounded diagnostic', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-model-facts-')); + try { + const owner = new ModelFactsDocumentOwner(); + assert.deepEqual((await owner.readWithDiagnostics(root)).document.overrides, {}); + await writeFile(join(root, 'model-facts.json'), '{not-json}', 'utf8'); + const result = await owner.readWithDiagnostics(root); + assert.equal(result.diagnostic, 'malformed'); + assert.deepEqual(result.document.overrides, {}); + await writeFile( + join(root, 'model-facts.json'), + JSON.stringify({ schemaVersion: 1, overrides: { 'openai:o4-mini': { unknown: true } } }), + 'utf8', + ); + assert.equal((await owner.readWithDiagnostics(root)).diagnostic, 'malformed'); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('model facts temporary writes are removed by runtime policy recovery', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-model-facts-recovery-')); + try { + await writeFile( + join(root, 'model-facts.json.00000000-0000-4000-8000-000000000000.tmp'), + '{}', + 'utf8', + ); + await cleanupRuntimePolicyDocumentTemps(root); + assert.deepEqual(await readdir(root), []); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('future model facts schemas fail closed without rewriting the document', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-model-facts-future-')); + try { + const owner = new ModelFactsDocumentOwner(); + const future = JSON.stringify({ + schemaVersion: 2, + overrides: { 'openai:o4-mini': { contextWindow: 1 } }, + }); + await writeFile(join(root, 'model-facts.json'), future, 'utf8'); + const read = await owner.readWithDiagnostics(root); + assert.equal(read.diagnostic, 'unsupported_schema'); + assert.equal(await readFile(join(root, 'model-facts.json'), 'utf8'), future); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/storage/src/__tests__/runtime-policy-model-facts.test.ts b/packages/storage/src/__tests__/runtime-policy-model-facts.test.ts new file mode 100644 index 0000000000..21f2411c75 --- /dev/null +++ b/packages/storage/src/__tests__/runtime-policy-model-facts.test.ts @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { RuntimePolicyCoordinator } from '../runtime-policy/coordinator.js'; + +test('runtime policy catalog overlays enabled custom model facts without changing the raw catalog', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const created = await coordinator.createConnection({ + expectedCatalogRevision: 0, + connection: { + slug: 'custom-openai', + name: 'Custom OpenAI', + providerType: 'ollama', + enabled: true, + enabledModelIds: ['custom-model'], + }, + }); + assert.equal(created.kind, 'committed'); + assert.equal(Object.isFrozen(created), true); + if (created.kind === 'committed') assert.equal(Object.isFrozen(created.snapshot), true); + await writeModelFacts(root, { 'ollama:custom-model': { contextWindow: 64_000 } }); + const snapshot = await coordinator.getCatalogSnapshot(); + const model = snapshot.connections[0]?.models.find( + (candidate) => candidate.id === 'custom-model', + ); + assert.equal(model?.contextWindow, 64_000); + const prepared = await coordinator.beginConnectionTest( + snapshot.connections[0]!.connectionId, + null, + ); + assert.equal(prepared.kind, 'ready'); + if (prepared.kind === 'ready') { + const tested = await coordinator.completeConnectionTest(prepared.ticket, { + status: 'verified', + checkedAt: '2026-08-01T00:00:00.000Z', + }); + assert.equal(tested.kind, 'committed'); + } + assert.equal( + (await coordinator.getCatalogSnapshot()).connections[0]?.lastTest?.status, + 'verified', + ); + const restarted = new RuntimePolicyCoordinator((operation) => operation(root)); + const persisted = await restarted.getCatalogSnapshot(); + assert.equal( + persisted.connections[0]?.models.find((candidate) => candidate.id === 'custom-model') + ?.contextWindow, + 64_000, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('legacy connection verification survives unrelated model facts overrides', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-legacy-verification-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + const prepared = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(prepared.kind, 'ready'); + if (prepared.kind === 'ready') { + assert.equal( + ( + await coordinator.completeConnectionTest( + prepared.ticket, + verifiedAt('2026-08-01T00:00:00.000Z'), + ) + ).kind, + 'committed', + ); + } + + const catalogPath = join(root, 'connection-catalog.json'); + const catalog = JSON.parse(await readFile(catalogPath, 'utf8')) as { + connections: Array>; + }; + delete catalog.connections[0]!.lastTestModelFactsFingerprint; + await writeFile(catalogPath, `${JSON.stringify(catalog)}\n`, 'utf8'); + + await writeModelFacts(root, { 'openai:unrelated-model': { contextWindow: 64_000 } }); + assert.equal( + (await coordinator.getCatalogSnapshot()).connections[0]?.lastTest?.status, + 'verified', + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('display-only model facts preserve verification and in-flight tests', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-display-only-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + const initial = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(initial.kind, 'ready'); + if (initial.kind !== 'ready') return; + assert.equal( + ( + await coordinator.completeConnectionTest( + initial.ticket, + verifiedAt('2026-08-01T00:00:00.000Z'), + ) + ).kind, + 'committed', + ); + + const inFlight = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(inFlight.kind, 'ready'); + await writeModelFacts(root, { 'ollama:custom-model': { displayName: 'Friendly name' } }); + assert.equal( + (await coordinator.getCatalogSnapshot()).connections[0]?.lastTest?.status, + 'verified', + ); + if (inFlight.kind === 'ready') { + assert.equal( + ( + await coordinator.completeConnectionTest( + inFlight.ticket, + verifiedAt('2026-08-01T00:01:00.000Z'), + ) + ).kind, + 'committed', + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('model fetch keeps an enabled facts-backed model outside provider inventory', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-refresh-')); + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + await writeModelFacts(root, { + 'ollama:custom-model': { contextWindow: 64_000 }, + 'ollama:unselected-model': { contextWindow: 128_000 }, + }); + const beforeRefresh = await coordinator.getCatalogSnapshot(); + const defaulted = await coordinator.setDefaultTarget({ + expectedCatalogRevision: beforeRefresh.revision, + target: { connectionId, modelId: 'custom-model' }, + }); + assert.equal(defaulted.kind, 'committed'); + + const fetch = await coordinator.beginModelFetch(connectionId); + assert.equal(fetch.kind, 'ready'); + if (fetch.kind !== 'ready') return; + const refreshed = await coordinator.completeModelFetch(fetch.ticket, { + models: [{ id: 'live-model' }], + source: 'fetched', + fetchedAt: 1, + }); + assert.equal(refreshed.kind, 'committed'); + if (refreshed.kind !== 'committed') return; + + const raw = await ( + coordinator as unknown as { + catalog: { + read(root: string): Promise<{ + connections: readonly { models: readonly unknown[] }[]; + }>; + }; + } + ).catalog.read(root); + assert.deepEqual(raw.connections[0]?.models, [{ id: 'live-model' }]); + const projected = refreshed.snapshot.connections[0]; + assert.deepEqual(projected?.enabledModelIds, ['custom-model']); + assert.deepEqual(refreshed.snapshot.defaultTarget, { + connectionId, + modelId: 'custom-model', + }); + assert.equal( + projected?.models.find((model) => model.id === 'custom-model')?.contextWindow, + 64_000, + ); + assert.equal( + projected?.models.some((model) => model.id === 'unselected-model'), + false, + ); + + const execution = await coordinator.resolveExecutionConnection({ + kind: 'catalog_slug', + connectionSlug: 'custom-openai', + }); + assert.equal(execution.kind, 'ready'); + if (execution.kind === 'ready') { + assert.equal( + execution.connection.models?.some((model) => model.id === 'custom-model'), + true, + ); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('protocol model facts edits clear verification, supersede tickets, and warn on malformed input', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-facts-external-edit-')); + const emitWarning = process.emitWarning; + const warnings: string[] = []; + process.emitWarning = ((warning: string | Error) => { + warnings.push(String(warning)); + }) as typeof process.emitWarning; + try { + const coordinator = new RuntimePolicyCoordinator((operation) => operation(root)); + const connectionId = await createTestConnection(coordinator); + await writeModelFacts(root, { 'ollama:custom-model': { contextWindow: 64_000 } }); + const verified = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(verified.kind, 'ready'); + if (verified.kind === 'ready') { + assert.equal( + ( + await coordinator.completeConnectionTest( + verified.ticket, + verifiedAt('2026-08-01T00:00:00.000Z'), + ) + ).kind, + 'committed', + ); + } + const ticket = await coordinator.beginConnectionTest(connectionId, null); + assert.equal(ticket.kind, 'ready'); + await writeFile( + join(root, 'model-facts.json'), + JSON.stringify({ + schemaVersion: 1, + overrides: { 'ollama:custom-model': { apiProtocol: 'openai-responses' } }, + }), + 'utf8', + ); + if (ticket.kind === 'ready') { + assert.deepEqual( + await coordinator.completeConnectionTest( + ticket.ticket, + verifiedAt('2026-08-01T00:01:00.000Z'), + ), + { kind: 'superseded', changed: ['connection'] }, + ); + } + assert.equal((await coordinator.getCatalogSnapshot()).connections[0]?.lastTest, undefined); + + await writeFile(join(root, 'model-facts.json'), '{not-json}', 'utf8'); + const snapshot = await coordinator.getCatalogSnapshot(); + assert.equal( + snapshot.connections[0]?.models.find((model) => model.id === 'custom-model')?.contextWindow, + undefined, + ); + assert.equal( + warnings.some((warning) => warning.includes('model-facts.json')), + true, + ); + } finally { + process.emitWarning = emitWarning; + await rm(root, { recursive: true, force: true }); + } +}); + +async function createTestConnection(coordinator: RuntimePolicyCoordinator): Promise { + const created = await coordinator.createConnection({ + expectedCatalogRevision: 0, + connection: { + slug: 'custom-openai', + name: 'Custom OpenAI', + providerType: 'ollama', + enabled: true, + enabledModelIds: ['custom-model'], + }, + }); + assert.equal(created.kind, 'committed'); + if (created.kind !== 'committed') throw new Error('Expected connection creation to commit'); + return created.snapshot.connections[0]!.connectionId; +} + +function verifiedAt(checkedAt: string) { + return { status: 'verified' as const, checkedAt }; +} + +async function writeModelFacts(root: string, overrides: Record): Promise { + await writeFile( + join(root, 'model-facts.json'), + JSON.stringify({ schemaVersion: 1, overrides }), + 'utf8', + ); +} diff --git a/packages/storage/src/__tests__/runtime-policy-stores.test.ts b/packages/storage/src/__tests__/runtime-policy-stores.test.ts index 2d1bbfa26f..aa0ab44efd 100644 --- a/packages/storage/src/__tests__/runtime-policy-stores.test.ts +++ b/packages/storage/src/__tests__/runtime-policy-stores.test.ts @@ -1543,7 +1543,7 @@ describe('runtime policy stores', () => { }); test('conditionally commits discovery and test facts from the latest admitted state with one-shot tickets', async () => { - await withInteractiveOwner(async ({ stores }) => { + await withInteractiveOwner(async ({ root, stores }) => { const connection = await createConnection( stores, 0, @@ -1566,6 +1566,14 @@ describe('runtime policy stores', () => { ); const fetch = await stores.operations.beginModelFetch(connection.connectionId); + await writeFile( + join(root, 'model-facts.json'), + JSON.stringify({ + schemaVersion: 1, + overrides: { 'openai:gpt-5': { apiProtocol: 'openai-responses' } }, + }), + 'utf8', + ); const testTicket = await stores.operations.beginConnectionTest( connection.connectionId, 'gpt-5', @@ -1574,6 +1582,7 @@ describe('runtime policy stores', () => { assert.equal(testTicket.kind, 'ready'); if (fetch.kind !== 'ready' || testTicket.kind !== 'ready') return; assert.equal(testTicket.modelId, 'gpt-5'); + assert.equal(testTicket.connection.models?.[0]?.apiProtocol, 'openai-responses'); assert.equal(fetch.secretMaterial.connection?.secret, 'effect-secret'); await assert.rejects( @@ -1608,9 +1617,17 @@ describe('runtime policy stores', () => { if (discovered.kind !== 'committed') return; const afterDiscovery = discovered.snapshot.connections[0]; assert.ok(afterDiscovery); - assert.deepEqual(afterDiscovery.models, [{ id: 'gpt-5.1' }, { id: 'gpt-5.2' }]); - // Discovery records what the provider reported; it does not re-decide what - // the user enabled. `gpt-5` was chosen and stays chosen (#1584). + assert.deepEqual(afterDiscovery.models, [ + { id: 'gpt-5.1' }, + { id: 'gpt-5.2' }, + { + id: 'gpt-5', + apiProtocol: 'openai-responses', + factOverriddenFields: ['apiProtocol'], + }, + ]); + // Discovery records what the provider reported while retaining the + // selected fact-backed model for selectors and execution. assert.deepEqual(afterDiscovery.enabledModelIds, ['gpt-5']); assert.equal(afterDiscovery.modelSource, 'fetched'); assert.equal(afterDiscovery.modelsFetchedAt, 42); diff --git a/packages/storage/src/model-facts-store.ts b/packages/storage/src/model-facts-store.ts new file mode 100644 index 0000000000..80af74d9b0 --- /dev/null +++ b/packages/storage/src/model-facts-store.ts @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import { + decodeModelFactsDocument, + MODEL_FACTS_SCHEMA_VERSION, + UnsupportedModelFactsSchemaError, + type ModelFactsDocument, +} from '@maka/core/model-facts'; +import type { ConnectionCatalogEntry } from '@maka/core/runtime-policy'; +import { readBoundedDocumentBytes } from './runtime-policy/document-io.js'; +import { RuntimePolicyStoreError } from './runtime-policy/errors.js'; + +export const MODEL_FACTS_DOCUMENT_MAX_BYTES = 256 * 1024; +const FILE = 'model-facts.json'; + +export interface ModelFactsReadResult { + readonly document: ModelFactsDocument; + readonly diagnostic?: 'malformed' | 'oversized' | 'unsupported_schema'; + readonly fingerprint: string; +} + +export class ModelFactsDocumentOwner { + async readWithDiagnostics(root: string): Promise { + let bytes: Buffer | undefined; + try { + bytes = await readBoundedDocumentBytes(root, FILE, MODEL_FACTS_DOCUMENT_MAX_BYTES); + } catch (error) { + if (error instanceof RuntimePolicyStoreError && error.code === 'invalid_document') { + return { + document: emptyDocument(), + diagnostic: error.message.includes('exceeds') ? 'oversized' : 'malformed', + fingerprint: `invalid:${error.message}`, + }; + } + throw error; + } + if (bytes === undefined) return { document: emptyDocument(), fingerprint: 'missing' }; + const fingerprint = fingerprintBytes(bytes); + let value: unknown; + try { + value = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) as unknown; + return { document: decodeModelFactsDocument(value), fingerprint }; + } catch (error) { + if (error instanceof UnsupportedModelFactsSchemaError) { + return { document: emptyDocument(), diagnostic: 'unsupported_schema', fingerprint }; + } + return { document: emptyDocument(), diagnostic: 'malformed', fingerprint }; + } + } + + fingerprintForConnection( + document: ModelFactsDocument, + connection: Pick, + ): string { + const modelIds = new Set([ + ...(connection.models ?? []).map((model) => model.id), + ...connection.enabledModelIds, + ]); + const entries = Object.entries(document.overrides) + .filter(([key]) => { + const separator = key.indexOf(':'); + return ( + separator > 0 && + key.slice(0, separator) === connection.providerType && + modelIds.has(key.slice(separator + 1)) && + Object.prototype.hasOwnProperty.call(document.overrides[key]!, 'apiProtocol') + ); + }) + .map(([key, override]) => [key, { apiProtocol: override.apiProtocol }] as const) + .sort(([left], [right]) => left.localeCompare(right)); + return fingerprintBytes(Buffer.from(JSON.stringify(entries), 'utf8')); + } +} + +function emptyDocument(): ModelFactsDocument { + return { schemaVersion: MODEL_FACTS_SCHEMA_VERSION, overrides: {} }; +} + +function fingerprintBytes(bytes: Buffer): string { + return createHash('sha256').update(bytes).digest('hex'); +} diff --git a/packages/storage/src/runtime-policy/connection-catalog-document.ts b/packages/storage/src/runtime-policy/connection-catalog-document.ts index 60245c940d..b063ae26f8 100644 --- a/packages/storage/src/runtime-policy/connection-catalog-document.ts +++ b/packages/storage/src/runtime-policy/connection-catalog-document.ts @@ -467,7 +467,9 @@ export class ConnectionCatalogDocumentOwner { hasModelInventory: previous.models.length > 0, }, result.models, - { aliases: modelIdAliasesForProvider(previous.providerType) }, + { + aliases: modelIdAliasesForProvider(previous.providerType), + }, ); // Discovery MOVES a target: a provider's model rename carries the default // across by alias. A default outside the selection the reconciler just @@ -668,6 +670,7 @@ export class ConnectionCatalogDocumentOwner { current: ConnectionCatalogDocument, expected: ConnectionVersionBasis, rawResult: ConnectionTestSummary, + modelFactsFingerprint: string, ): Promise { const result = decodeConnectionInput(() => decodeConnectionTestSummary(rawResult)); const index = findConnectionIndex(current, expected); @@ -679,6 +682,7 @@ export class ConnectionCatalogDocumentOwner { ...previous, revision: nextRevision(previous.revision), lastTest: result, + lastTestModelFactsFingerprint: modelFactsFingerprint, }); } @@ -700,7 +704,11 @@ export class ConnectionCatalogDocumentOwner { // provider that can no longer be tested, so there is nothing to // invalidate. if (previous.lastTest === undefined || isRetiredProvider(previous.providerType)) return false; - const { lastTest: _lastTest, ...withoutLastTest } = previous; + const { + lastTest: _lastTest, + lastTestModelFactsFingerprint: _lastTestModelFactsFingerprint, + ...withoutLastTest + } = previous; await this.writePatchedResult(root, current, index, { ...withoutLastTest, revision: nextRevision(previous.revision), @@ -723,7 +731,11 @@ export class ConnectionCatalogDocumentOwner { if (!current.connections.some(invalidates)) return false; const connections = current.connections.map((connection) => { if (!invalidates(connection)) return connection; - const { lastTest: _lastTest, ...withoutLastTest } = connection; + const { + lastTest: _lastTest, + lastTestModelFactsFingerprint: _lastTestModelFactsFingerprint, + ...withoutLastTest + } = connection; return { ...withoutLastTest, revision: nextRevision(connection.revision), diff --git a/packages/storage/src/runtime-policy/coordinator.ts b/packages/storage/src/runtime-policy/coordinator.ts index f4a3418b95..03f5ee14d5 100644 --- a/packages/storage/src/runtime-policy/coordinator.ts +++ b/packages/storage/src/runtime-policy/coordinator.ts @@ -52,6 +52,11 @@ import { type SetDefaultConnectionTargetInput, type UpdateCatalogConnectionInput, } from '@maka/core/runtime-policy'; +import { + applyModelFactOverridesToConnection, + applyModelFactOverridesToCatalogSnapshot, + type ModelFactsDocument, +} from '@maka/core/model-facts'; import { deriveProviderAuthContract, type ProviderAuthAction } from '@maka/core/provider-auth'; import { isRetiredProvider } from '@maka/core/provider-registry'; import { @@ -129,6 +134,7 @@ import { } from './onboarding-transaction.js'; import { policySnapshot, RuntimePolicyDocumentOwner } from './policy-document.js'; import { SerializedOperationLane } from '../serialized-operation-lane.js'; +import { ModelFactsDocumentOwner } from '../model-facts-store.js'; type RootExecutor = (operation: (root: string) => Promise) => Promise; @@ -178,6 +184,7 @@ type SemanticConnectionBasis = readonly kind: 'connection_test'; readonly requestBodyOverlayJson: string; readonly model: ConnectionTestModelBasis; + readonly modelFactsFingerprint: string; }); interface ConnectionTicketRecord { @@ -226,6 +233,8 @@ export class RuntimePolicyCoordinator { private readonly policy = new RuntimePolicyDocumentOwner(); private readonly catalog = new ConnectionCatalogDocumentOwner(); private readonly vault = new CredentialVaultDocumentOwner(); + private readonly modelFacts = new ModelFactsDocumentOwner(); + private warnedModelFactsFingerprint: string | undefined; private readonly tickets = new WeakMap(); private onboardingRecoveryRequired = false; @@ -252,7 +261,7 @@ export class RuntimePolicyCoordinator { } getCatalogSnapshot() { - return this.inLane(async (root) => catalogSnapshot(await this.catalog.read(root))); + return this.inLane(async (root) => this.projectCatalogSnapshot(root)); } getVaultSnapshot() { @@ -300,11 +309,15 @@ export class RuntimePolicyCoordinator { } createConnection(input: CreateCatalogConnectionInput) { - return this.inLane((root) => this.catalog.create(root, input)); + return this.inLane(async (root) => + this.projectCatalogMutation(root, await this.catalog.create(root, input)), + ); } updateConnection(input: UpdateCatalogConnectionInput) { - return this.inLane((root) => this.catalog.update(root, input)); + return this.inLane(async (root) => + this.projectCatalogMutation(root, await this.catalog.update(root, input)), + ); } removeConnection(rawInput: RemoveCatalogConnectionInput) { @@ -325,7 +338,10 @@ export class RuntimePolicyCoordinator { const vault = await this.vault.read(root); if (!connection) { await this.vault.deleteConnectionCredentials(root, vault, expected.connectionId); - return deepFreeze({ kind: 'committed' as const, snapshot: catalogSnapshot(catalog) }); + return deepFreeze({ + kind: 'committed' as const, + snapshot: await this.projectCatalogSnapshot(root), + }); } const result = await this.catalog.remove(root, { expected }); if (result.kind === 'committed') { @@ -338,12 +354,14 @@ export class RuntimePolicyCoordinator { ); } } - return result; + return this.projectCatalogMutation(root, result); }); } setDefaultTarget(input: SetDefaultConnectionTargetInput) { - return this.inLane((root) => this.catalog.setDefaultTarget(root, input)); + return this.inLane(async (root) => + this.projectCatalogMutation(root, await this.catalog.setDefaultTarget(root, input)), + ); } migrateSystemSeed(input: MigrateSystemSeedInput) { @@ -674,7 +692,10 @@ export class RuntimePolicyCoordinator { if (prepared.kind !== 'ready') return prepared; return deepFreeze({ kind: 'ready' as const, - connection: structuredClone(connection), + connection: applyModelFactOverridesToConnection( + structuredClone(connection), + (await this.readModelFacts(root)).document.overrides, + ), secretMaterial: prepared.secretMaterial, networkProxy: structuredClone(prepared.networkProxy), }); @@ -974,7 +995,10 @@ export class RuntimePolicyCoordinator { connectionBasis(checked.connection), result, ); - return deepFreeze({ kind: 'committed' as const, snapshot }); + return deepFreeze({ + kind: 'committed' as const, + snapshot: await this.projectCatalogSnapshot(root), + }); }), ); } @@ -1243,21 +1267,32 @@ export class RuntimePolicyCoordinator { 'test_credentials', ); if (prepared.kind !== 'ready') return prepared; + const facts = await this.readModelFacts(root); + const projectedConnection = applyModelFactOverridesToConnection( + structuredClone(prepared.connection), + facts.document.overrides, + ); const modelId = rawModelId === null ? null : decodeConnectionInput(() => decodeConnectionModelId(rawModelId)); - if (modelId !== null && !isCanonicalConnectionTestModel(prepared.connection, modelId)) { + if (modelId !== null && !isCanonicalConnectionTestModel(projectedConnection, modelId)) { throw codecError( 'invalid_connection_input', 'Connection test model is not in the canonical model set', ); } - const ticket = this.issueTicket('connection_test', connectionTestSemanticBasis(prepared)); + const ticket = this.issueTicket( + 'connection_test', + connectionTestSemanticBasis( + prepared, + this.modelFacts.fingerprintForConnection(facts.document, prepared.connection), + ), + ); return deepFreeze({ kind: 'ready' as const, ticket: ticket as ConnectionTestTicket, - connection: structuredClone(prepared.connection), + connection: projectedConnection, modelId, secretMaterial: prepared.secretMaterial, networkProxy: structuredClone(prepared.networkProxy), @@ -1272,6 +1307,9 @@ export class RuntimePolicyCoordinator { const claimed = this.claimTicket(ticket, 'connection_test'); return this.completeClaimedTicket(claimed, () => this.inLane(async (root) => { + if (claimed.basis.kind !== 'connection_test') { + throw new Error('Coordinator admitted a non-connection-test ticket'); + } const catalog = await this.catalog.read(root); const checked = await this.checkSemanticConnectionBasis(root, catalog, claimed.basis); if (checked.changed.length > 0 || !checked.connection) { @@ -1282,8 +1320,12 @@ export class RuntimePolicyCoordinator { catalog, connectionBasis(checked.connection), result, + claimed.basis.modelFactsFingerprint, ); - return deepFreeze({ kind: 'committed' as const, snapshot }); + return deepFreeze({ + kind: 'committed' as const, + snapshot: await this.projectCatalogSnapshot(root), + }); }), ); } @@ -1427,17 +1469,31 @@ export class RuntimePolicyCoordinator { }> { const connection = findConnection(catalog, { connectionId: basis.connectionId }); const changed: ConnectionEffectChangedDomain[] = []; + const facts = basis.kind === 'connection_test' ? await this.readModelFacts(root) : undefined; + if ( + basis.kind === 'connection_test' && + (!connection || + this.modelFacts.fingerprintForConnection(facts!.document, connection) !== + basis.modelFactsFingerprint) + ) { + changed.push('connection'); + } + const effectiveConnection = + connection && basis.kind === 'connection_test' + ? applyModelFactOverridesToConnection(connection, facts!.document.overrides) + : connection; if ( - !connection || - connection.providerType !== basis.providerType || - !connection.enabled || - canonicalEffectiveEndpoint(connection) !== basis.effectiveEndpoint || + !effectiveConnection || + effectiveConnection.providerType !== basis.providerType || + !effectiveConnection.enabled || + canonicalEffectiveEndpoint(effectiveConnection) !== basis.effectiveEndpoint || (basis.kind === 'model_fetch' && - !sameStringArray(connection.enabledModelIds, basis.enabledModelIds)) || + !sameStringArray(effectiveConnection.enabledModelIds, basis.enabledModelIds)) || (basis.kind === 'connection_test' && - JSON.stringify(connection.requestBodyOverlay ?? {}) !== basis.requestBodyOverlayJson) || + JSON.stringify(effectiveConnection.requestBodyOverlay ?? {}) !== + basis.requestBodyOverlayJson) || (basis.kind === 'connection_test' && - !sameConnectionTestModelBasis(connectionTestModelBasis(connection), basis.model)) + !sameConnectionTestModelBasis(connectionTestModelBasis(connection!), basis.model)) ) { changed.push('connection'); } @@ -1619,6 +1675,81 @@ export class RuntimePolicyCoordinator { return operation(root); }); } + + private async projectCatalogSnapshot(root: string): Promise { + const facts = await this.readModelFacts(root); + const snapshot = catalogSnapshot(await this.catalog.read(root)); + return deepFreeze( + hideStaleModelFactsVerification( + applyModelFactOverridesToCatalogSnapshot(snapshot, facts.document.overrides), + snapshot, + facts.document, + this.modelFacts, + ), + ); + } + + private async readModelFacts(root: string) { + const facts = await this.modelFacts.readWithDiagnostics(root); + if (facts.diagnostic !== undefined && this.warnedModelFactsFingerprint !== facts.fingerprint) { + process.emitWarning(`model-facts.json is ${facts.diagnostic}; ignoring its overrides`, { + type: 'RuntimePolicyWarning', + }); + this.warnedModelFactsFingerprint = facts.fingerprint; + } + return facts; + } + + private async projectCatalogMutation( + root: string, + result: T, + ): Promise { + if (result.kind !== 'committed' || !('snapshot' in result)) return result; + return deepFreeze({ + ...result, + snapshot: await this.projectCatalogSnapshot(root), + }) as T; + } +} + +function hideStaleModelFactsVerification( + projected: ConnectionCatalogSnapshot, + persisted: ConnectionCatalogSnapshot, + document: ModelFactsDocument, + owner: ModelFactsDocumentOwner, +): ConnectionCatalogSnapshot { + const persistedById = new Map( + persisted.connections.map((connection) => [connection.connectionId, connection] as const), + ); + return { + ...projected, + connections: projected.connections.map((connection) => { + if (connection.lastTest === undefined) return connection; + const raw = persistedById.get(connection.connectionId); + if (!raw) return connection; + const current = owner.fingerprintForConnection(document, raw); + const emptyFactsFingerprint = owner.fingerprintForConnection( + { ...document, overrides: {} }, + raw, + ); + // Catalogs written before model facts existed have no marker. They remain + // valid until facts for this connection actually exist; every test + // recorded by this feature carries a connection-scoped marker and is + // checked exactly. + if ( + raw.lastTestModelFactsFingerprint === current || + (raw.lastTestModelFactsFingerprint === undefined && current === emptyFactsFingerprint) + ) { + return connection; + } + const { + lastTest: _lastTest, + lastTestModelFactsFingerprint: _lastTestModelFactsFingerprint, + ...withoutLastTest + } = connection; + return withoutLastTest; + }), + }; } function isCommitOutcomeUnknown(error: unknown): error is RuntimePolicyStoreError { @@ -1690,12 +1821,14 @@ function modelFetchSemanticBasis( function connectionTestSemanticBasis( prepared: PreparedConnectionMaterial, + modelFactsFingerprint: string, ): Extract { return { kind: 'connection_test', ...commonSemanticConnectionBasis(prepared), requestBodyOverlayJson: JSON.stringify(prepared.connection.requestBodyOverlay ?? {}), model: connectionTestModelBasis(prepared.connection), + modelFactsFingerprint, }; } diff --git a/packages/storage/src/runtime-policy/document-io.ts b/packages/storage/src/runtime-policy/document-io.ts index df7d3bc137..4bb161dd9b 100644 --- a/packages/storage/src/runtime-policy/document-io.ts +++ b/packages/storage/src/runtime-policy/document-io.ts @@ -35,7 +35,7 @@ export const VAULT_DOCUMENT_MAX_BYTES = 2 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const RUNTIME_POLICY_TEMP_PATTERN = - /^(?:runtime-policy|connection-catalog|credential-vault|runtime-policy-onboarding)\.json\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.tmp$/; + /^(?:runtime-policy|connection-catalog|credential-vault|runtime-policy-onboarding|model-facts)\.json\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.tmp$/; export async function cleanupRuntimePolicyDocumentTemps(root: string): Promise { let failure: unknown; @@ -84,6 +84,26 @@ export async function readBoundedJsonDocument( file: string, maxBytes: number, ): Promise { + const bytes = await readBoundedDocumentBytes(root, file, maxBytes); + if (bytes === undefined) return undefined; + let text: string; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch (error) { + throw invalidDocument(`${file} is not valid UTF-8`, error); + } + try { + return JSON.parse(text) as unknown; + } catch (error) { + throw invalidDocument(`${file} is not valid JSON`, error); + } +} + +export async function readBoundedDocumentBytes( + root: string, + file: string, + maxBytes: number, +): Promise { const path = join(root, file); const flags = process.platform === 'win32' @@ -100,7 +120,7 @@ export async function readBoundedJsonDocument( throw ioFailed(`${file} could not be opened`, error); } - let result: unknown | undefined; + let result: Buffer | undefined; let failure: unknown; try { const metadata = await handle.stat(); @@ -121,17 +141,7 @@ export async function readBoundedJsonDocument( } if (total > maxBytes) throw invalidDocument(`${file} exceeds its ${maxBytes} byte limit`); - let text: string; - try { - text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks, total)); - } catch (error) { - throw invalidDocument(`${file} is not valid UTF-8`, error); - } - try { - result = JSON.parse(text) as unknown; - } catch (error) { - throw invalidDocument(`${file} is not valid JSON`, error); - } + result = Buffer.concat(chunks, total); } catch (error) { failure = error; } finally { @@ -154,6 +164,7 @@ export async function writeJsonDocument( file: string, value: unknown, maxBytes: number, + synchronizeDirectory: (root: string) => Promise = syncDirectory, ): Promise { const bytes = serializeJsonDocument(value); if (bytes.length > maxBytes) throw invalidDocument(`${file} exceeds its ${maxBytes} byte limit`); @@ -173,7 +184,7 @@ export async function writeJsonDocument( handle = undefined; await rename(temporaryPath, path); published = true; - await syncDirectory(root); + await synchronizeDirectory(root); } catch (error) { failure = error; } finally { diff --git a/packages/storage/src/runtime-policy/errors.ts b/packages/storage/src/runtime-policy/errors.ts index 612737002f..e0ed7b39d9 100644 --- a/packages/storage/src/runtime-policy/errors.ts +++ b/packages/storage/src/runtime-policy/errors.ts @@ -24,6 +24,7 @@ export type RuntimePolicyStoreErrorCode = | 'invalid_policy_input' | 'invalid_connection_input' | 'invalid_credential_input' + | 'revision_conflict' | 'io_failed' | 'commit_outcome_unknown';