diff --git a/bindings/flutter/packages/runanywhere/lib/public/api/namespaces/llm.dart b/bindings/flutter/packages/runanywhere/lib/public/api/namespaces/llm.dart index 8603c40991..6103979529 100644 --- a/bindings/flutter/packages/runanywhere/lib/public/api/namespaces/llm.dart +++ b/bindings/flutter/packages/runanywhere/lib/public/api/namespaces/llm.dart @@ -115,57 +115,45 @@ class LlmApi { /// Generate a completion constrained to [schema]. /// /// [mode] picks how the schema is enforced: - /// - [StructuredOutputMode.validationOnly] (default): generate freely, then validate. + /// - [StructuredOutputMode.constrained] (default): engine-constrained decoding (GBNF grammar sampling). + /// - [StructuredOutputMode.validationOnly]: generate freely, then validate against the schema. /// - [StructuredOutputMode.repair]: validate, then retry once with a repair instruction if invalid. - /// - [StructuredOutputMode.constrained]: engine-constrained decoding — fails - /// preflight until a constrained-decoding engine is wired in. /// /// Throws [SDKException] when [mode] cannot be honored, generation fails, /// or the output cannot be parsed against the schema. Future generateStructured( String prompt, String schema, { - StructuredOutputMode mode = StructuredOutputMode.validationOnly, + StructuredOutputMode mode = StructuredOutputMode.constrained, LlmOptions? options, }) async { - if (mode == StructuredOutputMode.constrained) { - throw SDKException.featureNotAvailable( - 'llm.generateStructured(mode: constrained) needs engine-level ' - 'constrained decoding, which is not wired in yet; use validationOnly or repair', - ); - } await ModelGate.ensureLoaded( modelId: options?.model, category: ModelCategory.MODEL_CATEGORY_LANGUAGE, ); - final effective = options ?? LlmOptions(); - final model = - await ModelGate.currentId(ModelCategory.MODEL_CATEGORY_LANGUAGE) ?? ''; - var proto = await RunAnywhereStructuredOutput.generateStructured( + final effectiveMode = options?.structuredOutput?.mode ?? mode; + final effective = (options ?? LlmOptions()).copyWith( + structuredOutput: StructuredOutput( + schema: schema, + mode: effectiveMode, + strict: options?.structuredOutput?.strict ?? true, + ), + toolChoice: ToolChoice.none, + ); + final protoOptions = effective.toProto(); + final generation = + await RunAnywhereStructuredOutput.generateWithStructuredOutput( prompt: prompt, + structuredOutput: protoOptions.structuredOutput, + options: protoOptions, + ); + final proto = RunAnywhereLLM.shared.extractStructuredOutput( + text: generation.text, schema: schema, - options: effective.toProto( - registeredTools: RunAnywhereTools.shared.getRegisteredTools(), - ), ); - if (mode == StructuredOutputMode.repair && !proto.validation.isValid) { - final repairPrompt = - '$prompt\n\n' - 'Your previous answer did not match the required JSON schema. ' - 'Reply again with ONLY JSON that satisfies this schema.\n\n' - 'Previous invalid answer: ' - '${proto.hasRawText() ? proto.rawText : ''}'; - proto = await RunAnywhereStructuredOutput.generateStructured( - prompt: repairPrompt, - schema: schema, - options: effective.toProto( - registeredTools: RunAnywhereTools.shared.getRegisteredTools(), - ), - ); - } return StructuredResult.fromProto( proto, - GenerationResult(text: proto.hasRawText() ? proto.rawText : '', model: model), + GenerationResult.fromLlm(generation), ); } diff --git a/bindings/flutter/packages/runanywhere/lib/public/api/types/options.dart b/bindings/flutter/packages/runanywhere/lib/public/api/types/options.dart index 80f1c7813a..85f8f9660a 100644 --- a/bindings/flutter/packages/runanywhere/lib/public/api/types/options.dart +++ b/bindings/flutter/packages/runanywhere/lib/public/api/types/options.dart @@ -21,6 +21,8 @@ import 'package:runanywhere/generated/rag.pb.dart' show RAGConfiguration; import 'package:runanywhere/generated/segmentation.pb.dart' as seg_pb; import 'package:runanywhere/generated/structured_output.pb.dart' show StructuredOutputOptions; +import 'package:runanywhere/generated/structured_output.pbenum.dart' + as so_pb; import 'package:runanywhere/generated/stt_options.pb.dart' show STTOptions; import 'package:runanywhere/generated/thinking_tag_pattern.pb.dart' as think_pb; @@ -145,7 +147,11 @@ class ToolChoice { /// Schema constraint applied to a generation's output. class StructuredOutput { /// Constrain output to [schema]. - const StructuredOutput({required this.schema, this.strict = true}); + const StructuredOutput({ + required this.schema, + this.mode = StructuredOutputMode.constrained, + this.strict = true, + }); /// JSON Schema the output must satisfy, as raw JSON Schema text. /// @@ -157,6 +163,9 @@ class StructuredOutput { /// callers must already hold the schema as text. final String schema; + /// How the schema constraint is enforced during generation. + final StructuredOutputMode mode; + /// True rejects output that does not validate instead of repairing it. /// /// `StructuredOutputOptions.strict_mode` was deleted outright — this knob @@ -165,8 +174,24 @@ class StructuredOutput { final bool strict; /// Build the generated structured-output options. - StructuredOutputOptions toProto() => - StructuredOutputOptions(schema: schema, includeSchemaInPrompt: true); + StructuredOutputOptions toProto() { + final proto = StructuredOutputOptions( + schema: schema, + includeSchemaInPrompt: true, + ); + switch (mode) { + case StructuredOutputMode.constrained: + proto.mode = so_pb.StructuredOutputMode.STRUCTURED_OUTPUT_MODE_CONSTRAINED; + break; + case StructuredOutputMode.validationOnly: + proto.mode = so_pb.StructuredOutputMode.STRUCTURED_OUTPUT_MODE_VALIDATION_ONLY; + break; + case StructuredOutputMode.repair: + proto.mode = so_pb.StructuredOutputMode.STRUCTURED_OUTPUT_MODE_REPAIR; + break; + } + return proto; + } } /// Sampling, prompting, reasoning, and tool controls for `llm` and `vlm`. @@ -304,6 +329,47 @@ class LlmOptions { return proto; } + /// Create a copy with modified fields. + LlmOptions copyWith({ + String? model, + int? maxOutputTokens, + double? temperature, + double? topP, + int? topK, + double? minP, + double? frequencyPenalty, + double? presencePenalty, + double? repetitionPenalty, + int? seed, + List? stopSequences, + String? systemPrompt, + ReasoningOptions? reasoning, + StructuredOutput? structuredOutput, + List? tools, + ToolChoice? toolChoice, + int? maxToolCalls, + bool? autoExecute, + }) => LlmOptions( + model: model ?? this.model, + maxOutputTokens: maxOutputTokens ?? _argMaxOutputTokens, + temperature: temperature ?? _argTemperature, + topP: topP ?? _argTopP, + topK: topK ?? this.topK, + minP: minP ?? this.minP, + frequencyPenalty: frequencyPenalty ?? this.frequencyPenalty, + presencePenalty: presencePenalty ?? this.presencePenalty, + repetitionPenalty: repetitionPenalty ?? _argRepetitionPenalty, + seed: seed ?? this.seed, + stopSequences: stopSequences ?? this.stopSequences, + systemPrompt: systemPrompt ?? this.systemPrompt, + reasoning: reasoning ?? this.reasoning, + structuredOutput: structuredOutput ?? this.structuredOutput, + tools: tools ?? this.tools, + toolChoice: toolChoice ?? this.toolChoice, + maxToolCalls: maxToolCalls ?? _argMaxToolCalls, + autoExecute: autoExecute ?? this.autoExecute, + ); + /// Build the generated VLM request envelope for [prompt] over [images]. /// /// `VLMGenerationOptions` was deleted outright (idl/vlm_options.proto): @@ -840,7 +906,7 @@ class LoadOptions { /// Enforcement level `llm.generateStructured` applies to its schema. enum StructuredOutputMode { - /// Engine-constrained decoding; fails preflight until wired in. + /// Engine-constrained decoding (GBNF grammar sampling). constrained, /// Generate freely, then validate against the schema. diff --git a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/LlmNamespace.kt b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/LlmNamespace.kt index ac8363d7d0..53d23c7cfe 100644 --- a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/LlmNamespace.kt +++ b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/LlmNamespace.kt @@ -126,10 +126,9 @@ public class LlmNamespace internal constructor() { * Generate a completion constrained to [schema] and parse it. * * [mode] picks how the schema is enforced: - * - [StructuredOutputMode.VALIDATION_ONLY] (default): generate freely, then validate. + * - [StructuredOutputMode.CONSTRAINED] (default): engine-constrained decoding (GBNF grammar sampling). + * - [StructuredOutputMode.VALIDATION_ONLY]: generate freely, then validate against the schema. * - [StructuredOutputMode.REPAIR]: validate, then retry once with a repair instruction if invalid. - * - [StructuredOutputMode.CONSTRAINED]: engine-constrained decoding — fails preflight - * until a constrained-decoding engine is wired in. * * @throws SDKException when no language model can be loaded, [mode] cannot * be honored, or generation fails. @@ -137,31 +136,22 @@ public class LlmNamespace internal constructor() { public suspend fun generateStructured( prompt: String, schema: JsonSchema, - mode: StructuredOutputMode = StructuredOutputMode.VALIDATION_ONLY, + mode: StructuredOutputMode = StructuredOutputMode.CONSTRAINED, options: LlmOptions? = null, ): StructuredResult { - if (mode == StructuredOutputMode.CONSTRAINED) { - throw SDKException.unsupportedCapability( - "llm.generateStructured(mode = CONSTRAINED)", - "needs engine-level constrained decoding, which is not wired in yet; use VALIDATION_ONLY or REPAIR", - ) - } - val opts = options.orDefault() - val structuredOutput = StructuredOutput(schema = schema) - var generation = generateStructuredUnary(prompt, opts, structuredOutput) - var parsed = parseStructuredOutput(generation, structuredOutput) - - if (mode == StructuredOutputMode.REPAIR && parsed.validation?.is_valid != true) { - val repairPrompt = structuredRepairPrompt(prompt, generation.text, schema) - generation = generateStructuredUnary(repairPrompt, opts, structuredOutput) - parsed = parseStructuredOutput(generation, structuredOutput) - } + val opts = options.orDefault().copy(toolChoice = ToolChoice.None) + val effectiveSchema = opts.structuredOutput?.schema ?: schema + val effectiveMode = opts.structuredOutput?.mode ?: mode + val effectiveStrict = opts.structuredOutput?.strict ?: true + val structuredOutput = StructuredOutput(schema = effectiveSchema, strict = effectiveStrict, mode = effectiveMode) + val generation = generateStructuredUnary(prompt, opts, structuredOutput) + val parsed = parseStructuredOutput(generation, structuredOutput) return StructuredResult( value = parsed.json, raw = parsed.raw_text ?: generation.text, valid = parsed.validation?.is_valid == true, - mode = mode, + mode = effectiveMode, inputTokens = generation.inputTokens, outputTokens = generation.outputTokens, timeToFirstTokenMs = generation.timeToFirstTokenMs, @@ -262,17 +252,6 @@ public class LlmNamespace internal constructor() { private fun newRequestId(): String = UUID.randomUUID().toString() -/** The one retry prompt `generateStructured(mode = REPAIR)` sends when the first pass did not validate. */ -private fun structuredRepairPrompt(original: String, invalidOutput: String, schema: JsonSchema): String = - """ - $original - - Your previous answer did not match the required JSON schema. Reply again with ONLY JSON that satisfies this schema. - - Schema: ${schema.rawJson} - Previous invalid answer: $invalidOutput - """.trimIndent() - private fun LlmOptions.usesTools(): Boolean = toolChoice != ToolChoice.None /** diff --git a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/MappingOptions.kt b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/MappingOptions.kt index 6a09b9c746..277531eb2c 100644 --- a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/MappingOptions.kt +++ b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/MappingOptions.kt @@ -34,6 +34,7 @@ import ai.runanywhere.proto.v1.DiarizationOptions as ProtoDiarizationOptions import ai.runanywhere.proto.v1.ReasoningMode as ProtoReasoningMode import ai.runanywhere.proto.v1.ReasoningOptions as ProtoReasoningOptions import ai.runanywhere.proto.v1.SegmentationOptions as ProtoSegmentationOptions +import ai.runanywhere.proto.v1.StructuredOutputMode as ProtoStructuredOutputMode /** The IDL's neutral repetition penalty, used when the caller leaves it unset. */ private val NEUTRAL_REPETITION_PENALTY = LLMGenerationOptions.defaults().repeat_penalty @@ -41,8 +42,8 @@ private val NEUTRAL_REPETITION_PENALTY = LLMGenerationOptions.defaults().repeat_ internal fun LlmOptions?.orDefault(): LlmOptions = this ?: LlmOptions() /** - * [structuredOutput] is never populated from a public [LlmOptions] field — - * only `llm.generateStructured` supplies it, through its own internal call path. + * [structuredOutput] is supplied by `llm.generateStructured`, which merges the + * caller's [LlmOptions.structuredOutput] overrides with its schema. */ internal fun LlmOptions.toProto(structuredOutput: StructuredOutput? = null): LLMGenerationOptions = LLMGenerationOptions( @@ -112,17 +113,18 @@ internal fun ReasoningOptions.toProto(): ProtoReasoningOptions = ) /** - * `StructuredOutputOptions.strict_mode`/`.mode`/`.repair_json` are all - * deleted (idl/structured_output.proto): the message shrank to - * `include_schema_in_prompt` plus a `oneof constraint { schema | grammar | - * regex }`. [StructuredOutput.strict] has no wire home any more — retry - * behaviour for an invalid first pass is owned entirely by the Kotlin-side - * `llm.generateStructured(mode = REPAIR)` loop, not by a commons flag. + * Build the proto structured-output options for [StructuredOutput]. */ internal fun StructuredOutput.toProto(): StructuredOutputOptions = StructuredOutputOptions( schema = schema.rawJson, include_schema_in_prompt = true, + mode = + when (mode) { + StructuredOutputMode.CONSTRAINED -> ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_CONSTRAINED + StructuredOutputMode.VALIDATION_ONLY -> ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_VALIDATION_ONLY + StructuredOutputMode.REPAIR -> ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_REPAIR + }, ) internal fun SttOptions?.orDefault(): SttOptions = this ?: SttOptions() diff --git a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Options.kt b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Options.kt index d8b7d8f8a3..98640c71ab 100644 --- a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Options.kt +++ b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Options.kt @@ -56,14 +56,20 @@ public sealed class ToolChoice { } /** Schema the generated text must conform to. */ -public data class StructuredOutput( - val schema: JsonSchema, - val strict: Boolean = true, -) +public data class StructuredOutput + @JvmOverloads + constructor( + val schema: JsonSchema, + val strict: Boolean = true, + val mode: StructuredOutputMode = StructuredOutputMode.CONSTRAINED, + ) { + /** Backward compatibility constructor for callers passing (schema, mode). */ + public constructor(schema: JsonSchema, mode: StructuredOutputMode) : this(schema, strict = true, mode = mode) + } /** Enforcement level `llm.generateStructured` applies to its schema. */ public enum class StructuredOutputMode { - /** Engine-constrained decoding; fails preflight until a constrained-decoding engine is wired in. */ + /** Engine-constrained decoding (GBNF grammar sampling). */ CONSTRAINED, /** Generate freely, then validate against the schema. */ @@ -89,8 +95,6 @@ public data class LlmOptions( val stopSequences: List = emptyList(), val systemPrompt: String? = null, val reasoning: ReasoningOptions? = null, - // No structured-output schema field: `llm.generateStructured` owns - // schema configuration through its own `schema`/`mode` parameters. /** Empty uses the tools registered through `llm.tools`. */ val tools: List = emptyList(), val toolChoice: ToolChoice = ToolChoice.Auto, @@ -103,6 +107,13 @@ public data class LlmOptions( * `llm.generate`/`llm.generateStream` with inline `tools`. */ val autoExecute: Boolean = true, + /** + * Optional structured-output override for `llm.generateStructured`, which supplies the + * schema and uses this field's `mode` in preference to its own `mode` argument. + * `strict` is not forwarded: `StructuredOutputOptions` has no strict field on the wire. + * Declared last so existing positional arguments and `componentN()` keep their slots. + */ + val structuredOutput: StructuredOutput? = null, ) { /** Sampling and tool-loop defaults, as annotated in idl/. */ public companion object { diff --git a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Results.kt b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Results.kt index e218a382c4..d2cca7e85f 100644 --- a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Results.kt +++ b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Results.kt @@ -50,7 +50,7 @@ public data class StructuredResult( val value: String, val raw: String, val valid: Boolean, - val mode: StructuredOutputMode = StructuredOutputMode.VALIDATION_ONLY, + val mode: StructuredOutputMode = StructuredOutputMode.CONSTRAINED, val inputTokens: Int = 0, val outputTokens: Int = 0, val timeToFirstTokenMs: Long = 0L, diff --git a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/extensions/LLM/StructuredOutputProtoHelpers.kt b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/extensions/LLM/StructuredOutputProtoHelpers.kt index dd50fa586c..29d41fd94a 100644 --- a/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/extensions/LLM/StructuredOutputProtoHelpers.kt +++ b/bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/extensions/LLM/StructuredOutputProtoHelpers.kt @@ -15,6 +15,7 @@ package com.runanywhere.sdk.public.extensions +import ai.runanywhere.proto.v1.StructuredOutputMode import ai.runanywhere.proto.v1.StructuredOutputOptions import com.runanywhere.sdk.public.api.JsonSchema import com.runanywhere.sdk.public.types.RAStructuredOutputResult @@ -23,20 +24,17 @@ import com.runanywhere.sdk.public.types.RAStructuredOutputResult /** * Default structured-output options mirroring Swift - * `RAStructuredOutputOptions.defaults(schema:includeSchemaInPrompt:strict:)`. - * - * `strict`/`repair` have no wire home any more (`strict_mode`/`mode`/ - * `repair_json` were all deleted from `StructuredOutputOptions`): retry - * behaviour for an invalid first pass is owned entirely by the Kotlin-side - * `llm.generateStructured(mode = REPAIR)` loop, not a commons flag. + * `RAStructuredOutputOptions.defaults(schema:includeSchemaInPrompt:mode:)`. */ fun StructuredOutputOptions.Companion.defaults( schema: JsonSchema, includeSchemaInPrompt: Boolean = true, + mode: StructuredOutputMode = StructuredOutputMode.STRUCTURED_OUTPUT_MODE_UNSPECIFIED, ): StructuredOutputOptions = StructuredOutputOptions( include_schema_in_prompt = includeSchemaInPrompt, schema = schema.rawJson, + mode = mode, ) // MARK: - StructuredOutputResult diff --git a/bindings/python/runanywhere/_options_bridge.py b/bindings/python/runanywhere/_options_bridge.py index f533ec47c5..94478e0127 100644 --- a/bindings/python/runanywhere/_options_bridge.py +++ b/bindings/python/runanywhere/_options_bridge.py @@ -21,6 +21,7 @@ PoolingMode, ReasoningMode, SttOptions, + StructuredOutputMode, TtsOptions, ) @@ -93,6 +94,11 @@ def llm_kwargs(options: Optional[LlmOptions], *, vlm: bool = False) -> dict: "structured_output on vlm: the bridge's generate_vlm() has no " "StructuredOutputOptions parameter" ) + if structured.mode not in (None, StructuredOutputMode.CONSTRAINED): + raise SDKException.not_implemented( + f"structured_output.mode={structured.mode}: only constrained decoding " + "(StructuredOutputMode.CONSTRAINED) is exposed on this bridge" + ) if not structured.strict: raise SDKException.not_implemented( "structured_output.strict=False: only constrained decoding " diff --git a/bindings/python/runanywhere/api/llm.py b/bindings/python/runanywhere/api/llm.py index f786ef230a..7c65794606 100644 --- a/bindings/python/runanywhere/api/llm.py +++ b/bindings/python/runanywhere/api/llm.py @@ -12,7 +12,7 @@ from ..errors import SDKException from ..events import GenerationEvent, GenerationEventKind from ..inputs import JsonSchema, ModelCategory, ToolDefinition -from ..options import LlmOptions, StructuredOutput, ToolChoice, ToolChoiceMode +from ..options import LlmOptions, StructuredOutput, StructuredOutputMode, ToolChoice, ToolChoiceMode from ..results import FinishReason, GenerationResult, StructuredResult, ToolCall from ..structured import ToolSpec, parse_structured, tool_call_prompt, tool_call_schema from ._common import Prompt, prepare @@ -318,6 +318,7 @@ def _structured(result: GenerationResult) -> StructuredResult: value=value, raw=result.text, valid=valid, + mode=StructuredOutputMode.CONSTRAINED, input_tokens=result.input_tokens, output_tokens=result.output_tokens, time_to_first_token_ms=result.time_to_first_token_ms, diff --git a/bindings/python/runanywhere/options.py b/bindings/python/runanywhere/options.py index cf9471fef2..6827d144bc 100644 --- a/bindings/python/runanywhere/options.py +++ b/bindings/python/runanywhere/options.py @@ -78,12 +78,17 @@ class StructuredOutput: schema: JsonSchema strict: bool = True + mode: "StructuredOutputMode" = None # type: ignore[assignment] + + def __post_init__(self) -> None: + if self.mode is None: + self.mode = StructuredOutputMode.CONSTRAINED class StructuredOutputMode(IntEnum): """Enforcement level ``llm.generate_structured`` applies to its schema.""" - #: Engine-constrained decoding; fails preflight until wired in. + #: Engine-constrained decoding (GBNF grammar sampling). CONSTRAINED = 0 #: Generate freely, then validate against the schema. VALIDATION_ONLY = 1 diff --git a/bindings/react-native/packages/core/src/Public/Api/Llm.ts b/bindings/react-native/packages/core/src/Public/Api/Llm.ts index 1dd9d5137a..dfebf018b8 100644 --- a/bindings/react-native/packages/core/src/Public/Api/Llm.ts +++ b/bindings/react-native/packages/core/src/Public/Api/Llm.ts @@ -338,10 +338,9 @@ export const llm = { * Generate a value that conforms to `schema`. * * `mode` picks how the schema is enforced: - * - `'validationOnly'` (default): generate freely, then validate. + * - `'constrained'` (default): engine-constrained decoding (GBNF grammar sampling). + * - `'validationOnly'`: generate freely, then validate against the schema. * - `'repair'`: validate, then retry once with a repair instruction if invalid. - * - `'constrained'`: engine-constrained decoding — fails preflight until a - * constrained-decoding engine is wired in. * * `StructuredOutputRequest` and the standalone * `rac_structured_output_generate_proto` ABI are deleted outright: idl's @@ -359,14 +358,8 @@ export const llm = { prompt: string, schema: JsonSchema, options?: LlmOptions, - mode: StructuredOutputMode = 'validationOnly' + mode: StructuredOutputMode = 'constrained' ): Promise> { - if (mode === 'constrained') { - throw SDKException.notImplemented( - "llm.generateStructured(mode: 'constrained') needs engine-level constrained decoding, " + - "which is not wired in yet; use 'validationOnly' or 'repair'" - ); - } const native = await preflight(); const requestId = nextRequestId('structured'); if (options?.model) { @@ -375,9 +368,15 @@ export const llm = { ModelCategory.MODEL_CATEGORY_LANGUAGE ); } + const effectiveMode = options?.structuredOutput?.mode ?? mode; const effectiveOptions: LlmOptions = { ...options, - structuredOutput: { schema, strict: options?.structuredOutput?.strict }, + structuredOutput: { + schema, + mode: effectiveMode, + strict: options?.structuredOutput?.strict, + }, + toolChoice: 'none', }; const extract = async (text: string): Promise => { @@ -395,28 +394,16 @@ export const llm = { return decode(resultBytes, StructuredOutputResult, 'structuredOutputParse'); }; - const runOnce = async (text: string): Promise => { - const request = await buildRequest(text, effectiveOptions, requestId, []); - const resultBytes = await native.llmGenerateProto( - encode(request, LLMGenerateRequest) - ); - return toGenerationResult( - decode(resultBytes, LLMGenerationResult, 'llmGenerate'), - requestId - ); - }; - - let generation = await runOnce(prompt); - let result = await extract(generation.text); - if (mode === 'repair' && !(result.validation?.isValid ?? true)) { - const repairPrompt = - `${prompt}\n\nYour previous answer did not match the required JSON schema. ` + - 'Reply again with ONLY JSON that satisfies this schema.\n\n' + - `Previous invalid answer: ${result.rawText ?? generation.text}`; - generation = await runOnce(repairPrompt); - result = await extract(generation.text); - } - return toStructuredResult(result, generation, mode); + const request = await buildRequest(prompt, effectiveOptions, requestId, []); + const resultBytes = await native.llmGenerateProto( + encode(request, LLMGenerateRequest) + ); + const generation = toGenerationResult( + decode(resultBytes, LLMGenerationResult, 'llmGenerate'), + requestId + ); + const result = await extract(generation.text); + return toStructuredResult(result, generation, effectiveMode); }, /** Tools the model may call during generation. */ diff --git a/bindings/react-native/packages/core/src/Public/Api/Options.ts b/bindings/react-native/packages/core/src/Public/Api/Options.ts index 918b5cb587..57582722a0 100644 --- a/bindings/react-native/packages/core/src/Public/Api/Options.ts +++ b/bindings/react-native/packages/core/src/Public/Api/Options.ts @@ -15,7 +15,10 @@ import { ReasoningOptions as ReasoningOptionsMessage, ThinkingTagPattern, } from '@runanywhere/proto-ts/thinking_tag_pattern'; -import { StructuredOutputOptions } from '@runanywhere/proto-ts/structured_output'; +import { + StructuredOutputMode as ProtoStructuredOutputMode, + StructuredOutputOptions, +} from '@runanywhere/proto-ts/structured_output'; import { ToolCallingOptions, ToolChoiceMode, @@ -121,18 +124,21 @@ function toReasoningOptions( }); } -/** - * `strict_mode` was deleted from `StructuredOutputOptions` outright (idl - * comment: "Not read by commons"). `structured.strict` has no wire - * counterpart to carry it in; only `schema` and `includeSchemaInPrompt` - * survive. - */ function toStructuredOutputOptions( structured: StructuredOutput ): StructuredOutputOptions { + let mode: ProtoStructuredOutputMode | undefined; + if (structured.mode === 'constrained') { + mode = ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_CONSTRAINED; + } else if (structured.mode === 'validationOnly') { + mode = ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_VALIDATION_ONLY; + } else if (structured.mode === 'repair') { + mode = ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_REPAIR; + } return StructuredOutputOptions.fromPartial({ schema: structured.schema, includeSchemaInPrompt: true, + ...(mode !== undefined ? { mode } : {}), }); } diff --git a/bindings/react-native/packages/core/src/Public/Api/Types.ts b/bindings/react-native/packages/core/src/Public/Api/Types.ts index 7da3d43c52..51c330b52b 100644 --- a/bindings/react-native/packages/core/src/Public/Api/Types.ts +++ b/bindings/react-native/packages/core/src/Public/Api/Types.ts @@ -134,6 +134,7 @@ export interface ReasoningOptions { /** Schema-constrained output for a generation. */ export interface StructuredOutput { schema: JsonSchema; + mode?: StructuredOutputMode; strict?: boolean; } diff --git a/bindings/swift/Sources/RunAnywhere/Public/API/Namespaces/LLMNamespace.swift b/bindings/swift/Sources/RunAnywhere/Public/API/Namespaces/LLMNamespace.swift index d2f08875c9..12d0a0fbd1 100644 --- a/bindings/swift/Sources/RunAnywhere/Public/API/Namespaces/LLMNamespace.swift +++ b/bindings/swift/Sources/RunAnywhere/Public/API/Namespaces/LLMNamespace.swift @@ -77,44 +77,24 @@ public extension RunAnywhere { /// Generate output that satisfies `schema`. /// /// `mode` picks how the schema is enforced: - /// - `.validationOnly` (default): generate freely, then validate. + /// - `.constrained` (default): engine-constrained decoding (GBNF grammar sampling). + /// - `.validationOnly`: generate freely, then validate against the schema. /// - `.repair`: validate, then retry once with a repair instruction if invalid. - /// - `.constrained`: engine-constrained decoding — fails preflight - /// until a constrained-decoding engine is wired in. /// /// - Throws: `SDKException` when no model can be loaded, `mode` cannot /// be honored, generation fails, or the output cannot be parsed. public func generateStructured( prompt: String, schema: JsonSchema, - mode: StructuredEnforcementMode = .validationOnly, + mode: StructuredEnforcementMode = .constrained, options: LlmOptions? = nil ) async throws -> StructuredResult { - guard mode != .constrained else { - throw SDKException( - code: .notSupported, - message: "generateStructured(mode: .constrained) needs engine-level constrained decoding, " + - "which is not wired in yet; use .validationOnly or .repair", - category: .validation - ) - } - var effective = options ?? LlmOptions() - effective.structuredOutput = StructuredOutput(schema: schema) + effective.structuredOutput = StructuredOutput(schema: schema, strict: effective.structuredOutput?.strict ?? true, mode: mode) + effective.toolChoice = .none - var generation = try await generate(prompt: prompt, history: [], options: effective) - var parsed = try RunAnywhere.parseStructuredOutput(text: generation.text, schema: schema) - - let isValid = parsed.hasValidation ? parsed.validation.isValid : false - if mode == .repair, !isValid { - let repairPrompt = RunAnywhere.structuredRepairPrompt( - original: prompt, - invalidOutput: generation.text, - schema: schema - ) - generation = try await generate(prompt: repairPrompt, history: [], options: effective) - parsed = try RunAnywhere.parseStructuredOutput(text: generation.text, schema: schema) - } + let generation = try await generate(prompt: prompt, history: [], options: effective) + let parsed = try RunAnywhere.parseStructuredOutput(text: generation.text, schema: schema) return StructuredResult(proto: parsed, generation: generation, mode: mode) } @@ -449,21 +429,4 @@ extension RunAnywhere { CppBridge.StructuredOutput.makeParseRequest(text: text, schema: schema) ) } - - /// Build the one retry prompt `generateStructured(mode: .repair)` sends - /// when the first pass did not validate against `schema`. - internal static func structuredRepairPrompt( - original: String, - invalidOutput: String, - schema: JsonSchema - ) -> String { - """ - \(original) - - Your previous answer did not match the required JSON schema. Reply again with ONLY JSON that satisfies this schema. - - Schema: \(schema) - Previous invalid answer: \(invalidOutput) - """ - } } diff --git a/bindings/swift/Sources/RunAnywhere/Public/API/Options.swift b/bindings/swift/Sources/RunAnywhere/Public/API/Options.swift index 3f86b40917..302f5a01fc 100644 --- a/bindings/swift/Sources/RunAnywhere/Public/API/Options.swift +++ b/bindings/swift/Sources/RunAnywhere/Public/API/Options.swift @@ -76,11 +76,14 @@ public struct ReasoningOptions: Sendable { // MARK: - StructuredOutput -/// Force generation to satisfy a JSON schema. +/// Schema constraints enforced on generation according to the specified mode. public struct StructuredOutput: Sendable { /// Schema the output must validate against, as raw JSON Schema text. public var schema: JsonSchema + /// How the schema constraint is enforced during generation. + public var mode: StructuredEnforcementMode = .constrained + /// Reject output that does not validate instead of returning it raw. /// /// `RAStructuredOutputOptions.strictMode` was deleted outright @@ -92,13 +95,27 @@ public struct StructuredOutput: Sendable { public var strict: Bool = true /// Build a structured-output constraint. - public init(schema: JsonSchema, strict: Bool = true) { + public init( + schema: JsonSchema, + strict: Bool = true, + mode: StructuredEnforcementMode = .constrained + ) { self.schema = schema self.strict = strict + self.mode = mode } func toProto() -> RAStructuredOutputOptions { - RAStructuredOutputOptions.defaults(schema: schema, includeSchemaInPrompt: true) + var proto = RAStructuredOutputOptions.defaults(schema: schema, includeSchemaInPrompt: true) + switch mode { + case .constrained: + proto.mode = .constrained + case .validationOnly: + proto.mode = .validationOnly + case .repair: + proto.mode = .repair + } + return proto } } diff --git a/bindings/swift/Sources/RunAnywhere/Public/Extensions/LLM/StructuredOutputProto+Helpers.swift b/bindings/swift/Sources/RunAnywhere/Public/Extensions/LLM/StructuredOutputProto+Helpers.swift index eba0649c41..880d0530e0 100644 --- a/bindings/swift/Sources/RunAnywhere/Public/Extensions/LLM/StructuredOutputProto+Helpers.swift +++ b/bindings/swift/Sources/RunAnywhere/Public/Extensions/LLM/StructuredOutputProto+Helpers.swift @@ -11,18 +11,18 @@ import Foundation // MARK: - RAStructuredOutputOptions extension RAStructuredOutputOptions { - /// `RAJSONSchema`/`strictMode`/`jsonSchema`/`mode` were all deleted - /// outright (idl/structured_output.proto): the message shrunk to just - /// `includeSchemaInPrompt` plus a `schema`/`grammar`/`regex` oneof of - /// raw strings, so `schema` here is JSON Schema text directly rather - /// than a typed tree to serialize. + /// Ergonomic factory for `RAStructuredOutputOptions`. public static func defaults( schema: String, - includeSchemaInPrompt: Bool = true + includeSchemaInPrompt: Bool = true, + mode: RAStructuredOutputMode = .unspecified ) -> RAStructuredOutputOptions { var options = RAStructuredOutputOptions() options.schema = schema options.includeSchemaInPrompt = includeSchemaInPrompt + if mode != .unspecified { + options.mode = mode + } return options } } diff --git a/bindings/web/packages/core/src/Public/API/Mapping.ts b/bindings/web/packages/core/src/Public/API/Mapping.ts index 8b11b743b4..7e0eac0d7d 100644 --- a/bindings/web/packages/core/src/Public/API/Mapping.ts +++ b/bindings/web/packages/core/src/Public/API/Mapping.ts @@ -13,7 +13,11 @@ import { import { FinishReason as ProtoFinishReason } from '@runanywhere/proto-ts/finish_reason'; import { ReasoningMode } from '@runanywhere/proto-ts/thinking_tag_pattern'; import { ToolChoiceMode } from '@runanywhere/proto-ts/tool_calling'; -import type { StructuredOutputOptions, StructuredOutputResult } from '@runanywhere/proto-ts/structured_output'; +import { + StructuredOutputMode as ProtoStructuredOutputMode, + type StructuredOutputOptions, + type StructuredOutputResult, +} from '@runanywhere/proto-ts/structured_output'; import type { STTOptions, STTOutput } from '@runanywhere/proto-ts/stt_options'; import type { TTSOptions, TTSOutput, TTSVoiceInfo } from '@runanywhere/proto-ts/tts_options'; import type { VADOptions, VADResult as ProtoVadResult } from '@runanywhere/proto-ts/vad_options'; @@ -306,21 +310,24 @@ export function vlmToGenerationResult(result: VLMResult, requestId = ''): Genera /** * Build the proto structured-output request options for one - * `llm.generateStructured` call. `CONSTRAINED` is not wired on Web (no - * grammar-constrained decoding hook) — callers must preflight-reject it - * before reaching this helper. `StructuredOutputOptions` no longer carries a - * `mode`/`strictMode`/`repairJson` knob: it is just - * `includeSchemaInPrompt`/`schema`/`grammar`/`regex` now, so `mode` is kept - * only as the caller-facing enforcement level threaded through to - * `toStructuredResult` — commons always validates+extracts on the result. + * `llm.generateStructured` call. */ export function toProtoStructuredOutputOptions( json: string, - _mode: Exclude, + mode?: StructuredOutputMode, ): StructuredOutputOptions { + let protoMode: ProtoStructuredOutputMode | undefined; + if (mode === 'constrained') { + protoMode = ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_CONSTRAINED; + } else if (mode === 'validationOnly') { + protoMode = ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_VALIDATION_ONLY; + } else if (mode === 'repair') { + protoMode = ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_REPAIR; + } return { includeSchemaInPrompt: true, schema: json, + ...(protoMode !== undefined ? { mode: protoMode } : {}), }; } diff --git a/bindings/web/packages/core/src/Public/API/Namespaces/llm.ts b/bindings/web/packages/core/src/Public/API/Namespaces/llm.ts index 7f35270f64..176aa9b409 100644 --- a/bindings/web/packages/core/src/Public/API/Namespaces/llm.ts +++ b/bindings/web/packages/core/src/Public/API/Namespaces/llm.ts @@ -289,25 +289,20 @@ export const llm = { * Generate JSON that satisfies a schema, returning the parsed value with * the raw text alongside it. * - * @param mode `'validationOnly'` (default) parses and validates after - * generation; `'repair'` additionally attempts to fix malformed JSON. - * `'constrained'` fails preflight — Web has no grammar-constrained - * decoding hook. - * @throws SDKException when the backend cannot parse structured output, - * or `mode` is `'constrained'`. + * @param mode `'constrained'` (default) asks commons to compile the schema + * into a grammar for constrained decoding. Only grammar-capable engines + * enforce it; other Web backends generate freely and rely on validation. + * `'validationOnly'` generates freely then validates. `'repair'` validates + * and, when the first response is invalid, parses it and retries once with + * a repair instruction. + * @throws SDKException when the backend cannot parse structured output. */ async generateStructured( prompt: string, schema: JsonSchema, - mode: StructuredOutputMode = 'validationOnly', + mode: StructuredOutputMode = 'constrained', options?: LlmOptions, ): Promise { - if (mode === 'constrained') { - throw SDKException.unsupportedCapability( - "llm.generateStructured(mode: 'constrained')", - 'The Web SDK has no grammar-constrained decoding hook; use "validationOnly" or "repair".', - ); - } const structuredOutput = toProtoStructuredOutputOptions(schema.json, mode); const generation = await generateCore(prompt, options, structuredOutput); const adapter = StructuredOutputProtoAdapter.tryDefault(); diff --git a/core/src/features/llm/structured_output.cpp b/core/src/features/llm/structured_output.cpp index 9ea4856611..595e8adff9 100644 --- a/core/src/features/llm/structured_output.cpp +++ b/core/src/features/llm/structured_output.cpp @@ -600,20 +600,13 @@ extern "C" rac_result_t rac_structured_output_extract_json(const char* text, cha // Small base models (e.g. SmolLM2-360M) // cannot reliably produce structured JSON from a free-form prompt because -// they have not been instruction-tuned on the "schema → JSON" task and the -// llama.cpp backend currently does not pipe a grammar/json-mode parameter -// through to the sampler. The system prompt below was hardened with an -// explicit start-token cue ("Your reply must begin with %s ...") so the -// model is more likely to emit JSON on the very first token instead of -// echoing the prompt. The proper fix is grammar-constrained decoding: -// -// TODO(cluster-12 follow-up): plumb rac_llm_options_t.grammar (or a -// schema-derived GBNF grammar) through rac_llm_llamacpp_generate and -// wire it to llama_sampler_init_grammar(). Until that lands, base -// models will produce best-effort (sometimes lenient) JSON output — -// tests should accept either a successful JSON parse or, for known -// base models, a prompt-echo PASS as documented in -// cross-platform-e2e-test-catalog.md. +// they have not been instruction-tuned on the "schema → JSON" task. +// The system prompt below is hardened with an explicit start-token cue +// ("Your reply must begin with %s ...") so the model is more likely to +// emit JSON on the very first token instead of echoing the prompt. +// On the ordinary LLM generate path with the llama.cpp backend, schema-derived +// GBNF grammar is also piped through to the sampler when constrained decoding +// is enabled. extern "C" rac_result_t rac_structured_output_get_system_prompt(const char* json_schema, char** out_prompt) {