fix(llm): map StructuredOutputMode across SDKs and update docs (#917)

bindings/flutter/packages/runanywhere/lib/public/api/namespaces/llm.dart

@@ -115,57 +115,45 @@ class LlmApi {
   /// Generate a completion constrained to [schema].
   ///
   /// [mode] picks how the schema is enforced:
-  /// - [StructuredOutputMode.validationOnly] (default): generate freely, then validate.
+  /// - [StructuredOutputMode.constrained] (default): engine-constrained decoding (GBNF grammar sampling).
+  /// - [StructuredOutputMode.validationOnly]: generate freely, then validate against the schema.
   /// - [StructuredOutputMode.repair]: validate, then retry once with a repair instruction if invalid.
-  /// - [StructuredOutputMode.constrained]: engine-constrained decoding — fails
-  ///   preflight until a constrained-decoding engine is wired in.
   ///
   /// Throws [SDKException] when [mode] cannot be honored, generation fails,
   /// or the output cannot be parsed against the schema.
   Future<StructuredResult> generateStructured(
     String prompt,
     String schema, {
-    StructuredOutputMode mode = StructuredOutputMode.validationOnly,
+    StructuredOutputMode mode = StructuredOutputMode.constrained,
     LlmOptions? options,
   }) async {
-    if (mode == StructuredOutputMode.constrained) {
-      throw SDKException.featureNotAvailable(
-        'llm.generateStructured(mode: constrained) needs engine-level '
-        'constrained decoding, which is not wired in yet; use validationOnly or repair',
-      );
-    }
     await ModelGate.ensureLoaded(
       modelId: options?.model,
       category: ModelCategory.MODEL_CATEGORY_LANGUAGE,
     );
-    final effective = options ?? LlmOptions();
-    final model =
-        await ModelGate.currentId(ModelCategory.MODEL_CATEGORY_LANGUAGE) ?? '';
-    var proto = await RunAnywhereStructuredOutput.generateStructured(
+    final effectiveMode = options?.structuredOutput?.mode ?? mode;
+    final effective = (options ?? LlmOptions()).copyWith(
+      structuredOutput: StructuredOutput(
+        schema: schema,
+        mode: effectiveMode,
+        strict: options?.structuredOutput?.strict ?? true,
+      ),
+      toolChoice: ToolChoice.none,
+    );
+    final protoOptions = effective.toProto();
+    final generation =
+        await RunAnywhereStructuredOutput.generateWithStructuredOutput(
       prompt: prompt,
+      structuredOutput: protoOptions.structuredOutput,
+      options: protoOptions,
+    );
+    final proto = RunAnywhereLLM.shared.extractStructuredOutput(
+      text: generation.text,
       schema: schema,
-      options: effective.toProto(
-        registeredTools: RunAnywhereTools.shared.getRegisteredTools(),
-      ),
     );
-    if (mode == StructuredOutputMode.repair && !proto.validation.isValid) {
-      final repairPrompt =
-          '$prompt\n\n'
-          'Your previous answer did not match the required JSON schema. '
-          'Reply again with ONLY JSON that satisfies this schema.\n\n'
-          'Previous invalid answer: '
-          '${proto.hasRawText() ? proto.rawText : ''}';
-      proto = await RunAnywhereStructuredOutput.generateStructured(
-        prompt: repairPrompt,
-        schema: schema,
-        options: effective.toProto(
-          registeredTools: RunAnywhereTools.shared.getRegisteredTools(),
-        ),
-      );
-    }
     return StructuredResult.fromProto(
       proto,
-      GenerationResult(text: proto.hasRawText() ? proto.rawText : '', model: model),
+      GenerationResult.fromLlm(generation),
     );
   }
 

bindings/flutter/packages/runanywhere/lib/public/api/types/options.dart

@@ -21,6 +21,8 @@ import 'package:runanywhere/generated/rag.pb.dart' show RAGConfiguration;
 import 'package:runanywhere/generated/segmentation.pb.dart' as seg_pb;
 import 'package:runanywhere/generated/structured_output.pb.dart'
     show StructuredOutputOptions;
+import 'package:runanywhere/generated/structured_output.pbenum.dart'
+    as so_pb;
 import 'package:runanywhere/generated/stt_options.pb.dart' show STTOptions;
 import 'package:runanywhere/generated/thinking_tag_pattern.pb.dart'
     as think_pb;
@@ -145,7 +147,11 @@ class ToolChoice {
 /// Schema constraint applied to a generation's output.
 class StructuredOutput {
   /// Constrain output to [schema].
-  const StructuredOutput({required this.schema, this.strict = true});
+  const StructuredOutput({
+    required this.schema,
+    this.mode = StructuredOutputMode.constrained,
+    this.strict = true,
+  });
 
   /// JSON Schema the output must satisfy, as raw JSON Schema text.
   ///
@@ -157,6 +163,9 @@ class StructuredOutput {
   /// callers must already hold the schema as text.
   final String schema;
 
+  /// How the schema constraint is enforced during generation.
+  final StructuredOutputMode mode;
+
   /// True rejects output that does not validate instead of repairing it.
   ///
   /// `StructuredOutputOptions.strict_mode` was deleted outright — this knob
@@ -165,8 +174,24 @@ class StructuredOutput {
   final bool strict;
 
   /// Build the generated structured-output options.
-  StructuredOutputOptions toProto() =>
-      StructuredOutputOptions(schema: schema, includeSchemaInPrompt: true);
+  StructuredOutputOptions toProto() {
+    final proto = StructuredOutputOptions(
+      schema: schema,
+      includeSchemaInPrompt: true,
+    );
+    switch (mode) {
+      case StructuredOutputMode.constrained:
+        proto.mode = so_pb.StructuredOutputMode.STRUCTURED_OUTPUT_MODE_CONSTRAINED;
+        break;
+      case StructuredOutputMode.validationOnly:
+        proto.mode = so_pb.StructuredOutputMode.STRUCTURED_OUTPUT_MODE_VALIDATION_ONLY;
+        break;
+      case StructuredOutputMode.repair:
+        proto.mode = so_pb.StructuredOutputMode.STRUCTURED_OUTPUT_MODE_REPAIR;
+        break;
+    }
+    return proto;
+  }
 }
 
 /// Sampling, prompting, reasoning, and tool controls for `llm` and `vlm`.
@@ -304,6 +329,47 @@ class LlmOptions {
     return proto;
   }
 
+  /// Create a copy with modified fields.
+  LlmOptions copyWith({
+    String? model,
+    int? maxOutputTokens,
+    double? temperature,
+    double? topP,
+    int? topK,
+    double? minP,
+    double? frequencyPenalty,
+    double? presencePenalty,
+    double? repetitionPenalty,
+    int? seed,
+    List<String>? stopSequences,
+    String? systemPrompt,
+    ReasoningOptions? reasoning,
+    StructuredOutput? structuredOutput,
+    List<ToolDefinition>? tools,
+    ToolChoice? toolChoice,
+    int? maxToolCalls,
+    bool? autoExecute,
+  }) => LlmOptions(
+    model: model ?? this.model,
+    maxOutputTokens: maxOutputTokens ?? _argMaxOutputTokens,
+    temperature: temperature ?? _argTemperature,
+    topP: topP ?? _argTopP,
+    topK: topK ?? this.topK,
+    minP: minP ?? this.minP,
+    frequencyPenalty: frequencyPenalty ?? this.frequencyPenalty,
+    presencePenalty: presencePenalty ?? this.presencePenalty,
+    repetitionPenalty: repetitionPenalty ?? _argRepetitionPenalty,
+    seed: seed ?? this.seed,
+    stopSequences: stopSequences ?? this.stopSequences,
+    systemPrompt: systemPrompt ?? this.systemPrompt,
+    reasoning: reasoning ?? this.reasoning,
+    structuredOutput: structuredOutput ?? this.structuredOutput,
+    tools: tools ?? this.tools,
+    toolChoice: toolChoice ?? this.toolChoice,
+    maxToolCalls: maxToolCalls ?? _argMaxToolCalls,
+    autoExecute: autoExecute ?? this.autoExecute,
+  );
+
   /// Build the generated VLM request envelope for [prompt] over [images].
   ///
   /// `VLMGenerationOptions` was deleted outright (idl/vlm_options.proto):
@@ -840,7 +906,7 @@ class LoadOptions {
 
 /// Enforcement level `llm.generateStructured` applies to its schema.
 enum StructuredOutputMode {
-  /// Engine-constrained decoding; fails preflight until wired in.
+  /// Engine-constrained decoding (GBNF grammar sampling).
   constrained,
 
   /// Generate freely, then validate against the schema.

bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/LlmNamespace.kt

@@ -126,42 +126,32 @@ public class LlmNamespace internal constructor() {
      * Generate a completion constrained to [schema] and parse it.
      *
      * [mode] picks how the schema is enforced:
-     * - [StructuredOutputMode.VALIDATION_ONLY] (default): generate freely, then validate.
+     * - [StructuredOutputMode.CONSTRAINED] (default): engine-constrained decoding (GBNF grammar sampling).
+     * - [StructuredOutputMode.VALIDATION_ONLY]: generate freely, then validate against the schema.
      * - [StructuredOutputMode.REPAIR]: validate, then retry once with a repair instruction if invalid.
-     * - [StructuredOutputMode.CONSTRAINED]: engine-constrained decoding — fails preflight
-     *   until a constrained-decoding engine is wired in.
      *
      * @throws SDKException when no language model can be loaded, [mode] cannot
      *   be honored, or generation fails.
      */
     public suspend fun generateStructured(
         prompt: String,
         schema: JsonSchema,
-        mode: StructuredOutputMode = StructuredOutputMode.VALIDATION_ONLY,
+        mode: StructuredOutputMode = StructuredOutputMode.CONSTRAINED,
         options: LlmOptions? = null,
     ): StructuredResult {
-        if (mode == StructuredOutputMode.CONSTRAINED) {
-            throw SDKException.unsupportedCapability(
-                "llm.generateStructured(mode = CONSTRAINED)",
-                "needs engine-level constrained decoding, which is not wired in yet; use VALIDATION_ONLY or REPAIR",
-            )
-        }
-        val opts = options.orDefault()
-        val structuredOutput = StructuredOutput(schema = schema)
-        var generation = generateStructuredUnary(prompt, opts, structuredOutput)
-        var parsed = parseStructuredOutput(generation, structuredOutput)
-
-        if (mode == StructuredOutputMode.REPAIR && parsed.validation?.is_valid != true) {
-            val repairPrompt = structuredRepairPrompt(prompt, generation.text, schema)
-            generation = generateStructuredUnary(repairPrompt, opts, structuredOutput)
-            parsed = parseStructuredOutput(generation, structuredOutput)
-        }
+        val opts = options.orDefault().copy(toolChoice = ToolChoice.None)
+        val effectiveSchema = opts.structuredOutput?.schema ?: schema
+        val effectiveMode = opts.structuredOutput?.mode ?: mode
+        val effectiveStrict = opts.structuredOutput?.strict ?: true
+        val structuredOutput = StructuredOutput(schema = effectiveSchema, strict = effectiveStrict, mode = effectiveMode)
+        val generation = generateStructuredUnary(prompt, opts, structuredOutput)
+        val parsed = parseStructuredOutput(generation, structuredOutput)
 
         return StructuredResult(
             value = parsed.json,
             raw = parsed.raw_text ?: generation.text,
             valid = parsed.validation?.is_valid == true,
-            mode = mode,
+            mode = effectiveMode,
             inputTokens = generation.inputTokens,
             outputTokens = generation.outputTokens,
             timeToFirstTokenMs = generation.timeToFirstTokenMs,
@@ -262,17 +252,6 @@ public class LlmNamespace internal constructor() {
 
 private fun newRequestId(): String = UUID.randomUUID().toString()
 
-/** The one retry prompt `generateStructured(mode = REPAIR)` sends when the first pass did not validate. */
-private fun structuredRepairPrompt(original: String, invalidOutput: String, schema: JsonSchema): String =
-    """
-    $original
-
-    Your previous answer did not match the required JSON schema. Reply again with ONLY JSON that satisfies this schema.
-
-    Schema: ${schema.rawJson}
-    Previous invalid answer: $invalidOutput
-    """.trimIndent()
-
 private fun LlmOptions.usesTools(): Boolean = toolChoice != ToolChoice.None
 
 /**

bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/MappingOptions.kt

@@ -34,15 +34,16 @@ import ai.runanywhere.proto.v1.DiarizationOptions as ProtoDiarizationOptions
 import ai.runanywhere.proto.v1.ReasoningMode as ProtoReasoningMode
 import ai.runanywhere.proto.v1.ReasoningOptions as ProtoReasoningOptions
 import ai.runanywhere.proto.v1.SegmentationOptions as ProtoSegmentationOptions
+import ai.runanywhere.proto.v1.StructuredOutputMode as ProtoStructuredOutputMode
 
 /** The IDL's neutral repetition penalty, used when the caller leaves it unset. */
 private val NEUTRAL_REPETITION_PENALTY = LLMGenerationOptions.defaults().repeat_penalty
 
 internal fun LlmOptions?.orDefault(): LlmOptions = this ?: LlmOptions()
 
 /**
- * [structuredOutput] is never populated from a public [LlmOptions] field —
- * only `llm.generateStructured` supplies it, through its own internal call path.
+ * [structuredOutput] is supplied by `llm.generateStructured`, which merges the
+ * caller's [LlmOptions.structuredOutput] overrides with its schema.
  */
 internal fun LlmOptions.toProto(structuredOutput: StructuredOutput? = null): LLMGenerationOptions =
     LLMGenerationOptions(
@@ -112,17 +113,18 @@ internal fun ReasoningOptions.toProto(): ProtoReasoningOptions =
     )
 
 /**
- * `StructuredOutputOptions.strict_mode`/`.mode`/`.repair_json` are all
- * deleted (idl/structured_output.proto): the message shrank to
- * `include_schema_in_prompt` plus a `oneof constraint { schema | grammar |
- * regex }`. [StructuredOutput.strict] has no wire home any more — retry
- * behaviour for an invalid first pass is owned entirely by the Kotlin-side
- * `llm.generateStructured(mode = REPAIR)` loop, not by a commons flag.
+ * Build the proto structured-output options for [StructuredOutput].
  */
 internal fun StructuredOutput.toProto(): StructuredOutputOptions =
     StructuredOutputOptions(
         schema = schema.rawJson,
         include_schema_in_prompt = true,
+        mode =
+            when (mode) {
+                StructuredOutputMode.CONSTRAINED -> ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_CONSTRAINED
+                StructuredOutputMode.VALIDATION_ONLY -> ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_VALIDATION_ONLY
+                StructuredOutputMode.REPAIR -> ProtoStructuredOutputMode.STRUCTURED_OUTPUT_MODE_REPAIR
+            },
     )
 
 internal fun SttOptions?.orDefault(): SttOptions = this ?: SttOptions()

bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Options.kt

@@ -56,14 +56,20 @@ public sealed class ToolChoice {
 }
 
 /** Schema the generated text must conform to. */
-public data class StructuredOutput(
-    val schema: JsonSchema,
-    val strict: Boolean = true,
-)
+public data class StructuredOutput
+    @JvmOverloads
+    constructor(
+        val schema: JsonSchema,
+        val strict: Boolean = true,
+        val mode: StructuredOutputMode = StructuredOutputMode.CONSTRAINED,
+    ) {
+        /** Backward compatibility constructor for callers passing (schema, mode). */
+        public constructor(schema: JsonSchema, mode: StructuredOutputMode) : this(schema, strict = true, mode = mode)
+    }
 
 /** Enforcement level `llm.generateStructured` applies to its schema. */
 public enum class StructuredOutputMode {
-    /** Engine-constrained decoding; fails preflight until a constrained-decoding engine is wired in. */
+    /** Engine-constrained decoding (GBNF grammar sampling). */
     CONSTRAINED,
 
     /** Generate freely, then validate against the schema. */
@@ -89,8 +95,6 @@ public data class LlmOptions(
     val stopSequences: List<String> = emptyList(),
     val systemPrompt: String? = null,
     val reasoning: ReasoningOptions? = null,
-    // No structured-output schema field: `llm.generateStructured` owns
-    // schema configuration through its own `schema`/`mode` parameters.
     /** Empty uses the tools registered through `llm.tools`. */
     val tools: List<ToolDefinition> = emptyList(),
     val toolChoice: ToolChoice = ToolChoice.Auto,
@@ -103,6 +107,13 @@ public data class LlmOptions(
      * `llm.generate`/`llm.generateStream` with inline `tools`.
      */
     val autoExecute: Boolean = true,
+    /**
+     * Optional structured-output override for `llm.generateStructured`, which supplies the
+     * schema and uses this field's `mode` in preference to its own `mode` argument.
+     * `strict` is not forwarded: `StructuredOutputOptions` has no strict field on the wire.
+     * Declared last so existing positional arguments and `componentN()` keep their slots.
+     */
+    val structuredOutput: StructuredOutput? = null,
 ) {
     /** Sampling and tool-loop defaults, as annotated in idl/. */
     public companion object {

bindings/kotlin/src/main/kotlin/com/runanywhere/sdk/public/api/Results.kt

@@ -50,7 +50,7 @@ public data class StructuredResult(
     val value: String,
     val raw: String,
     val valid: Boolean,
-    val mode: StructuredOutputMode = StructuredOutputMode.VALIDATION_ONLY,
+    val mode: StructuredOutputMode = StructuredOutputMode.CONSTRAINED,
     val inputTokens: Int = 0,
     val outputTokens: Int = 0,
     val timeToFirstTokenMs: Long = 0L,