feat: [Orchestration] Add fallback configs to Spring AI integration

docs/release_notes.md

@@ -12,7 +12,7 @@
 
 ### ✨ New Functionality
 
--
+- [Spring AI] Chat completion calls via the Spring AI integration now can have multiple module configs to support fallback modules as well.
 
 ### 📈 Improvements
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModel.java

@@ -6,6 +6,7 @@
 import com.sap.ai.sdk.orchestration.AssistantMessage;
 import com.sap.ai.sdk.orchestration.OrchestrationChatCompletionDelta;
 import com.sap.ai.sdk.orchestration.OrchestrationClient;
+import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig;
 import com.sap.ai.sdk.orchestration.OrchestrationPrompt;
 import com.sap.ai.sdk.orchestration.SystemMessage;
 import com.sap.ai.sdk.orchestration.ToolMessage;
@@ -69,7 +70,10 @@ public ChatResponse call(@Nonnull final Prompt prompt) {
       val orchestrationPrompt = toOrchestrationPrompt(prompt);
       val response =
           new OrchestrationSpringChatResponse(
-              client.chatCompletion(orchestrationPrompt, options.getConfig()));
+              client.chatCompletion(
+                  orchestrationPrompt,
+                  options.getConfig(),
+                  options.getFallbackConfigs().toArray(OrchestrationModuleConfig[]::new)));
 
       if (ToolCallingChatOptions.isInternalToolExecutionEnabled(prompt.getOptions())
           && response.hasToolCalls()) {
@@ -99,7 +103,11 @@ public Flux<ChatResponse> stream(@Nonnull final Prompt prompt) {
     if (prompt.getOptions() instanceof OrchestrationChatOptions options) {
 
       val orchestrationPrompt = toOrchestrationPrompt(prompt);
-      val request = toCompletionPostRequest(orchestrationPrompt, options.getConfig());
+      val request =
+          toCompletionPostRequest(
+              orchestrationPrompt,
+              options.getConfig(),
+              options.getFallbackConfigs().toArray(OrchestrationModuleConfig[]::new));
       val stream = client.streamChatCompletionDeltas(request);
 
       final Flux<OrchestrationChatCompletionDelta> flux =

orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatOptions.java

@@ -41,6 +41,8 @@ public class OrchestrationChatOptions implements ToolCallingChatOptions {
 
   @Nonnull private OrchestrationModuleConfig config;
 
+  @Nonnull private List<OrchestrationModuleConfig> fallbackConfigs = List.of();
+
   @Nonnull private List<ToolCallback> toolCallbacks = List.of();
 
   @Getter(AccessLevel.NONE)
@@ -169,6 +171,7 @@ public <T extends ChatOptions> T copy() {
             .withGroundingConfig(config.getGroundingConfig());
     val result = new OrchestrationChatOptions(copyConfig);
     result.setToolCallbacks(toolCallbacks);
+    result.setFallbackConfigs(fallbackConfigs);
     result.setInternalToolExecutionEnabled(internalToolExecutionEnabled);
     return (T) result;
   }

orchestration/src/test/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModelTest.java

@@ -20,6 +20,7 @@
 
 import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
 import com.github.tomakehurst.wiremock.junit5.WireMockTest;
+import com.sap.ai.sdk.orchestration.OrchestrationAiModel;
 import com.sap.ai.sdk.orchestration.OrchestrationClient;
 import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig;
 import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
@@ -28,16 +29,19 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
 import lombok.val;
 import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.core5.http.ClassicHttpRequest;
 import org.apache.hc.core5.http.ContentType;
 import org.apache.hc.core5.http.io.entity.InputStreamEntity;
 import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 import org.springframework.ai.chat.client.ChatClient;
 import org.springframework.ai.chat.client.advisor.MessageChatMemoryAdvisor;
@@ -260,4 +264,86 @@ void testChatMemory() throws IOException {
       verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request)));
     }
   }
+
+  @Test
+  void testFallbackModules() throws IOException {
+    stubFor(
+        post(urlPathEqualTo("/v2/completion"))
+            .willReturn(
+                aResponse()
+                    .withBodyFile("fallbackResponse.json")
+                    .withHeader("Content-Type", "application/json")));
+
+    val brokenConfig =
+        new OrchestrationModuleConfig()
+            .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest"));
+    val workingConfig = new OrchestrationModuleConfig().withLlmConfig(GPT_4O);
+
+    val options = new OrchestrationChatOptions(brokenConfig);
+    options.setFallbackConfigs(List.of(workingConfig));
+
+    val result = client.call(new Prompt("Hello World! Why is this phrase so famous?", options));
+
+    assertThat(result).isNotNull();
+    assertThat(result.getResult().getOutput().getText()).isNotEmpty();
+    assertThat(result).isInstanceOf(OrchestrationSpringChatResponse.class);
+
+    val intermediateFailures =
+        ((OrchestrationSpringChatResponse) result)
+            .getOrchestrationResponse()
+            .getOriginalResponse()
+            .getIntermediateFailures();
+    assertThat(intermediateFailures).hasSize(1);
+    assertThat(intermediateFailures.get(0).getCode()).isEqualTo(400);
+    assertThat(intermediateFailures.get(0).getMessage()).contains("broken_name");
+
+    try (var requestInputStream = fileLoader.apply("springFallbackRequest.json")) {
+      final String request = new String(requestInputStream.readAllBytes());
+      verify(
+          postRequestedFor(urlPathEqualTo("/v2/completion")).withRequestBody(equalToJson(request)));
+    }
+  }
+
+  @Test
+  void testStreamFallbackModules() throws IOException {
+    try (val inputStream = spy(fileLoader.apply("streamFallbackChatCompletion.txt"))) {
+
+      val httpClient = mock(HttpClient.class);
+      ApacheHttpClient5Accessor.setHttpClientFactory(destination -> httpClient);
+
+      val mockResponse = new BasicClassicHttpResponse(200, "OK");
+      mockResponse.setEntity(new InputStreamEntity(inputStream, ContentType.TEXT_PLAIN));
+      mockResponse.setHeader("Content-Type", "text/event-flux");
+
+      val requestCaptor = ArgumentCaptor.forClass(ClassicHttpRequest.class);
+      doReturn(mockResponse).when(httpClient).executeOpen(any(), requestCaptor.capture(), any());
+
+      val brokenConfig =
+          new OrchestrationModuleConfig()
+              .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest"));
+      val workingConfig = new OrchestrationModuleConfig().withLlmConfig(GPT_4O);
+
+      val options = new OrchestrationChatOptions(brokenConfig);
+      options.setFallbackConfigs(List.of(workingConfig));
+
+      Flux<ChatResponse> flux =
+          client.stream(new Prompt("Hello World! Why is this phrase so famous?", options));
+      val deltaList = flux.toStream().toList();
+
+      assertThat(deltaList).hasSize(3);
+      assertThat(deltaList.get(0).getResult().getOutput().getText()).isEqualTo("");
+      assertThat(deltaList.get(1).getResult().getOutput().getText()).isEqualTo("Sure");
+      assertThat(deltaList.get(2).getResult().getOutput().getText()).isEqualTo("!");
+      assertThat(deltaList.get(2).getResult().getMetadata().getFinishReason()).isEqualTo("stop");
+
+      try (var requestInputStream = fileLoader.apply("springFallbackStreamRequest.json")) {
+        final String expectedBody = new String(requestInputStream.readAllBytes());
+        final String actualBody =
+            new String(requestCaptor.getValue().getEntity().getContent().readAllBytes());
+        assertThat(actualBody).isEqualToIgnoringWhitespace(expectedBody);
+      }
+
+      Mockito.verify(inputStream, times(1)).close();
+    }
+  }
 }

orchestration/src/test/resources/springFallbackRequest.json

@@ -0,0 +1,43 @@
+{
+  "config" : {
+    "modules" : [ {
+      "prompt_templating" : {
+        "prompt" : {
+          "template" : [ {
+            "content" : "Hello World! Why is this phrase so famous?",
+            "role" : "user"
+          } ],
+          "defaults" : { },
+          "tools" : [ ]
+        },
+        "model" : {
+          "name" : "broken_name",
+          "version" : "latest",
+          "params" : { },
+          "timeout" : 600,
+          "max_retries" : 2
+        }
+      }
+    }, {
+      "prompt_templating" : {
+        "prompt" : {
+          "template" : [ {
+            "content" : "Hello World! Why is this phrase so famous?",
+            "role" : "user"
+          } ],
+          "defaults" : { },
+          "tools" : [ ]
+        },
+        "model" : {
+          "name" : "gpt-4o",
+          "version" : "latest",
+          "params" : { },
+          "timeout" : 600,
+          "max_retries" : 2
+        }
+      }
+    } ]
+  },
+  "placeholder_values" : { },
+  "messages_history" : [ ]
+}

orchestration/src/test/resources/springFallbackStreamRequest.json

@@ -0,0 +1,47 @@
+{
+  "config" : {
+    "modules" : [ {
+      "prompt_templating" : {
+        "prompt" : {
+          "template" : [ {
+            "content" : "Hello World! Why is this phrase so famous?",
+            "role" : "user"
+          } ],
+          "defaults" : { },
+          "tools" : [ ]
+        },
+        "model" : {
+          "name" : "broken_name",
+          "version" : "latest",
+          "params" : { },
+          "timeout" : 600,
+          "max_retries" : 2
+        }
+      }
+    }, {
+      "prompt_templating" : {
+        "prompt" : {
+          "template" : [ {
+            "content" : "Hello World! Why is this phrase so famous?",
+            "role" : "user"
+          } ],
+          "defaults" : { },
+          "tools" : [ ]
+        },
+        "model" : {
+          "name" : "gpt-4o",
+          "version" : "latest",
+          "params" : { },
+          "timeout" : 600,
+          "max_retries" : 2
+        }
+      }
+    } ],
+    "stream" : {
+      "enabled" : true,
+      "chunk_size" : 100
+    }
+  },
+  "placeholder_values" : { },
+  "messages_history" : [ ]
+}

orchestration/src/test/resources/streamFallbackChatCompletion.txt

@@ -0,0 +1,4 @@
+data: {"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "intermediate_results": {"templating": [{"role": "user", "content": "Hello World! Why is this phrase so famous?"}]}, "intermediate_failures": [{"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "code": 400, "message": "400 - Request Body: Model broken_name not supported.", "location": "Request Body", "headers": {"Content-Type": "application/json"}}], "final_result": {"id": "", "object": "", "created": 0, "model": "", "system_fingerprint": "", "choices": [{"index": 0, "delta": {"role": "", "content": ""}, "finish_reason": ""}]}}
+data: {"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "intermediate_results": {"llm": {"id": "chatcmpl-AYZSQQwWv7ajJsyDBpMG4X01BBJxq", "object": "chat.completion.chunk", "created": 1732802814, "model": "gpt-4o", "system_fingerprint": "fp_808245b034", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Sure"}, "finish_reason": ""}]}}, "intermediate_failures": [{"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "code": 400, "message": "400 - Request Body: Model broken_name not supported.", "location": "Request Body", "headers": {"Content-Type": "application/json"}}], "final_result": {"id": "chatcmpl-AYZSQQwWv7ajJsyDBpMG4X01BBJxq", "object": "chat.completion.chunk", "created": 1732802814, "model": "gpt-4o", "system_fingerprint": "fp_808245b034", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Sure"}, "finish_reason": ""}]}}
+data: {"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "intermediate_results": {"llm": {"id": "chatcmpl-AYZSQQwWv7ajJsyDBpMG4X01BBJxq", "object": "chat.completion.chunk", "created": 1732802814, "model": "gpt-4o", "system_fingerprint": "fp_808245b034", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "!"}, "finish_reason": "stop"}]}}, "intermediate_failures": [{"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "code": 400, "message": "400 - Request Body: Model broken_name not supported.", "location": "Request Body", "headers": {"Content-Type": "application/json"}}], "final_result": {"id": "chatcmpl-AYZSQQwWv7ajJsyDBpMG4X01BBJxq", "object": "chat.completion.chunk", "created": 1732802814, "model": "gpt-4o", "system_fingerprint": "fp_808245b034", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "!"}, "finish_reason": "stop"}]}}
+data: [DONE]

sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationController.java

@@ -164,4 +164,17 @@ Object embedding(
       @Nullable @RequestParam(value = "format", required = false) final String format) {
     return service.embed("Hello, world!");
   }
+
+  @GetMapping("/completionWithFallback")
+  Object completionWithFallback(
+      @Nullable @RequestParam(value = "format", required = false) final String format) {
+    val response = service.completionWithFallback();
+
+    if ("json".equals(format)) {
+      return ((OrchestrationSpringChatResponse) response)
+          .getOrchestrationResponse()
+          .getOriginalResponse();
+    }
+    return response.getResult().getOutput().getText();
+  }
 }

sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/SpringAiOrchestrationService.java

@@ -8,6 +8,7 @@
 import com.sap.ai.sdk.orchestration.AzureContentFilter;
 import com.sap.ai.sdk.orchestration.AzureFilterThreshold;
 import com.sap.ai.sdk.orchestration.DpiMasking;
+import com.sap.ai.sdk.orchestration.OrchestrationAiModel;
 import com.sap.ai.sdk.orchestration.OrchestrationClientException;
 import com.sap.ai.sdk.orchestration.OrchestrationEmbeddingModel;
 import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig;
@@ -244,6 +245,24 @@ public ChatResponse chatMemory() {
         "Chat response is null");
   }
 
+  /**
+   * Chat request using the Spring AI integration with fallback configs. The first config uses an
+   * invalid model name, so the orchestration service falls back to the second config.
+   *
+   * @return the assistant response object
+   */
+  @Nonnull
+  public ChatResponse completionWithFallback() {
+    val brokenConfig =
+        new OrchestrationModuleConfig()
+            .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest"));
+    val workingConfig = new OrchestrationModuleConfig().withLlmConfig(GPT_41);
+    val options = new OrchestrationChatOptions(brokenConfig);
+    options.setFallbackConfigs(List.of(workingConfig));
+    val prompt = new Prompt("Why is 'Hello World' so famous?", options);
+    return client.call(prompt);
+  }
+
   /**
    * A simple record to demonstrate the response format feature of the orchestration service.
    *

sample-code/spring-app/src/main/resources/static/index.html

@@ -1218,6 +1218,20 @@ <h5 class="mb-1">Orchestration Integration</h5>
                                     </div>
                                 </div>
                             </li>
+                            <li class="list-group-item">
+                                <div class="info-tooltip">
+                                    <button type="submit"
+                                            formaction="/spring-ai-orchestration/completionWithFallback"
+                                            class="link-offset-2-hover link-underline link-underline-opacity-0 link-underline-opacity-75-hover endpoint">
+                                        <code>/spring-ai-orchestration/completionWithFallback</code>
+                                    </button>
+                                    <div class="tooltip-content">
+                                        Uses fallback module configs: the primary config specifies an
+                                        invalid model, so the orchestration service automatically falls
+                                        back to the second config with a valid model.
+                                    </div>
+                                </div>
+                            </li>
                         </ul>
                     </div>
                     <div class="card-body">

sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationTest.java

@@ -172,4 +172,19 @@ void testEmbedding() {
     assertThat(embeddings).isInstanceOf(float[].class);
     assertThat(embeddings.length).isGreaterThan(0);
   }
+
+  @Test
+  void testCompletionWithFallback() {
+    var response = service.completionWithFallback();
+    assertThat(response).isNotNull();
+    assertThat(response.getResult().getOutput().getText()).isNotEmpty();
+    var intermediateFailures =
+        ((OrchestrationSpringChatResponse) response)
+            .getOrchestrationResponse()
+            .getOriginalResponse()
+            .getIntermediateFailures();
+    assertThat(intermediateFailures).hasSize(1);
+    assertThat(intermediateFailures.get(0).getCode()).isEqualTo(400);
+    assertThat(intermediateFailures.get(0).getMessage()).contains("broken_name");
+  }
 }