diff --git a/docs/release_notes.md b/docs/release_notes.md index 35655f140..b9489692a 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -12,7 +12,7 @@ ### ✨ New Functionality -- +- [Spring AI] Chat completion calls via the Spring AI integration now can have multiple module configs to support fallback modules as well. ### 📈 Improvements diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModel.java index 44310e005..ef43d1c52 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModel.java @@ -6,6 +6,7 @@ import com.sap.ai.sdk.orchestration.AssistantMessage; import com.sap.ai.sdk.orchestration.OrchestrationChatCompletionDelta; import com.sap.ai.sdk.orchestration.OrchestrationClient; +import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig; import com.sap.ai.sdk.orchestration.OrchestrationPrompt; import com.sap.ai.sdk.orchestration.SystemMessage; import com.sap.ai.sdk.orchestration.ToolMessage; @@ -69,7 +70,10 @@ public ChatResponse call(@Nonnull final Prompt prompt) { val orchestrationPrompt = toOrchestrationPrompt(prompt); val response = new OrchestrationSpringChatResponse( - client.chatCompletion(orchestrationPrompt, options.getConfig())); + client.chatCompletion( + orchestrationPrompt, + options.getConfig(), + options.getFallbackConfigs().toArray(OrchestrationModuleConfig[]::new))); if (ToolCallingChatOptions.isInternalToolExecutionEnabled(prompt.getOptions()) && response.hasToolCalls()) { @@ -99,7 +103,11 @@ public Flux stream(@Nonnull final Prompt prompt) { if (prompt.getOptions() instanceof OrchestrationChatOptions options) { val orchestrationPrompt = toOrchestrationPrompt(prompt); - val request = toCompletionPostRequest(orchestrationPrompt, options.getConfig()); + val request = + toCompletionPostRequest( + orchestrationPrompt, + options.getConfig(), + options.getFallbackConfigs().toArray(OrchestrationModuleConfig[]::new)); val stream = client.streamChatCompletionDeltas(request); final Flux flux = diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatOptions.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatOptions.java index b6b32aa3a..e2bed0d5b 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatOptions.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatOptions.java @@ -41,6 +41,8 @@ public class OrchestrationChatOptions implements ToolCallingChatOptions { @Nonnull private OrchestrationModuleConfig config; + @Nonnull private List fallbackConfigs = List.of(); + @Nonnull private List toolCallbacks = List.of(); @Getter(AccessLevel.NONE) @@ -169,6 +171,7 @@ public T copy() { .withGroundingConfig(config.getGroundingConfig()); val result = new OrchestrationChatOptions(copyConfig); result.setToolCallbacks(toolCallbacks); + result.setFallbackConfigs(fallbackConfigs); result.setInternalToolExecutionEnabled(internalToolExecutionEnabled); return (T) result; } diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModelTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModelTest.java index 17b1eaa82..ed53cf8cc 100644 --- a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModelTest.java +++ b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModelTest.java @@ -20,6 +20,7 @@ import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.sap.ai.sdk.orchestration.OrchestrationAiModel; import com.sap.ai.sdk.orchestration.OrchestrationClient; import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor; @@ -28,16 +29,19 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Function; import lombok.val; import org.apache.hc.client5.http.classic.HttpClient; +import org.apache.hc.core5.http.ClassicHttpRequest; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.io.entity.InputStreamEntity; import org.apache.hc.core5.http.message.BasicClassicHttpResponse; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.chat.client.advisor.MessageChatMemoryAdvisor; @@ -260,4 +264,86 @@ void testChatMemory() throws IOException { verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); } } + + @Test + void testFallbackModules() throws IOException { + stubFor( + post(urlPathEqualTo("/v2/completion")) + .willReturn( + aResponse() + .withBodyFile("fallbackResponse.json") + .withHeader("Content-Type", "application/json"))); + + val brokenConfig = + new OrchestrationModuleConfig() + .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest")); + val workingConfig = new OrchestrationModuleConfig().withLlmConfig(GPT_4O); + + val options = new OrchestrationChatOptions(brokenConfig); + options.setFallbackConfigs(List.of(workingConfig)); + + val result = client.call(new Prompt("Hello World! Why is this phrase so famous?", options)); + + assertThat(result).isNotNull(); + assertThat(result.getResult().getOutput().getText()).isNotEmpty(); + assertThat(result).isInstanceOf(OrchestrationSpringChatResponse.class); + + val intermediateFailures = + ((OrchestrationSpringChatResponse) result) + .getOrchestrationResponse() + .getOriginalResponse() + .getIntermediateFailures(); + assertThat(intermediateFailures).hasSize(1); + assertThat(intermediateFailures.get(0).getCode()).isEqualTo(400); + assertThat(intermediateFailures.get(0).getMessage()).contains("broken_name"); + + try (var requestInputStream = fileLoader.apply("springFallbackRequest.json")) { + final String request = new String(requestInputStream.readAllBytes()); + verify( + postRequestedFor(urlPathEqualTo("/v2/completion")).withRequestBody(equalToJson(request))); + } + } + + @Test + void testStreamFallbackModules() throws IOException { + try (val inputStream = spy(fileLoader.apply("streamFallbackChatCompletion.txt"))) { + + val httpClient = mock(HttpClient.class); + ApacheHttpClient5Accessor.setHttpClientFactory(destination -> httpClient); + + val mockResponse = new BasicClassicHttpResponse(200, "OK"); + mockResponse.setEntity(new InputStreamEntity(inputStream, ContentType.TEXT_PLAIN)); + mockResponse.setHeader("Content-Type", "text/event-flux"); + + val requestCaptor = ArgumentCaptor.forClass(ClassicHttpRequest.class); + doReturn(mockResponse).when(httpClient).executeOpen(any(), requestCaptor.capture(), any()); + + val brokenConfig = + new OrchestrationModuleConfig() + .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest")); + val workingConfig = new OrchestrationModuleConfig().withLlmConfig(GPT_4O); + + val options = new OrchestrationChatOptions(brokenConfig); + options.setFallbackConfigs(List.of(workingConfig)); + + Flux flux = + client.stream(new Prompt("Hello World! Why is this phrase so famous?", options)); + val deltaList = flux.toStream().toList(); + + assertThat(deltaList).hasSize(3); + assertThat(deltaList.get(0).getResult().getOutput().getText()).isEqualTo(""); + assertThat(deltaList.get(1).getResult().getOutput().getText()).isEqualTo("Sure"); + assertThat(deltaList.get(2).getResult().getOutput().getText()).isEqualTo("!"); + assertThat(deltaList.get(2).getResult().getMetadata().getFinishReason()).isEqualTo("stop"); + + try (var requestInputStream = fileLoader.apply("springFallbackStreamRequest.json")) { + final String expectedBody = new String(requestInputStream.readAllBytes()); + final String actualBody = + new String(requestCaptor.getValue().getEntity().getContent().readAllBytes()); + assertThat(actualBody).isEqualToIgnoringWhitespace(expectedBody); + } + + Mockito.verify(inputStream, times(1)).close(); + } + } } diff --git a/orchestration/src/test/resources/springFallbackRequest.json b/orchestration/src/test/resources/springFallbackRequest.json new file mode 100644 index 000000000..8af249f19 --- /dev/null +++ b/orchestration/src/test/resources/springFallbackRequest.json @@ -0,0 +1,43 @@ +{ + "config" : { + "modules" : [ { + "prompt_templating" : { + "prompt" : { + "template" : [ { + "content" : "Hello World! Why is this phrase so famous?", + "role" : "user" + } ], + "defaults" : { }, + "tools" : [ ] + }, + "model" : { + "name" : "broken_name", + "version" : "latest", + "params" : { }, + "timeout" : 600, + "max_retries" : 2 + } + } + }, { + "prompt_templating" : { + "prompt" : { + "template" : [ { + "content" : "Hello World! Why is this phrase so famous?", + "role" : "user" + } ], + "defaults" : { }, + "tools" : [ ] + }, + "model" : { + "name" : "gpt-4o", + "version" : "latest", + "params" : { }, + "timeout" : 600, + "max_retries" : 2 + } + } + } ] + }, + "placeholder_values" : { }, + "messages_history" : [ ] +} diff --git a/orchestration/src/test/resources/springFallbackStreamRequest.json b/orchestration/src/test/resources/springFallbackStreamRequest.json new file mode 100644 index 000000000..e7188dff0 --- /dev/null +++ b/orchestration/src/test/resources/springFallbackStreamRequest.json @@ -0,0 +1,47 @@ +{ + "config" : { + "modules" : [ { + "prompt_templating" : { + "prompt" : { + "template" : [ { + "content" : "Hello World! Why is this phrase so famous?", + "role" : "user" + } ], + "defaults" : { }, + "tools" : [ ] + }, + "model" : { + "name" : "broken_name", + "version" : "latest", + "params" : { }, + "timeout" : 600, + "max_retries" : 2 + } + } + }, { + "prompt_templating" : { + "prompt" : { + "template" : [ { + "content" : "Hello World! Why is this phrase so famous?", + "role" : "user" + } ], + "defaults" : { }, + "tools" : [ ] + }, + "model" : { + "name" : "gpt-4o", + "version" : "latest", + "params" : { }, + "timeout" : 600, + "max_retries" : 2 + } + } + } ], + "stream" : { + "enabled" : true, + "chunk_size" : 100 + } + }, + "placeholder_values" : { }, + "messages_history" : [ ] +} diff --git a/orchestration/src/test/resources/streamFallbackChatCompletion.txt b/orchestration/src/test/resources/streamFallbackChatCompletion.txt new file mode 100644 index 000000000..ba7afb306 --- /dev/null +++ b/orchestration/src/test/resources/streamFallbackChatCompletion.txt @@ -0,0 +1,4 @@ +data: {"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "intermediate_results": {"templating": [{"role": "user", "content": "Hello World! Why is this phrase so famous?"}]}, "intermediate_failures": [{"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "code": 400, "message": "400 - Request Body: Model broken_name not supported.", "location": "Request Body", "headers": {"Content-Type": "application/json"}}], "final_result": {"id": "", "object": "", "created": 0, "model": "", "system_fingerprint": "", "choices": [{"index": 0, "delta": {"role": "", "content": ""}, "finish_reason": ""}]}} +data: {"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "intermediate_results": {"llm": {"id": "chatcmpl-AYZSQQwWv7ajJsyDBpMG4X01BBJxq", "object": "chat.completion.chunk", "created": 1732802814, "model": "gpt-4o", "system_fingerprint": "fp_808245b034", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Sure"}, "finish_reason": ""}]}}, "intermediate_failures": [{"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "code": 400, "message": "400 - Request Body: Model broken_name not supported.", "location": "Request Body", "headers": {"Content-Type": "application/json"}}], "final_result": {"id": "chatcmpl-AYZSQQwWv7ajJsyDBpMG4X01BBJxq", "object": "chat.completion.chunk", "created": 1732802814, "model": "gpt-4o", "system_fingerprint": "fp_808245b034", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Sure"}, "finish_reason": ""}]}} +data: {"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "intermediate_results": {"llm": {"id": "chatcmpl-AYZSQQwWv7ajJsyDBpMG4X01BBJxq", "object": "chat.completion.chunk", "created": 1732802814, "model": "gpt-4o", "system_fingerprint": "fp_808245b034", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "!"}, "finish_reason": "stop"}]}}, "intermediate_failures": [{"request_id": "5bd87b41-6368-4c18-aaae-47ab82e9475b", "code": 400, "message": "400 - Request Body: Model broken_name not supported.", "location": "Request Body", "headers": {"Content-Type": "application/json"}}], "final_result": {"id": "chatcmpl-AYZSQQwWv7ajJsyDBpMG4X01BBJxq", "object": "chat.completion.chunk", "created": 1732802814, "model": "gpt-4o", "system_fingerprint": "fp_808245b034", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "!"}, "finish_reason": "stop"}]}} +data: [DONE] diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationController.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationController.java index e4226674c..769a89b8a 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationController.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationController.java @@ -164,4 +164,17 @@ Object embedding( @Nullable @RequestParam(value = "format", required = false) final String format) { return service.embed("Hello, world!"); } + + @GetMapping("/completionWithFallback") + Object completionWithFallback( + @Nullable @RequestParam(value = "format", required = false) final String format) { + val response = service.completionWithFallback(); + + if ("json".equals(format)) { + return ((OrchestrationSpringChatResponse) response) + .getOrchestrationResponse() + .getOriginalResponse(); + } + return response.getResult().getOutput().getText(); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/SpringAiOrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/SpringAiOrchestrationService.java index 62c9cdc8a..f01215230 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/SpringAiOrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/SpringAiOrchestrationService.java @@ -8,6 +8,7 @@ import com.sap.ai.sdk.orchestration.AzureContentFilter; import com.sap.ai.sdk.orchestration.AzureFilterThreshold; import com.sap.ai.sdk.orchestration.DpiMasking; +import com.sap.ai.sdk.orchestration.OrchestrationAiModel; import com.sap.ai.sdk.orchestration.OrchestrationClientException; import com.sap.ai.sdk.orchestration.OrchestrationEmbeddingModel; import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig; @@ -244,6 +245,24 @@ public ChatResponse chatMemory() { "Chat response is null"); } + /** + * Chat request using the Spring AI integration with fallback configs. The first config uses an + * invalid model name, so the orchestration service falls back to the second config. + * + * @return the assistant response object + */ + @Nonnull + public ChatResponse completionWithFallback() { + val brokenConfig = + new OrchestrationModuleConfig() + .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest")); + val workingConfig = new OrchestrationModuleConfig().withLlmConfig(GPT_41); + val options = new OrchestrationChatOptions(brokenConfig); + options.setFallbackConfigs(List.of(workingConfig)); + val prompt = new Prompt("Why is 'Hello World' so famous?", options); + return client.call(prompt); + } + /** * A simple record to demonstrate the response format feature of the orchestration service. * diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index e1e45b008..8c8c9cdcf 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1218,6 +1218,20 @@
Orchestration Integration
+
  • +
    + +
    + Uses fallback module configs: the primary config specifies an + invalid model, so the orchestration service automatically falls + back to the second config with a valid model. +
    +
    +
    • diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationTest.java index 369520b5c..ab8a8a5a6 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/SpringAiOrchestrationTest.java @@ -172,4 +172,19 @@ void testEmbedding() { assertThat(embeddings).isInstanceOf(float[].class); assertThat(embeddings.length).isGreaterThan(0); } + + @Test + void testCompletionWithFallback() { + var response = service.completionWithFallback(); + assertThat(response).isNotNull(); + assertThat(response.getResult().getOutput().getText()).isNotEmpty(); + var intermediateFailures = + ((OrchestrationSpringChatResponse) response) + .getOrchestrationResponse() + .getOriginalResponse() + .getIntermediateFailures(); + assertThat(intermediateFailures).hasSize(1); + assertThat(intermediateFailures.get(0).getCode()).isEqualTo(400); + assertThat(intermediateFailures.get(0).getMessage()).contains("broken_name"); + } }