From e52f3702bb40188642aadf48fc7e835f72159a50 Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Sun, 1 Nov 2026 22:23:50 +0100 Subject: [PATCH 1/3] feat(agent): add LiveLlmProvider abstraction and Gemini Live WebSocket provider --- .../agent/data/DataStoreSettingsRepository.kt | 12 + .../agent/data/GeminiJsonUtils.kt | 43 ++ .../agent/data/GeminiLiveProviderImpl.kt | 415 ++++++++++++++++++ .../agent/data/GeminiLiveVoice.kt | 72 +++ .../agent/data/GeminiProviderImpl.kt | 42 +- .../agent/data/GeminiToolConverter.kt | 62 +++ .../agent/data/LlmProviderFactoryImpl.kt | 10 +- .../agent/data/SettingsRepository.kt | 6 + .../appfunctions/agent/di/DataModule.kt | 8 +- .../agent/domain/LiveLlmProvider.kt | 65 +++ .../agent/domain/LlmProviderFactory.kt | 7 +- .../agent/data/GeminiLiveProviderImplTest.kt | 396 +++++++++++++++++ .../agent/data/GeminiToolConverterTest.kt | 64 +++ .../agent/data/SettingsRepositoryTest.kt | 15 + .../agentdemo/FakeSettingsRepository.kt | 8 + 15 files changed, 1185 insertions(+), 40 deletions(-) create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt index aac24d4d..329fe35a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt @@ -37,6 +37,7 @@ class DataStoreSettingsRepository val GEMINI_API_KEY = stringPreferencesKey("gemini_api_key") val SELECTED_PROVIDER = stringPreferencesKey("selected_provider") val SERVICE_TIER = stringPreferencesKey("service_tier") + val GEMINI_LIVE_VOICE = stringPreferencesKey("gemini_live_voice") val PINNED_APPS = stringSetPreferencesKey("pinned_apps") val DISCONNECTED_APPS = stringSetPreferencesKey("disconnected_apps") } @@ -65,6 +66,11 @@ class DataStoreSettingsRepository } ?: ServiceTier.PRIORITY } + override val geminiLiveVoice: Flow = + dataStore.data.map { preferences -> + GeminiLiveVoice.fromStoredValue(preferences[PreferencesKeys.GEMINI_LIVE_VOICE]) + } + override val pinnedApps: Flow> = dataStore.data.map { preferences -> preferences[PreferencesKeys.PINNED_APPS] ?: emptySet() } @@ -87,6 +93,12 @@ class DataStoreSettingsRepository dataStore.edit { preferences -> preferences[PreferencesKeys.SERVICE_TIER] = tier.name } } + override suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) { + dataStore.edit { preferences -> + preferences[PreferencesKeys.GEMINI_LIVE_VOICE] = voice.name + } + } + override suspend fun setAppPinned( packageName: String, pinned: Boolean, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt new file mode 100644 index 00000000..22ad48b9 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive + +/** Converts a [JsonElement] recursively into standard Kotlin primitive, [Map], or [List] values. */ +internal fun JsonElement.toPrimitive(): Any? { + return when (this) { + JsonNull -> null + is JsonPrimitive -> { + if (this.isString) return this.content + return this.content.toBooleanStrictOrNull() + ?: this.content.toLongOrNull() + ?: this.content.toDoubleOrNull() + } + is JsonObject -> this.mapValues { it.value.toPrimitive() } + is JsonArray -> this.map { it.toPrimitive() } + } +} + +/** Converts a nullable JSON arguments object into a Kotlin arguments map. */ +internal fun JsonObject?.toArgumentsMap(): Map { + if (this == null) return emptyMap() + return mapValues { (_, value) -> value.toPrimitive() } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt new file mode 100644 index 00000000..a8c778f5 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt @@ -0,0 +1,415 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.domain.LiveLlmEvent +import com.example.appfunctions.agent.domain.LiveLlmProvider +import com.example.appfunctions.agent.domain.LiveLlmSession +import com.example.appfunctions.agent.domain.LlmResponsePart +import com.example.appfunctions.agent.domain.ToolOutput +import io.ktor.client.HttpClient +import io.ktor.client.plugins.websocket.webSocket +import io.ktor.websocket.CloseReason +import io.ktor.websocket.Frame +import io.ktor.websocket.WebSocketSession +import io.ktor.websocket.close +import io.ktor.websocket.readBytes +import io.ktor.websocket.readText +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.isActive +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put +import java.time.LocalDate +import java.util.Base64 +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Gemini implementation of [LiveLlmProvider] over the `BidiGenerateContent` WebSocket API + * (`models/gemini-3.8-live`). + */ +@Singleton +class GeminiLiveProviderImpl + @Inject + constructor( + private val httpClient: HttpClient, + private val toolConverter: GeminiToolConverter, + private val settingsRepository: SettingsRepository, + ) : LiveLlmProvider { + internal var webSocketConnector: + suspend (wsUrl: String, block: suspend WebSocketSession.() -> Unit) -> Unit = + { wsUrl, block -> + httpClient.webSocket(urlString = wsUrl, block = block) + } + + override suspend fun connect( + apiKey: String, + tools: List, + block: suspend (LiveLlmSession) -> Unit, + ) { + val selectedVoice = settingsRepository.geminiLiveVoice.first() + val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" + webSocketConnector(wsUrl) { + val setupMessage = buildSetupMessage(tools = tools, voice = selectedVoice) + Log.d( + TAG, + "Sending Gemini Live setup for model $LIVE_MODEL_NAME (voice=${selectedVoice.apiName})", + ) + send(Frame.Text(setupMessage.toString())) + + val firstFrameText = + receiveNextJsonText() + ?: throw IllegalStateException( + "WebSocket closed before setupComplete was received", + ) + val firstJson = Json.parseToJsonElement(firstFrameText).jsonObject + if (!firstJson.containsKey("setupComplete")) { + throw IllegalStateException("Unexpected setup response: $firstFrameText") + } + + val session = createSession(webSocketSession = this, tools = tools) + try { + block(session) + } finally { + try { + close( + CloseReason( + CloseReason.Codes.NORMAL, + "Session ended by client", + ), + ) + } catch (e: Exception) { + Log.w(TAG, "Failed to close Gemini Live WebSocket cleanly", e) + } + } + } + } + + private fun createSession( + webSocketSession: WebSocketSession, + tools: List, + ): LiveLlmSession = + object : LiveLlmSession { + override val events: Flow = + flow { + while (currentCoroutineContext().isActive && webSocketSession.isActive) { + val messageText = webSocketSession.receiveNextJsonText() ?: break + val serverJson = + try { + Json.parseToJsonElement(messageText).jsonObject + } catch (e: Exception) { + Log.w(TAG, "Failed to parse Gemini Live server frame", e) + continue + } + + for (event in parseServerMessage(serverJson, tools)) { + emit(event) + } + } + } + + override suspend fun sendAudio(pcmBytes: ByteArray) { + val realtimeInput = buildRealtimeAudioMessage(pcmBytes) + webSocketSession.send(Frame.Text(realtimeInput.toString())) + } + + override suspend fun sendToolResponses(outputs: List) { + val toolResponseMessage = buildToolResponseMessage(outputs, tools) + Log.d(TAG, "Sending Gemini Live toolResponse: $toolResponseMessage") + webSocketSession.send(Frame.Text(toolResponseMessage.toString())) + } + } + + private suspend fun WebSocketSession.receiveNextJsonText(): String? { + while (isActive) { + val frame = + try { + incoming.receive() + } catch (_: Exception) { + return null + } + when (frame) { + is Frame.Text -> return frame.readText() + is Frame.Binary -> return frame.readBytes().decodeToString() + is Frame.Close -> return null + else -> continue + } + } + return null + } + + internal fun parseServerMessage( + serverJson: JsonObject, + tools: List, + ): List { + val events = mutableListOf() + + val serverContent = serverJson["serverContent"]?.jsonObject + if (serverContent != null) { + val interrupted = + serverContent["interrupted"]?.jsonPrimitive?.booleanOrNull ?: false + if (interrupted) { + events.add(LiveLlmEvent.Interrupted) + } + + val inputTranscription = + serverContent["inputTranscription"] + ?.jsonObject + ?.get("text") + ?.jsonPrimitive + ?.contentOrNull + if (!inputTranscription.isNullOrEmpty()) { + events.add(LiveLlmEvent.InputTranscription(inputTranscription)) + } + + val outputTranscription = + serverContent["outputTranscription"] + ?.jsonObject + ?.get("text") + ?.jsonPrimitive + ?.contentOrNull + if (!outputTranscription.isNullOrEmpty()) { + events.add(LiveLlmEvent.OutputTranscription(outputTranscription)) + } + + val parts = serverContent["modelTurn"]?.jsonObject?.get("parts")?.jsonArray + if (parts != null) { + for (part in parts) { + val inlineData = part.jsonObject["inlineData"]?.jsonObject ?: continue + val mimeType = + inlineData["mimeType"]?.jsonPrimitive?.contentOrNull ?: "" + val base64Data = + inlineData["data"]?.jsonPrimitive?.contentOrNull ?: continue + if (mimeType.startsWith("audio/pcm")) { + val pcmBytes = Base64.getMimeDecoder().decode(base64Data) + events.add(LiveLlmEvent.AudioChunk(pcmBytes)) + } + } + } + + val turnComplete = + serverContent["turnComplete"]?.jsonPrimitive?.booleanOrNull ?: false + if (turnComplete) { + events.add(LiveLlmEvent.TurnComplete) + } + } + + val toolCallObj = serverJson["toolCall"]?.jsonObject + if (toolCallObj != null) { + val functionCalls = + toolCallObj["functionCalls"]?.jsonArray ?: JsonArray(emptyList()) + val parsedCalls = + functionCalls.mapNotNull { callElement -> + val callObj = callElement.jsonObject + val callId = callObj["id"]?.jsonPrimitive?.contentOrNull ?: "" + val name = + callObj["name"]?.jsonPrimitive?.contentOrNull ?: return@mapNotNull null + val matchingTool = toolConverter.findToolByName(name, tools) + val argumentsMap = callObj["args"]?.jsonObject.toArgumentsMap() + + LlmResponsePart.ToolCall( + packageName = matchingTool?.packageName ?: "", + functionId = matchingTool?.id ?: name, + arguments = argumentsMap, + callId = callId, + ) + } + if (parsedCalls.isNotEmpty()) { + events.add(LiveLlmEvent.ToolCallRequest(parsedCalls)) + } + } + + return events + } + + internal fun buildSetupMessage( + tools: List, + voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + ): JsonObject { + val functionDeclarations = + toolConverter + .selectDistinctTools(tools) + .mapNotNull { tool -> + try { + toolConverter.convertForLiveApi(tool) + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null + } + } + + return buildJsonObject { + put( + "setup", + buildJsonObject { + put("model", LIVE_MODEL_NAME) + put( + "generationConfig", + buildJsonObject { + put( + "responseModalities", + buildJsonArray { add(JsonPrimitive("AUDIO")) }, + ) + put( + "speechConfig", + buildJsonObject { + put( + "voiceConfig", + buildJsonObject { + put( + "prebuiltVoiceConfig", + buildJsonObject { + put("voiceName", voice.apiName) + }, + ) + }, + ) + }, + ) + }, + ) + put( + "systemInstruction", + buildJsonObject { + put( + "parts", + buildJsonArray { + add( + buildJsonObject { + put("text", getSystemInstruction()) + }, + ) + }, + ) + }, + ) + if (functionDeclarations.isNotEmpty()) { + put( + "tools", + buildJsonArray { + add( + buildJsonObject { + put( + "functionDeclarations", + JsonArray(functionDeclarations), + ) + }, + ) + }, + ) + } + put("inputAudioTranscription", buildJsonObject {}) + put("outputAudioTranscription", buildJsonObject {}) + }, + ) + } + } + + internal fun buildRealtimeAudioMessage(pcmBytes: ByteArray): JsonObject { + val base64Audio = Base64.getEncoder().encodeToString(pcmBytes) + return buildJsonObject { + put( + "realtimeInput", + buildJsonObject { + put( + "audio", + buildJsonObject { + put("mimeType", INPUT_AUDIO_MIME_TYPE) + put("data", base64Audio) + }, + ) + }, + ) + } + } + + internal fun buildToolResponseMessage( + toolOutputs: List, + tools: List, + ): JsonObject { + return buildJsonObject { + put( + "toolResponse", + buildJsonObject { + put( + "functionResponses", + buildJsonArray { + for (output in toolOutputs) { + val mappedName = + toolConverter.resolveToolName(output.functionId, tools) + val parsedResult = + try { + Json.parseToJsonElement(output.result) + } catch (_: Exception) { + JsonPrimitive(output.result) + } + add( + buildJsonObject { + if (output.callId.isNotEmpty()) { + put("id", output.callId) + } + put("name", mappedName) + put( + "response", + buildJsonObject { + put("result", parsedResult) + }, + ) + }, + ) + } + }, + ) + }, + ) + } + } + + private fun getSystemInstruction(): String { + val currentDate = LocalDate.now().toString() + return """ + You are an AI voice assistant running on Android using Gemini Live. Today's date is $currentDate. + Default to English for understanding and replying unless the user clearly speaks in or asks to switch to another language. + Always reply to the user using concise, friendly, natural spoken language. + You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device. + Execute user-requested actions directly without claiming an on-screen confirmation card or button is shown; if you ever need confirmation before a sensitive action, ask the user verbally. + When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence. + """.trimIndent() + } + + companion object { + private const val TAG = "GeminiLiveProvider" + const val LIVE_MODEL_NAME = "models/gemini-3.8-live" + private const val INPUT_AUDIO_MIME_TYPE = "audio/pcm;rate=16000" + private const val GEMINI_LIVE_WS_URL = + "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt new file mode 100644 index 00000000..b9ea6c17 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +/** + * Prebuilt studio voices supported by Gemini Live (`models/gemini-3.8-live`) via + * `speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName`. + */ +enum class GeminiLiveVoice( + /** Voice identifier sent in `prebuiltVoiceConfig.voiceName` and shown in Settings. */ + val apiName: String, + /** Short vocal style descriptor for the voice. */ + val style: String, +) { + ZEPHYR("Zephyr", "Bright"), + PUCK("Puck", "Upbeat"), + CHARON("Charon", "Informative"), + KORE("Kore", "Firm"), + FENRIR("Fenrir", "Excitable"), + LEDA("Leda", "Youthful"), + ORUS("Orus", "Firm"), + AOEDE("Aoede", "Breezy"), + CALLIRRHOE("Callirrhoe", "Easy-going"), + AUTONOE("Autonoe", "Bright"), + ENCELADUS("Enceladus", "Breathy"), + IAPETUS("Iapetus", "Clear"), + UMBRIEL("Umbriel", "Easy-going"), + ALGIEBA("Algieba", "Smooth"), + DESPINA("Despina", "Smooth"), + ERINOME("Erinome", "Clear"), + ALGENIB("Algenib", "Gravelly"), + RASALGETHI("Rasalgethi", "Informative"), + LAOMEDEIA("Laomedeia", "Upbeat"), + ACHERNAR("Achernar", "Soft"), + ALNILAM("Alnilam", "Firm"), + SCHEDAR("Schedar", "Even"), + GACRUX("Gacrux", "Mature"), + PULCHERRIMA("Pulcherrima", "Forward"), + ACHIRD("Achird", "Friendly"), + ZUBENELGENUBI("Zubenelgenubi", "Casual"), + VINDEMIATRIX("Vindemiatrix", "Gentle"), + SADACHBIA("Sadachbia", "Lively"), + SADALTAGER("Sadaltager", "Knowledgeable"), + SULAFAT("Sulafat", "Warm"), + ; + + companion object { + val DEFAULT: GeminiLiveVoice = ZEPHYR + + /** Resolves a persisted preference string back to a [GeminiLiveVoice], falling back to [DEFAULT]. */ + fun fromStoredValue(value: String?): GeminiLiveVoice { + if (value == null) return DEFAULT + return entries.find { + it.name.equals(value, ignoreCase = true) || + it.apiName.equals(value, ignoreCase = true) + } ?: DEFAULT + } + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index a6da3257..e89546ff 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -32,9 +32,8 @@ import io.ktor.http.ContentType import io.ktor.http.contentType import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonArray @@ -119,9 +118,8 @@ class GeminiProviderImpl serviceTier: ServiceTier, ): LlmResponse { val convertedTools = - tools - .sortedByDescending { it.id.startsWith(it.packageName) } - .distinctBy { toolConverter.getToolName(it) } + toolConverter + .selectDistinctTools(tools) .mapNotNull { tool -> try { buildJsonObject { @@ -148,15 +146,10 @@ class GeminiProviderImpl when (input) { is LlmInput.ToolResponse -> { val inputElement = - kotlinx.serialization.json.buildJsonArray { + buildJsonArray { input.outputs.forEach { output -> - val matchingTool = tools.find { it.id == output.functionId } val mappedName = - if (matchingTool != null) { - toolConverter.getToolName(matchingTool) - } else { - output.functionId - } + toolConverter.resolveToolName(output.functionId, tools) add( buildJsonObject { @@ -178,7 +171,7 @@ class GeminiProviderImpl } if (convertedTools.isNotEmpty()) { - put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) + put(KEY_TOOLS, JsonArray(convertedTools)) } if (previousInteractionId != null) { @@ -259,19 +252,13 @@ class GeminiProviderImpl stepObj[KEY_NAME]?.jsonPrimitive?.content ?: return LlmResponse.Error("Called function without a name") val matchingTool = - tools.find { tool -> toolConverter.getToolName(tool) == name } + toolConverter.findToolByName(name, tools) ?: return LlmResponse.Error("Called unknown function: $name") val packageName = matchingTool.packageName val functionId = matchingTool.id - val args = stepObj[KEY_ARGUMENTS]?.jsonObject - val argumentsMap = mutableMapOf() - if (args != null) { - for ((key, value) in args) { - argumentsMap[key] = value.toPrimitive() - } - } + val argumentsMap = stepObj[KEY_ARGUMENTS]?.jsonObject.toArgumentsMap() val callId = stepObj["id"]?.jsonPrimitive?.content ?: return LlmResponse.Error( @@ -297,19 +284,6 @@ class GeminiProviderImpl ) } - private fun JsonElement.toPrimitive(): Any? { - return when (this) { - is JsonPrimitive -> { - if (this.isString) return this.content - return this.content.toBooleanStrictOrNull() - ?: this.content.toLongOrNull() - ?: this.content.toDoubleOrNull() - } - is JsonObject -> this.mapValues { it.value.toPrimitive() } - is JsonArray -> this.map { it.toPrimitive() } - } - } - private fun getSystemInstruction(): String { val currentDate = LocalDate.now().toString() return """ diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index cc109813..d7c85303 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -45,10 +45,71 @@ class GeminiToolConverter return convertToolToGeminiSchema(tool) } + /** + * Converts an [AppFunctionMetadata] into a Gemini Live (`BidiGenerateContent`) + * `FunctionDeclaration` JSON object, normalizing OpenAPI schema fields. + */ + fun convertForLiveApi(tool: AppFunctionMetadata): JsonObject { + val baseSchema = convertToolToGeminiSchema(tool) + return normalizeForLiveApi(baseSchema) + } + + private fun normalizeForLiveApi(obj: JsonObject): JsonObject { + val typeValue = (obj[KEY_TYPE] as? JsonPrimitive)?.content + return buildJsonObject { + for ((key, value) in obj) { + when { + key == KEY_FORMAT && + (value as? JsonPrimitive)?.content == VALUE_FILE_REFERENCE -> { + // Omit Interactions-API-specific file_reference format in BidiGenerateContent + } + key == KEY_ENUMS -> { + if (typeValue == VALUE_STRING) { + put(KEY_ENUM_LIVE, value) + } + } + value is JsonObject -> { + put(key, normalizeForLiveApi(value)) + } + else -> { + put(key, value) + } + } + } + } + } + override fun getToolName(tool: AppFunctionMetadata): String { return getGeminiFunctionName(tool) } + /** + * Deduplicates [tools] by their converted Gemini function name, prioritizing functions + * whose ID starts with their package name. + */ + fun selectDistinctTools(tools: List): List = + tools + .sortedByDescending { it.id.startsWith(it.packageName) } + .distinctBy { getToolName(it) } + + /** Finds the [AppFunctionMetadata] in [tools] matching a converted Gemini function [name]. */ + fun findToolByName( + name: String, + tools: List, + ): AppFunctionMetadata? = tools.find { getToolName(it) == name } + + /** + * Resolves the converted Gemini function name for [functionId] from [tools], falling back + * to [functionId] if no matching tool metadata is present. + */ + fun resolveToolName( + functionId: String, + tools: List, + ): String { + val matchingTool = tools.find { it.id == functionId } ?: return functionId + return getToolName(matchingTool) + } + private fun convertToolToGeminiSchema(tool: AppFunctionMetadata): JsonObject { val combinedName = getGeminiFunctionName(tool) return buildJsonObject { @@ -295,6 +356,7 @@ class GeminiToolConverter private const val VALUE_ARRAY = "array" private const val KEY_ITEMS = "items" private const val KEY_ENUMS = "enums" + private const val KEY_ENUM_LIVE = "enum" private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]") private const val MAX_NAME_LENGTH = 64 } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt index 0475b8c0..8d543c37 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt @@ -15,21 +15,29 @@ */ package com.example.appfunctions.agent.data +import com.example.appfunctions.agent.domain.LiveLlmProvider import com.example.appfunctions.agent.domain.LlmProvider import com.example.appfunctions.agent.domain.LlmProviderFactory import javax.inject.Inject import javax.inject.Singleton -/** Implementation of LlmProviderFactory. */ +/** Implementation of [LlmProviderFactory]. */ @Singleton class LlmProviderFactoryImpl @Inject constructor( private val geminiProvider: GeminiProviderImpl, + private val geminiLiveProvider: GeminiLiveProviderImpl, ) : LlmProviderFactory { override fun getProvider(name: LlmProviderName): LlmProvider { return when (name) { LlmProviderName.GEMINI -> geminiProvider } } + + override fun getLiveProvider(name: LlmProviderName): LiveLlmProvider { + return when (name) { + LlmProviderName.GEMINI -> geminiLiveProvider + } + } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt index 0feefb9d..92a2ca13 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt @@ -37,6 +37,12 @@ interface SettingsRepository { /** Sets the service tier requested for LLM API calls. */ suspend fun setServiceTier(tier: ServiceTier) + /** Flow of the prebuilt voice selected for Gemini Live sessions. */ + val geminiLiveVoice: Flow + + /** Sets the prebuilt voice selected for Gemini Live sessions. */ + suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) + /** Flow of pinned app package names. */ val pinnedApps: Flow> diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index f92fae70..2e0cd274 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -25,11 +25,13 @@ import com.example.appfunctions.agent.data.DataStoreSettingsRepository import com.example.appfunctions.agent.data.FileDownloadHelper import com.example.appfunctions.agent.data.FileDownloadHelperImpl import com.example.appfunctions.agent.data.InMemoryPendingIntentRepository +import com.example.appfunctions.agent.data.LlmProviderFactoryImpl import com.example.appfunctions.agent.data.PendingIntentRepository import com.example.appfunctions.agent.data.RoomChatRepository import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.AppDatabase import com.example.appfunctions.agent.data.db.dao.ChatDao +import com.example.appfunctions.agent.domain.LlmProviderFactory import dagger.Binds import dagger.Module import dagger.Provides @@ -40,6 +42,7 @@ import io.ktor.client.HttpClient import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.websocket.WebSockets import io.ktor.serialization.kotlinx.json.json import kotlinx.serialization.json.Json import javax.inject.Singleton @@ -64,9 +67,7 @@ abstract class DataModule { @Binds @Singleton - abstract fun bindLlmProviderFactory( - impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl, - ): com.example.appfunctions.agent.domain.LlmProviderFactory + abstract fun bindLlmProviderFactory(impl: LlmProviderFactoryImpl): LlmProviderFactory @Binds @Singleton @@ -115,6 +116,7 @@ abstract class DataModule { ) } install(HttpTimeout) { socketTimeoutMillis = 30000 } + install(WebSockets) } } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt new file mode 100644 index 00000000..2b1c2025 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import androidx.appfunctions.metadata.AppFunctionMetadata +import kotlinx.coroutines.flow.Flow + +/** Interface for LLM providers supporting bidirectional realtime audio and tool sessions. */ +interface LiveLlmProvider { + /** + * Connects to the live LLM backend, completes session initialization with [tools], and invokes + * [block] with an active [LiveLlmSession] until [block] returns or is cancelled. + */ + suspend fun connect( + apiKey: String, + tools: List, + block: suspend (LiveLlmSession) -> Unit, + ) +} + +/** Represents an active bidirectional voice and tool session with an LLM provider. */ +interface LiveLlmSession { + /** Sequential stream of server-originated events for this live session. */ + val events: Flow + + /** Sends a chunk of raw 16-bit 16kHz mono PCM user microphone audio to the model. */ + suspend fun sendAudio(pcmBytes: ByteArray) + + /** Sends tool execution [outputs] back to the model in response to a [LiveLlmEvent.ToolCallRequest]. */ + suspend fun sendToolResponses(outputs: List) +} + +/** Provider-agnostic events emitted by a [LiveLlmSession]. */ +sealed class LiveLlmEvent { + /** Indicates the user interrupted the model's current audio response. */ + object Interrupted : LiveLlmEvent() + + /** Incremental speech-to-text transcription of the user's audio input. */ + data class InputTranscription(val text: String) : LiveLlmEvent() + + /** Incremental transcription of the model's spoken response. */ + data class OutputTranscription(val text: String) : LiveLlmEvent() + + /** Raw PCM audio chunk generated by the model for playback. */ + data class AudioChunk(val pcmBytes: ByteArray) : LiveLlmEvent() + + /** Indicates the model finished its current conversational turn. */ + object TurnComplete : LiveLlmEvent() + + /** Requests execution of one or more [calls] before continuing the live conversation. */ + data class ToolCallRequest(val calls: List) : LiveLlmEvent() +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt index 054c287a..c9ae94f8 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt @@ -17,8 +17,11 @@ package com.example.appfunctions.agent.domain import com.example.appfunctions.agent.data.LlmProviderName -/** Factory interface to get the correct LlmProvider. */ +/** Factory interface to get the correct [LlmProvider] or [LiveLlmProvider]. */ interface LlmProviderFactory { - /** Returns the LlmProvider instance for the given name. */ + /** Returns the [LlmProvider] instance for the given [name]. */ fun getProvider(name: LlmProviderName): LlmProvider + + /** Returns the [LiveLlmProvider] instance for the given [name]. */ + fun getLiveProvider(name: LlmProviderName): LiveLlmProvider } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt new file mode 100644 index 00000000..eaebf3ad --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt @@ -0,0 +1,396 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import com.example.appfunctions.agent.domain.LiveLlmEvent +import com.example.appfunctions.agent.domain.ToolOutput +import io.ktor.client.HttpClient +import io.ktor.websocket.Frame +import io.ktor.websocket.WebSocketSession +import io.ktor.websocket.readText +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkStatic +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Before +import org.junit.Test +import java.util.Base64 + +class GeminiLiveProviderImplTest { + private lateinit var settingsRepository: SettingsRepository + private lateinit var provider: GeminiLiveProviderImpl + + @Before + fun setUp() { + mockkStatic(Log::class) + every { Log.d(any(), any()) } returns 0 + every { Log.w(any(), any()) } returns 0 + every { Log.e(any(), any(), any()) } returns 0 + + settingsRepository = mockk(relaxed = true) + every { settingsRepository.geminiLiveVoice } returns flowOf(GeminiLiveVoice.DEFAULT) + + provider = + GeminiLiveProviderImpl( + httpClient = mockk(relaxed = true), + toolConverter = GeminiToolConverter(), + settingsRepository = settingsRepository, + ) + } + + @Test + fun `buildSetupMessage configures gemini-3_8-live model, audio modality, transcriptions, and tools`() { + val tool = createSampleNoteTool() + + val setupEnvelope = provider.buildSetupMessage(listOf(tool)) + val setup = setupEnvelope["setup"]!!.jsonObject + + assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content) + + val modalities = + setup["generationConfig"]!! + .jsonObject["responseModalities"]!! + .jsonArray + .map { it.jsonPrimitive.content } + assertEquals(listOf("AUDIO"), modalities) + + val voiceName = + setup["generationConfig"]!! + .jsonObject["speechConfig"]!! + .jsonObject["voiceConfig"]!! + .jsonObject["prebuiltVoiceConfig"]!! + .jsonObject["voiceName"]!! + .jsonPrimitive + .content + assertEquals("Zephyr", voiceName) + + assertTrue(setup.containsKey("inputAudioTranscription")) + assertTrue(setup.containsKey("outputAudioTranscription")) + val systemInstructionText = + setup["systemInstruction"]!! + .jsonObject["parts"]!! + .jsonArray + .first() + .jsonObject["text"]!! + .jsonPrimitive + .content + assertTrue( + systemInstructionText.contains( + "Default to English for understanding and replying unless the user clearly " + + "speaks in or asks to switch to another language.", + ), + ) + assertTrue( + systemInstructionText.contains( + "Execute user-requested actions directly without claiming an on-screen " + + "confirmation card or button is shown", + ), + ) + assertFalse(systemInstructionText.contains("confirmation_required")) + + val declarations = + setup["tools"]!! + .jsonArray + .first() + .jsonObject["functionDeclarations"]!! + .jsonArray + assertEquals(1, declarations.size) + assertEquals( + "com_example_notes_createNote", + declarations.first().jsonObject["name"]!!.jsonPrimitive.content, + ) + } + + @Test + fun `connect throws IllegalStateException when WebSocket closes before setupComplete`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.close() + + provider.webSocketConnector = { _, block -> + coroutineScope { + createMockWebSocketSession(coroutineContext, incomingFrames).block() + } + } + + try { + provider.connect(apiKey = "key-123", tools = emptyList()) {} + fail("Expected IllegalStateException") + } catch (e: IllegalStateException) { + assertTrue(e.message!!.contains("setupComplete")) + } + } + + @Test + fun `connect sends selected GeminiLiveVoice from SettingsRepository in setup message`() = + runTest { + every { settingsRepository.geminiLiveVoice } returns flowOf(GeminiLiveVoice.AOEDE) + + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.close() + + val sentFrames = mutableListOf() + provider.webSocketConnector = { _, block -> + coroutineScope { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ).block() + } + } + + provider.connect(apiKey = "key-123", tools = emptyList()) {} + + val setupVoiceName = + Json.parseToJsonElement(sentFrames.first()) + .jsonObject["setup"]!! + .jsonObject["generationConfig"]!! + .jsonObject["speechConfig"]!! + .jsonObject["voiceConfig"]!! + .jsonObject["prebuiltVoiceConfig"]!! + .jsonObject["voiceName"]!! + .jsonPrimitive + .content + assertEquals("Aoede", setupVoiceName) + } + + @Test + fun `connect exchanges realtimeInput audio, parses server events, and sends mapped toolResponse`() = + runTest { + val tool = createSampleNoteTool() + val expectedPcm = byteArrayOf(10, 20, 30, 40) + val encodedPcm = Base64.getEncoder().encodeToString(expectedPcm) + + val incomingFrames = Channel(Channel.UNLIMITED) + val sentFrames = mutableListOf() + var capturedWsUrl: String? = null + + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.trySend( + Frame.Text( + """ + { + "serverContent": { + "interrupted": true, + "inputTranscription": { "text": "Create a note" }, + "outputTranscription": { "text": "Creating note" }, + "modelTurn": { + "parts": [ + { + "inlineData": { + "mimeType": "audio/pcm;rate=24000", + "data": "$encodedPcm" + } + } + ] + }, + "turnComplete": true + } + } + """.trimIndent(), + ), + ) + incomingFrames.trySend( + Frame.Text( + """ + { + "toolCall": { + "functionCalls": [ + { + "id": "call-1", + "name": "com_example_notes_createNote", + "args": { + "title": "Shopping", + "meta": { "pinned": true, "priority": 2 } + } + }, + { + "id": "call-unknown", + "name": "hallucinated_tool", + "args": {} + } + ] + } + } + """.trimIndent(), + ), + ) + incomingFrames.close() + + provider.webSocketConnector = { wsUrl, block -> + capturedWsUrl = wsUrl + coroutineScope { + val ws = + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ) + ws.block() + } + } + + val receivedEvents = mutableListOf() + provider.connect(apiKey = "secret-key", tools = listOf(tool)) { session -> + session.sendAudio(byteArrayOf(1, 2, 3)) + receivedEvents.addAll(session.events.toList()) + session.sendToolResponses( + listOf( + ToolOutput( + functionId = tool.id, + callId = "call-1", + result = """{"noteId":"n-1"}""", + ), + ToolOutput( + functionId = "hallucinated_tool", + callId = "call-unknown", + result = "Tool not found: hallucinated_tool", + ), + ), + ) + } + + assertTrue(capturedWsUrl!!.endsWith("?key=secret-key")) + assertEquals(3, sentFrames.size) + + // Frame 0: setup + assertTrue(Json.parseToJsonElement(sentFrames[0]).jsonObject.containsKey("setup")) + + // Frame 1: realtimeInput audio + val realtimeAudio = + Json.parseToJsonElement(sentFrames[1]) + .jsonObject["realtimeInput"]!! + .jsonObject["audio"]!! + .jsonObject + assertEquals("audio/pcm;rate=16000", realtimeAudio["mimeType"]!!.jsonPrimitive.content) + assertArrayEquals( + byteArrayOf(1, 2, 3), + Base64.getDecoder().decode(realtimeAudio["data"]!!.jsonPrimitive.content), + ) + + // Received events + assertEquals(6, receivedEvents.size) + assertEquals(LiveLlmEvent.Interrupted, receivedEvents[0]) + assertEquals(LiveLlmEvent.InputTranscription("Create a note"), receivedEvents[1]) + assertEquals(LiveLlmEvent.OutputTranscription("Creating note"), receivedEvents[2]) + assertArrayEquals(expectedPcm, (receivedEvents[3] as LiveLlmEvent.AudioChunk).pcmBytes) + assertEquals(LiveLlmEvent.TurnComplete, receivedEvents[4]) + + val toolCallReq = receivedEvents[5] as LiveLlmEvent.ToolCallRequest + assertEquals(2, toolCallReq.calls.size) + assertEquals("com.example.notes", toolCallReq.calls[0].packageName) + assertEquals("com.example.notes.createNote", toolCallReq.calls[0].functionId) + assertEquals("Shopping", toolCallReq.calls[0].arguments["title"]) + @Suppress("UNCHECKED_CAST") + val meta = toolCallReq.calls[0].arguments["meta"] as Map + assertEquals(true, meta["pinned"]) + assertEquals(2L, meta["priority"]) + + assertEquals("", toolCallReq.calls[1].packageName) + assertEquals("hallucinated_tool", toolCallReq.calls[1].functionId) + + // Frame 2: toolResponse + val responses = + Json.parseToJsonElement(sentFrames[2]) + .jsonObject["toolResponse"]!! + .jsonObject["functionResponses"]!! + .jsonArray + assertEquals(2, responses.size) + assertEquals( + "com_example_notes_createNote", + responses[0].jsonObject["name"]!!.jsonPrimitive.content, + ) + assertEquals( + "n-1", + responses[0] + .jsonObject["response"]!! + .jsonObject["result"]!! + .jsonObject["noteId"]!! + .jsonPrimitive + .content, + ) + assertEquals( + "hallucinated_tool", + responses[1].jsonObject["name"]!!.jsonPrimitive.content, + ) + } + + private fun createMockWebSocketSession( + context: kotlin.coroutines.CoroutineContext, + incomingFrames: Channel, + onSendText: (String) -> Unit = {}, + ): WebSocketSession { + val outgoingFrames = Channel(Channel.UNLIMITED) + return mockk(relaxed = true).apply { + every { coroutineContext } returns context + every { incoming } returns incomingFrames + every { outgoing } returns outgoingFrames + coEvery { send(any()) } coAnswers { + val frame = firstArg() + if (frame is Frame.Text) { + onSendText(frame.readText()) + } + } + } + } + + private fun createSampleNoteTool(): AppFunctionMetadata = + AppFunctionMetadata( + id = "com.example.notes.createNote", + packageName = "com.example.notes", + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "title", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "Note title", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Creates a note", + deprecation = null, + ) +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt index 62cb16ab..8388855d 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt @@ -520,4 +520,68 @@ class GeminiToolConverterTest { assertEquals(expectedJson, schema) } + + @Test + fun convertForLiveApi_normalizesEnumsAndStripsFileReferenceFormat() { + val enumParam = + AppFunctionParameterMetadata( + name = "mode", + isRequired = true, + dataType = + AppFunctionStringTypeMetadata(isNullable = false, enumValues = setOf("A", "B")), + description = "Mode parameter", + ) + val uriParam = + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = false, + dataType = AppFunctionStringTypeMetadata(isNullable = true), + description = "Wallpaper URI", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(enumParam, uriParam), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val liveSchema = converter.convertForLiveApi(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["A", "B"], + "description": "Mode parameter" + }, + "wallpaperUri": { + "type": "string", + "description": "Wallpaper URI" + } + }, + "required": ["mode"] + } + } + """, + ) + + assertEquals(expectedJson, liveSchema) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt index ea05a1fb..b18b913c 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt @@ -77,4 +77,19 @@ class SettingsRepositoryTest { val value = repository.serviceTier.first() assertEquals(ServiceTier.PRIORITY, value) } + + @Test + fun geminiLiveVoice_defaultsToZephyr() = + runTest { + val value = repository.geminiLiveVoice.first() + assertEquals(GeminiLiveVoice.ZEPHYR, value) + } + + @Test + fun setGeminiLiveVoice_persistsValue() = + runTest { + repository.setGeminiLiveVoice(GeminiLiveVoice.AOEDE) + val value = repository.geminiLiveVoice.first() + assertEquals(GeminiLiveVoice.AOEDE, value) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt index 7ea3b0be..e36b55b0 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt @@ -15,6 +15,7 @@ */ package com.example.appfunctions.agent.ui.screens.agentdemo +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.ServiceTier import com.example.appfunctions.agent.data.SettingsRepository @@ -31,6 +32,9 @@ class FakeSettingsRepository : SettingsRepository { private val _serviceTier = MutableStateFlow(ServiceTier.STANDARD) override val serviceTier: Flow = _serviceTier + private val _geminiLiveVoice = MutableStateFlow(GeminiLiveVoice.DEFAULT) + override val geminiLiveVoice: Flow = _geminiLiveVoice + private val _pinnedApps = MutableStateFlow>(emptySet()) override val pinnedApps: Flow> = _pinnedApps @@ -46,6 +50,10 @@ class FakeSettingsRepository : SettingsRepository { _serviceTier.value = tier } + override suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) { + _geminiLiveVoice.value = voice + } + override suspend fun setAppPinned( packageName: String, pinned: Boolean, From 368f43c421cd87347f464870244e9ca5b4a98f9f Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Mon, 2 Nov 2026 17:38:56 +0100 Subject: [PATCH 2/3] refactor(agent): address PR #69 review comments on LiveLlmProvider and GeminiLiveProviderImpl --- .../agent/data/GeminiLiveProviderImpl.kt | 541 +++++++++++---- .../agent/data/GeminiLiveToolConverter.kt | 78 +++ .../agent/data/GeminiProviderImpl.kt | 4 + .../agent/data/GeminiToolConverter.kt | 62 -- .../appfunctions/agent/di/DataModule.kt | 7 +- .../JsonExtensions.kt} | 2 +- .../agent/domain/LiveLlmProvider.kt | 80 ++- .../agent/domain/appfunction/ToolConverter.kt | 27 + .../agent/data/GeminiLiveProviderImplTest.kt | 628 +++++++++++++----- .../agent/data/GeminiLiveToolConverterTest.kt | 207 ++++++ .../agent/data/GeminiToolConverterTest.kt | 64 -- .../agent/data/SettingsRepositoryTest.kt | 13 + .../domain/appfunction/ToolConverterTest.kt | 90 +++ 13 files changed, 1349 insertions(+), 454 deletions(-) create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveToolConverter.kt rename agent/app/src/main/java/com/example/appfunctions/agent/{data/GeminiJsonUtils.kt => domain/JsonExtensions.kt} (97%) create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ToolConverterTest.kt diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt index a8c778f5..d4c67a8e 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt @@ -16,25 +16,35 @@ package com.example.appfunctions.agent.data import android.util.Log +import androidx.annotation.VisibleForTesting import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.domain.DEFAULT_LIVE_OUTPUT_SAMPLE_RATE_HZ import com.example.appfunctions.agent.domain.LiveLlmEvent import com.example.appfunctions.agent.domain.LiveLlmProvider import com.example.appfunctions.agent.domain.LiveLlmSession import com.example.appfunctions.agent.domain.LlmResponsePart import com.example.appfunctions.agent.domain.ToolOutput +import com.example.appfunctions.agent.domain.appfunction.findToolByName +import com.example.appfunctions.agent.domain.appfunction.resolveToolName +import com.example.appfunctions.agent.domain.appfunction.selectDistinctTools +import com.example.appfunctions.agent.domain.toArgumentsMap import io.ktor.client.HttpClient -import io.ktor.client.plugins.websocket.webSocket +import io.ktor.client.plugins.websocket.webSocketSession import io.ktor.websocket.CloseReason import io.ktor.websocket.Frame import io.ktor.websocket.WebSocketSession import io.ktor.websocket.close import io.ktor.websocket.readBytes +import io.ktor.websocket.readReason import io.ktor.websocket.readText +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.channels.ClosedReceiveChannelException import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.flow import kotlinx.coroutines.isActive +import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonObject @@ -61,55 +71,63 @@ class GeminiLiveProviderImpl @Inject constructor( private val httpClient: HttpClient, - private val toolConverter: GeminiToolConverter, - private val settingsRepository: SettingsRepository, + private val toolConverter: GeminiLiveToolConverter, ) : LiveLlmProvider { - internal var webSocketConnector: - suspend (wsUrl: String, block: suspend WebSocketSession.() -> Unit) -> Unit = - { wsUrl, block -> - httpClient.webSocket(urlString = wsUrl, block = block) - } + private var webSocketConnector: suspend (wsUrl: String) -> WebSocketSession = { wsUrl -> + httpClient.webSocketSession(urlString = wsUrl) + } + + @VisibleForTesting + internal constructor( + httpClient: HttpClient, + toolConverter: GeminiLiveToolConverter, + webSocketConnector: suspend (wsUrl: String) -> WebSocketSession, + ) : this( + httpClient = httpClient, + toolConverter = toolConverter, + ) { + this.webSocketConnector = webSocketConnector + } override suspend fun connect( apiKey: String, tools: List, - block: suspend (LiveLlmSession) -> Unit, - ) { - val selectedVoice = settingsRepository.geminiLiveVoice.first() + voice: GeminiLiveVoice, + ): LiveLlmSession { val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" - webSocketConnector(wsUrl) { - val setupMessage = buildSetupMessage(tools = tools, voice = selectedVoice) + val webSocketSession = webSocketConnector(wsUrl) + try { + val setupMessage = buildSetupMessage(tools = tools, voice = voice) Log.d( TAG, - "Sending Gemini Live setup for model $LIVE_MODEL_NAME (voice=${selectedVoice.apiName})", + "Sending Gemini Live setup for model $LIVE_MODEL_NAME (voice=${voice.apiName})", ) - send(Frame.Text(setupMessage.toString())) + webSocketSession.send(Frame.Text(setupMessage.toString())) val firstFrameText = - receiveNextJsonText() - ?: throw IllegalStateException( - "WebSocket closed before setupComplete was received", - ) + when (val firstFrame = webSocketSession.receiveNextFrame()) { + is FrameReadResult.JsonText -> firstFrame.text + is FrameReadResult.NormalClose -> + throw IllegalStateException( + "WebSocket closed before $KEY_SETUP_COMPLETE was received", + ) + is FrameReadResult.ErrorClose -> + throw IllegalStateException( + "WebSocket failed before $KEY_SETUP_COMPLETE was received: ${firstFrame.message}", + firstFrame.cause, + ) + } val firstJson = Json.parseToJsonElement(firstFrameText).jsonObject - if (!firstJson.containsKey("setupComplete")) { + if (!firstJson.containsKey(KEY_SETUP_COMPLETE)) { throw IllegalStateException("Unexpected setup response: $firstFrameText") } - val session = createSession(webSocketSession = this, tools = tools) - try { - block(session) - } finally { - try { - close( - CloseReason( - CloseReason.Codes.NORMAL, - "Session ended by client", - ), - ) - } catch (e: Exception) { - Log.w(TAG, "Failed to close Gemini Live WebSocket cleanly", e) - } + return createSession(webSocketSession = webSocketSession, tools = tools) + } catch (t: Throwable) { + withContext(NonCancellable) { + closeWebSocketQuietly(webSocketSession) } + throw t } } @@ -121,12 +139,31 @@ class GeminiLiveProviderImpl override val events: Flow = flow { while (currentCoroutineContext().isActive && webSocketSession.isActive) { - val messageText = webSocketSession.receiveNextJsonText() ?: break + val messageText = + when (val frameResult = webSocketSession.receiveNextFrame()) { + is FrameReadResult.JsonText -> frameResult.text + is FrameReadResult.NormalClose -> break + is FrameReadResult.ErrorClose -> { + emit( + LiveLlmEvent.Error( + message = frameResult.message, + cause = frameResult.cause, + ), + ) + break + } + } val serverJson = try { Json.parseToJsonElement(messageText).jsonObject } catch (e: Exception) { Log.w(TAG, "Failed to parse Gemini Live server frame", e) + emit( + LiveLlmEvent.Error( + message = "Failed to parse Gemini Live server frame", + cause = e, + ), + ) continue } @@ -136,8 +173,15 @@ class GeminiLiveProviderImpl } } - override suspend fun sendAudio(pcmBytes: ByteArray) { - val realtimeInput = buildRealtimeAudioMessage(pcmBytes) + override suspend fun sendAudio( + pcmBytes: ByteArray, + sampleRateHz: Int, + ) { + val realtimeInput = + buildRealtimeAudioMessage( + pcmBytes = pcmBytes, + sampleRateHz = sampleRateHz, + ) webSocketSession.send(Frame.Text(realtimeInput.toString())) } @@ -146,111 +190,267 @@ class GeminiLiveProviderImpl Log.d(TAG, "Sending Gemini Live toolResponse: $toolResponseMessage") webSocketSession.send(Frame.Text(toolResponseMessage.toString())) } + + override suspend fun close() { + closeWebSocketQuietly(webSocketSession) + } } - private suspend fun WebSocketSession.receiveNextJsonText(): String? { + private suspend fun closeWebSocketQuietly(webSocketSession: WebSocketSession) { + try { + webSocketSession.close( + CloseReason( + CloseReason.Codes.NORMAL, + CLOSE_REASON_CLIENT_ENDED, + ), + ) + } catch (e: Exception) { + Log.w(TAG, "Failed to close Gemini Live WebSocket cleanly", e) + } + } + + private suspend fun WebSocketSession.receiveNextFrame(): FrameReadResult { while (isActive) { val frame = try { incoming.receive() - } catch (_: Exception) { - return null + } catch (_: ClosedReceiveChannelException) { + return FrameReadResult.NormalClose + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Log.w(TAG, "Gemini Live WebSocket transport error", e) + return FrameReadResult.ErrorClose( + message = "WebSocket transport error: ${e.message}", + cause = e, + ) } when (frame) { - is Frame.Text -> return frame.readText() - is Frame.Binary -> return frame.readBytes().decodeToString() - is Frame.Close -> return null + is Frame.Text -> return FrameReadResult.JsonText(frame.readText()) + is Frame.Binary -> return FrameReadResult.JsonText(frame.readBytes().decodeToString()) + is Frame.Close -> { + val reason = frame.readReason() + return if (reason != null && reason.code != CloseReason.Codes.NORMAL.code) { + Log.w( + TAG, + "Gemini Live WebSocket closed by server: code=${reason.code}, message=${reason.message}", + ) + FrameReadResult.ErrorClose( + message = + "WebSocket closed by server (code=${reason.code}): ${reason.message}", + ) + } else { + FrameReadResult.NormalClose + } + } else -> continue } } - return null + return FrameReadResult.NormalClose } - internal fun parseServerMessage( + private fun parseServerMessage( serverJson: JsonObject, tools: List, ): List { - val events = mutableListOf() - - val serverContent = serverJson["serverContent"]?.jsonObject - if (serverContent != null) { - val interrupted = - serverContent["interrupted"]?.jsonPrimitive?.booleanOrNull ?: false - if (interrupted) { - events.add(LiveLlmEvent.Interrupted) + return when { + KEY_SERVER_CONTENT in serverJson -> { + val serverContent = serverJson[KEY_SERVER_CONTENT]?.jsonObject + if (serverContent != null) { + parseServerContent(serverContent) + } else { + listOf( + LiveLlmEvent.Error( + "Invalid '$KEY_SERVER_CONTENT' payload in server message: $serverJson", + ), + ) + } } - - val inputTranscription = - serverContent["inputTranscription"] - ?.jsonObject - ?.get("text") - ?.jsonPrimitive - ?.contentOrNull - if (!inputTranscription.isNullOrEmpty()) { - events.add(LiveLlmEvent.InputTranscription(inputTranscription)) + KEY_TOOL_CALL in serverJson -> { + val toolCallObj = serverJson[KEY_TOOL_CALL]?.jsonObject + if (toolCallObj != null) { + parseToolCall(toolCallObj, tools) + } else { + listOf( + LiveLlmEvent.Error( + "Invalid '$KEY_TOOL_CALL' payload in server message: $serverJson", + ), + ) + } } + else -> emptyList() + } + } - val outputTranscription = - serverContent["outputTranscription"] - ?.jsonObject - ?.get("text") - ?.jsonPrimitive - ?.contentOrNull - if (!outputTranscription.isNullOrEmpty()) { - events.add(LiveLlmEvent.OutputTranscription(outputTranscription)) + private fun parseServerContent(serverContent: JsonObject): List { + return when { + serverContent[KEY_INTERRUPTED]?.jsonPrimitive?.booleanOrNull == true -> { + listOf(LiveLlmEvent.Interrupted) } - - val parts = serverContent["modelTurn"]?.jsonObject?.get("parts")?.jsonArray - if (parts != null) { - for (part in parts) { - val inlineData = part.jsonObject["inlineData"]?.jsonObject ?: continue - val mimeType = - inlineData["mimeType"]?.jsonPrimitive?.contentOrNull ?: "" - val base64Data = - inlineData["data"]?.jsonPrimitive?.contentOrNull ?: continue - if (mimeType.startsWith("audio/pcm")) { - val pcmBytes = Base64.getMimeDecoder().decode(base64Data) - events.add(LiveLlmEvent.AudioChunk(pcmBytes)) - } + serverContent[KEY_TURN_COMPLETE]?.jsonPrimitive?.booleanOrNull == true -> { + listOf(LiveLlmEvent.TurnComplete) + } + KEY_INPUT_TRANSCRIPTION in serverContent -> { + val text = + serverContent[KEY_INPUT_TRANSCRIPTION] + ?.jsonObject + ?.get(KEY_TEXT) + ?.jsonPrimitive + ?.contentOrNull + if (!text.isNullOrEmpty()) { + listOf(LiveLlmEvent.InputTranscription(text)) + } else { + emptyList() } } + KEY_OUTPUT_TRANSCRIPTION in serverContent -> { + val text = + serverContent[KEY_OUTPUT_TRANSCRIPTION] + ?.jsonObject + ?.get(KEY_TEXT) + ?.jsonPrimitive + ?.contentOrNull + if (!text.isNullOrEmpty()) { + listOf(LiveLlmEvent.OutputTranscription(text)) + } else { + emptyList() + } + } + KEY_MODEL_TURN in serverContent -> { + val modelTurn = serverContent[KEY_MODEL_TURN]?.jsonObject + if (modelTurn != null) { + parseModelTurn(modelTurn) + } else { + listOf( + LiveLlmEvent.Error( + "Invalid '$KEY_MODEL_TURN' payload in serverContent: $serverContent", + ), + ) + } + } + else -> emptyList() + } + } - val turnComplete = - serverContent["turnComplete"]?.jsonPrimitive?.booleanOrNull ?: false - if (turnComplete) { - events.add(LiveLlmEvent.TurnComplete) + private fun parseModelTurn(modelTurn: JsonObject): List { + val parts = + modelTurn[KEY_PARTS]?.jsonArray + ?: return listOf( + LiveLlmEvent.Error("modelTurn is missing '$KEY_PARTS' array: $modelTurn"), + ) + val events = mutableListOf() + for (part in parts) { + val partObj = part.jsonObject + val inlineData = partObj[KEY_INLINE_DATA]?.jsonObject ?: continue + val mimeType = inlineData[KEY_MIME_TYPE]?.jsonPrimitive?.contentOrNull + if (mimeType == null || !mimeType.startsWith(MIME_TYPE_AUDIO_PCM_PREFIX)) { + events.add( + LiveLlmEvent.Error( + "Unsupported inlineData mimeType in modelTurn: $mimeType", + ), + ) + continue + } + val base64Data = inlineData[KEY_DATA]?.jsonPrimitive?.contentOrNull + if (base64Data.isNullOrEmpty()) { + events.add( + LiveLlmEvent.Error("inlineData is missing audio '$KEY_DATA' in modelTurn"), + ) + continue + } + try { + val pcmBytes = Base64.getMimeDecoder().decode(base64Data) + val sampleRateHz = extractSampleRateHz(mimeType) + events.add( + LiveLlmEvent.AudioChunk( + pcmBytes = pcmBytes, + sampleRateHz = sampleRateHz, + ), + ) + } catch (e: IllegalArgumentException) { + Log.w(TAG, "Failed to decode base64 PCM audio chunk", e) + events.add( + LiveLlmEvent.Error( + message = "Failed to decode base64 PCM audio chunk", + cause = e, + ), + ) } } + return events + } - val toolCallObj = serverJson["toolCall"]?.jsonObject - if (toolCallObj != null) { - val functionCalls = - toolCallObj["functionCalls"]?.jsonArray ?: JsonArray(emptyList()) - val parsedCalls = - functionCalls.mapNotNull { callElement -> - val callObj = callElement.jsonObject - val callId = callObj["id"]?.jsonPrimitive?.contentOrNull ?: "" - val name = - callObj["name"]?.jsonPrimitive?.contentOrNull ?: return@mapNotNull null - val matchingTool = toolConverter.findToolByName(name, tools) - val argumentsMap = callObj["args"]?.jsonObject.toArgumentsMap() - - LlmResponsePart.ToolCall( - packageName = matchingTool?.packageName ?: "", - functionId = matchingTool?.id ?: name, - arguments = argumentsMap, - callId = callId, - ) - } - if (parsedCalls.isNotEmpty()) { - events.add(LiveLlmEvent.ToolCallRequest(parsedCalls)) + private fun extractSampleRateHz(mimeType: String): Int { + val rateParam = + mimeType + .split(';') + .map { it.trim() } + .firstOrNull { it.startsWith(MIME_TYPE_RATE_PARAM, ignoreCase = true) } + val parsedRate = rateParam?.substringAfter('=')?.toIntOrNull() + return parsedRate ?: DEFAULT_LIVE_OUTPUT_SAMPLE_RATE_HZ + } + + private fun parseToolCall( + toolCallObj: JsonObject, + tools: List, + ): List { + val functionCalls = + toolCallObj[KEY_FUNCTION_CALLS]?.jsonArray + ?: return listOf( + LiveLlmEvent.Error( + "toolCall is missing '$KEY_FUNCTION_CALLS' array: $toolCallObj", + ), + ) + if (functionCalls.isEmpty()) { + return listOf( + LiveLlmEvent.Error("toolCall '$KEY_FUNCTION_CALLS' array is empty"), + ) + } + + val events = mutableListOf() + val validCalls = mutableListOf() + + for (callElement in functionCalls) { + val callObj = callElement.jsonObject + val callId = callObj[KEY_ID]?.jsonPrimitive?.contentOrNull + if (callId.isNullOrBlank()) { + events.add( + LiveLlmEvent.Error("Function call missing '$KEY_ID' in response: $callObj"), + ) + continue } + val name = callObj[KEY_NAME]?.jsonPrimitive?.contentOrNull + if (name.isNullOrBlank()) { + events.add( + LiveLlmEvent.Error("Called function without a '$KEY_NAME': $callObj"), + ) + continue + } + val matchingTool = toolConverter.findToolByName(name, tools) + if (matchingTool == null) { + events.add( + LiveLlmEvent.Error("Called unknown function: $name"), + ) + continue + } + val argumentsMap = callObj[KEY_ARGS]?.jsonObject.toArgumentsMap() + validCalls.add( + LlmResponsePart.ToolCall( + packageName = matchingTool.packageName, + functionId = matchingTool.id, + arguments = argumentsMap, + callId = callId, + ), + ) } + if (validCalls.isNotEmpty()) { + events.add(LiveLlmEvent.ToolCallRequest(validCalls)) + } return events } - internal fun buildSetupMessage( + private fun buildSetupMessage( tools: List, voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, ): JsonObject { @@ -259,7 +459,7 @@ class GeminiLiveProviderImpl .selectDistinctTools(tools) .mapNotNull { tool -> try { - toolConverter.convertForLiveApi(tool) + toolConverter.convert(tool) } catch (e: IllegalArgumentException) { Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) null @@ -268,26 +468,26 @@ class GeminiLiveProviderImpl return buildJsonObject { put( - "setup", + KEY_SETUP, buildJsonObject { - put("model", LIVE_MODEL_NAME) + put(KEY_MODEL, LIVE_MODEL_NAME) put( - "generationConfig", + KEY_GENERATION_CONFIG, buildJsonObject { put( - "responseModalities", - buildJsonArray { add(JsonPrimitive("AUDIO")) }, + KEY_RESPONSE_MODALITIES, + buildJsonArray { add(JsonPrimitive(VALUE_MODALITY_AUDIO)) }, ) put( - "speechConfig", + KEY_SPEECH_CONFIG, buildJsonObject { put( - "voiceConfig", + KEY_VOICE_CONFIG, buildJsonObject { put( - "prebuiltVoiceConfig", + KEY_PREBUILT_VOICE_CONFIG, buildJsonObject { - put("voiceName", voice.apiName) + put(KEY_VOICE_NAME, voice.apiName) }, ) }, @@ -297,14 +497,14 @@ class GeminiLiveProviderImpl }, ) put( - "systemInstruction", + KEY_SYSTEM_INSTRUCTION, buildJsonObject { put( - "parts", + KEY_PARTS, buildJsonArray { add( buildJsonObject { - put("text", getSystemInstruction()) + put(KEY_TEXT, getSystemInstruction()) }, ) }, @@ -313,12 +513,12 @@ class GeminiLiveProviderImpl ) if (functionDeclarations.isNotEmpty()) { put( - "tools", + KEY_TOOLS, buildJsonArray { add( buildJsonObject { put( - "functionDeclarations", + KEY_FUNCTION_DECLARATIONS, JsonArray(functionDeclarations), ) }, @@ -326,24 +526,27 @@ class GeminiLiveProviderImpl }, ) } - put("inputAudioTranscription", buildJsonObject {}) - put("outputAudioTranscription", buildJsonObject {}) + put(KEY_INPUT_AUDIO_TRANSCRIPTION, buildJsonObject {}) + put(KEY_OUTPUT_AUDIO_TRANSCRIPTION, buildJsonObject {}) }, ) } } - internal fun buildRealtimeAudioMessage(pcmBytes: ByteArray): JsonObject { + private fun buildRealtimeAudioMessage( + pcmBytes: ByteArray, + sampleRateHz: Int, + ): JsonObject { val base64Audio = Base64.getEncoder().encodeToString(pcmBytes) return buildJsonObject { put( - "realtimeInput", + KEY_REALTIME_INPUT, buildJsonObject { put( - "audio", + KEY_AUDIO, buildJsonObject { - put("mimeType", INPUT_AUDIO_MIME_TYPE) - put("data", base64Audio) + put(KEY_MIME_TYPE, "$MIME_TYPE_AUDIO_PCM_PREFIX;$MIME_TYPE_RATE_PARAM$sampleRateHz") + put(KEY_DATA, base64Audio) }, ) }, @@ -351,16 +554,16 @@ class GeminiLiveProviderImpl } } - internal fun buildToolResponseMessage( + private fun buildToolResponseMessage( toolOutputs: List, tools: List, ): JsonObject { return buildJsonObject { put( - "toolResponse", + KEY_TOOL_RESPONSE, buildJsonObject { put( - "functionResponses", + KEY_FUNCTION_RESPONSES, buildJsonArray { for (output in toolOutputs) { val mappedName = @@ -374,13 +577,13 @@ class GeminiLiveProviderImpl add( buildJsonObject { if (output.callId.isNotEmpty()) { - put("id", output.callId) + put(KEY_ID, output.callId) } - put("name", mappedName) + put(KEY_NAME, mappedName) put( - "response", + KEY_RESPONSE, buildJsonObject { - put("result", parsedResult) + put(KEY_RESULT, parsedResult) }, ) }, @@ -405,11 +608,67 @@ class GeminiLiveProviderImpl """.trimIndent() } + private sealed class FrameReadResult { + data class JsonText(val text: String) : FrameReadResult() + + object NormalClose : FrameReadResult() + + data class ErrorClose( + val message: String, + val cause: Throwable? = null, + ) : FrameReadResult() + } + companion object { private const val TAG = "GeminiLiveProvider" const val LIVE_MODEL_NAME = "models/gemini-3.8-live" - private const val INPUT_AUDIO_MIME_TYPE = "audio/pcm;rate=16000" private const val GEMINI_LIVE_WS_URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" + private const val CLOSE_REASON_CLIENT_ENDED = "Session ended by client" + + private const val KEY_SETUP = "setup" + private const val KEY_SETUP_COMPLETE = "setupComplete" + private const val KEY_MODEL = "model" + private const val KEY_GENERATION_CONFIG = "generationConfig" + private const val KEY_RESPONSE_MODALITIES = "responseModalities" + private const val VALUE_MODALITY_AUDIO = "AUDIO" + private const val KEY_SPEECH_CONFIG = "speechConfig" + private const val KEY_VOICE_CONFIG = "voiceConfig" + private const val KEY_PREBUILT_VOICE_CONFIG = "prebuiltVoiceConfig" + private const val KEY_VOICE_NAME = "voiceName" + private const val KEY_SYSTEM_INSTRUCTION = "systemInstruction" + private const val KEY_PARTS = "parts" + private const val KEY_TEXT = "text" + private const val KEY_TOOLS = "tools" + private const val KEY_FUNCTION_DECLARATIONS = "functionDeclarations" + private const val KEY_INPUT_AUDIO_TRANSCRIPTION = "inputAudioTranscription" + private const val KEY_OUTPUT_AUDIO_TRANSCRIPTION = "outputAudioTranscription" + + private const val KEY_SERVER_CONTENT = "serverContent" + private const val KEY_INTERRUPTED = "interrupted" + private const val KEY_INPUT_TRANSCRIPTION = "inputTranscription" + private const val KEY_OUTPUT_TRANSCRIPTION = "outputTranscription" + private const val KEY_MODEL_TURN = "modelTurn" + private const val KEY_INLINE_DATA = "inlineData" + private const val KEY_MIME_TYPE = "mimeType" + private const val KEY_DATA = "data" + private const val KEY_TURN_COMPLETE = "turnComplete" + + private const val KEY_TOOL_CALL = "toolCall" + private const val KEY_FUNCTION_CALLS = "functionCalls" + private const val KEY_ID = "id" + private const val KEY_NAME = "name" + private const val KEY_ARGS = "args" + + private const val KEY_REALTIME_INPUT = "realtimeInput" + private const val KEY_AUDIO = "audio" + + private const val KEY_TOOL_RESPONSE = "toolResponse" + private const val KEY_FUNCTION_RESPONSES = "functionResponses" + private const val KEY_RESPONSE = "response" + private const val KEY_RESULT = "result" + + private const val MIME_TYPE_AUDIO_PCM_PREFIX = "audio/pcm" + private const val MIME_TYPE_RATE_PARAM = "rate=" } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveToolConverter.kt new file mode 100644 index 00000000..3c8ef639 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveToolConverter.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.domain.appfunction.ToolConverter +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Implementation of [ToolConverter] for Gemini Live (`BidiGenerateContent`) `FunctionDeclaration` + * schemas, normalizing OpenAPI schema fields produced by [GeminiToolConverter]. + */ +@Singleton +class GeminiLiveToolConverter + @Inject + constructor( + private val geminiToolConverter: GeminiToolConverter, + ) : ToolConverter { + override fun convert(tool: AppFunctionMetadata): JsonObject { + val baseSchema = geminiToolConverter.convert(tool) + return normalizeForLiveApi(baseSchema) + } + + override fun getToolName(tool: AppFunctionMetadata): String { + return geminiToolConverter.getToolName(tool) + } + + private fun normalizeForLiveApi(obj: JsonObject): JsonObject { + val typeValue = (obj[KEY_TYPE] as? JsonPrimitive)?.content + return buildJsonObject { + for ((key, value) in obj) { + when { + key == KEY_FORMAT && + (value as? JsonPrimitive)?.content == VALUE_FILE_REFERENCE -> { + // Omit Interactions-API-specific file_reference format in BidiGenerateContent + } + key == KEY_ENUMS -> { + if (typeValue == VALUE_STRING) { + put(KEY_ENUM_LIVE, value) + } + } + value is JsonObject -> { + put(key, normalizeForLiveApi(value)) + } + else -> { + put(key, value) + } + } + } + } + } + + companion object { + private const val KEY_TYPE = "type" + private const val KEY_FORMAT = "format" + private const val VALUE_FILE_REFERENCE = "file_reference" + private const val VALUE_STRING = "string" + private const val KEY_ENUMS = "enums" + private const val KEY_ENUM_LIVE = "enum" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index e89546ff..b7eaf5f7 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -21,6 +21,10 @@ import com.example.appfunctions.agent.domain.LlmInput import com.example.appfunctions.agent.domain.LlmProvider import com.example.appfunctions.agent.domain.LlmResponse import com.example.appfunctions.agent.domain.LlmResponsePart +import com.example.appfunctions.agent.domain.appfunction.findToolByName +import com.example.appfunctions.agent.domain.appfunction.resolveToolName +import com.example.appfunctions.agent.domain.appfunction.selectDistinctTools +import com.example.appfunctions.agent.domain.toArgumentsMap import io.ktor.client.HttpClient import io.ktor.client.request.get import io.ktor.client.request.header diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index d7c85303..cc109813 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -45,71 +45,10 @@ class GeminiToolConverter return convertToolToGeminiSchema(tool) } - /** - * Converts an [AppFunctionMetadata] into a Gemini Live (`BidiGenerateContent`) - * `FunctionDeclaration` JSON object, normalizing OpenAPI schema fields. - */ - fun convertForLiveApi(tool: AppFunctionMetadata): JsonObject { - val baseSchema = convertToolToGeminiSchema(tool) - return normalizeForLiveApi(baseSchema) - } - - private fun normalizeForLiveApi(obj: JsonObject): JsonObject { - val typeValue = (obj[KEY_TYPE] as? JsonPrimitive)?.content - return buildJsonObject { - for ((key, value) in obj) { - when { - key == KEY_FORMAT && - (value as? JsonPrimitive)?.content == VALUE_FILE_REFERENCE -> { - // Omit Interactions-API-specific file_reference format in BidiGenerateContent - } - key == KEY_ENUMS -> { - if (typeValue == VALUE_STRING) { - put(KEY_ENUM_LIVE, value) - } - } - value is JsonObject -> { - put(key, normalizeForLiveApi(value)) - } - else -> { - put(key, value) - } - } - } - } - } - override fun getToolName(tool: AppFunctionMetadata): String { return getGeminiFunctionName(tool) } - /** - * Deduplicates [tools] by their converted Gemini function name, prioritizing functions - * whose ID starts with their package name. - */ - fun selectDistinctTools(tools: List): List = - tools - .sortedByDescending { it.id.startsWith(it.packageName) } - .distinctBy { getToolName(it) } - - /** Finds the [AppFunctionMetadata] in [tools] matching a converted Gemini function [name]. */ - fun findToolByName( - name: String, - tools: List, - ): AppFunctionMetadata? = tools.find { getToolName(it) == name } - - /** - * Resolves the converted Gemini function name for [functionId] from [tools], falling back - * to [functionId] if no matching tool metadata is present. - */ - fun resolveToolName( - functionId: String, - tools: List, - ): String { - val matchingTool = tools.find { it.id == functionId } ?: return functionId - return getToolName(matchingTool) - } - private fun convertToolToGeminiSchema(tool: AppFunctionMetadata): JsonObject { val combinedName = getGeminiFunctionName(tool) return buildJsonObject { @@ -356,7 +295,6 @@ class GeminiToolConverter private const val VALUE_ARRAY = "array" private const val KEY_ITEMS = "items" private const val KEY_ENUMS = "enums" - private const val KEY_ENUM_LIVE = "enum" private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]") private const val MAX_NAME_LENGTH = 64 } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index 2e0cd274..d384df23 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -115,9 +115,12 @@ abstract class DataModule { }, ) } - install(HttpTimeout) { socketTimeoutMillis = 30000 } - install(WebSockets) + install(HttpTimeout) { socketTimeoutMillis = SOCKET_TIMEOUT_MS } + install(WebSockets) { pingIntervalMillis = WEBSOCKET_PING_INTERVAL_MS } } } + + private const val SOCKET_TIMEOUT_MS = 30_000L + private const val WEBSOCKET_PING_INTERVAL_MS = 20_000L } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/JsonExtensions.kt similarity index 97% rename from agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt rename to agent/app/src/main/java/com/example/appfunctions/agent/domain/JsonExtensions.kt index 22ad48b9..c49c1f4a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/JsonExtensions.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.example.appfunctions.agent.data +package com.example.appfunctions.agent.domain import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt index 2b1c2025..f04dc24f 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt @@ -16,19 +16,49 @@ package com.example.appfunctions.agent.domain import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.data.GeminiLiveVoice +import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.withContext + +/** Default microphone PCM input sample rate in Hz for live audio sessions. */ +const val DEFAULT_LIVE_INPUT_SAMPLE_RATE_HZ = 16_000 + +/** Default model PCM output sample rate in Hz for live audio sessions. */ +const val DEFAULT_LIVE_OUTPUT_SAMPLE_RATE_HZ = 24_000 /** Interface for LLM providers supporting bidirectional realtime audio and tool sessions. */ interface LiveLlmProvider { /** - * Connects to the live LLM backend, completes session initialization with [tools], and invokes - * [block] with an active [LiveLlmSession] until [block] returns or is cancelled. + * Connects to the live LLM backend, completes session initialization with [tools] and [voice], + * and returns the active [LiveLlmSession]. The caller is responsible for closing the returned + * session via [LiveLlmSession.close] or using [withSession]. */ suspend fun connect( apiKey: String, tools: List, - block: suspend (LiveLlmSession) -> Unit, - ) + voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + ): LiveLlmSession +} + +/** + * Connects to the [LiveLlmProvider], invokes [block] with the active [LiveLlmSession], and ensures + * [LiveLlmSession.close] is called when [block] completes or is cancelled. + */ +suspend inline fun LiveLlmProvider.withSession( + apiKey: String, + tools: List, + voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + block: (LiveLlmSession) -> Unit, +) { + val session = connect(apiKey = apiKey, tools = tools, voice = voice) + try { + block(session) + } finally { + withContext(NonCancellable) { + session.close() + } + } } /** Represents an active bidirectional voice and tool session with an LLM provider. */ @@ -36,11 +66,22 @@ interface LiveLlmSession { /** Sequential stream of server-originated events for this live session. */ val events: Flow - /** Sends a chunk of raw 16-bit 16kHz mono PCM user microphone audio to the model. */ - suspend fun sendAudio(pcmBytes: ByteArray) + /** + * Sends a chunk of raw 16-bit mono PCM user microphone audio at [sampleRateHz] to the model. + */ + suspend fun sendAudio( + pcmBytes: ByteArray, + sampleRateHz: Int = DEFAULT_LIVE_INPUT_SAMPLE_RATE_HZ, + ) - /** Sends tool execution [outputs] back to the model in response to a [LiveLlmEvent.ToolCallRequest]. */ + /** + * Sends tool execution [outputs] back to the model in response to a + * [LiveLlmEvent.ToolCallRequest]. + */ suspend fun sendToolResponses(outputs: List) + + /** Closes the live session and releases the underlying transport connection. */ + suspend fun close() } /** Provider-agnostic events emitted by a [LiveLlmSession]. */ @@ -54,12 +95,33 @@ sealed class LiveLlmEvent { /** Incremental transcription of the model's spoken response. */ data class OutputTranscription(val text: String) : LiveLlmEvent() - /** Raw PCM audio chunk generated by the model for playback. */ - data class AudioChunk(val pcmBytes: ByteArray) : LiveLlmEvent() + /** Raw 16-bit mono PCM audio chunk generated by the model for playback at [sampleRateHz]. */ + data class AudioChunk( + val pcmBytes: ByteArray, + val sampleRateHz: Int = DEFAULT_LIVE_OUTPUT_SAMPLE_RATE_HZ, + ) : LiveLlmEvent() { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is AudioChunk) return false + return pcmBytes.contentEquals(other.pcmBytes) && sampleRateHz == other.sampleRateHz + } + + override fun hashCode(): Int { + var result = pcmBytes.contentHashCode() + result = 31 * result + sampleRateHz + return result + } + } /** Indicates the model finished its current conversational turn. */ object TurnComplete : LiveLlmEvent() /** Requests execution of one or more [calls] before continuing the live conversation. */ data class ToolCallRequest(val calls: List) : LiveLlmEvent() + + /** Indicates an error occurred during the live session. */ + data class Error( + val message: String, + val cause: Throwable? = null, + ) : LiveLlmEvent() } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ToolConverter.kt index 022524ff..338be8c0 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ToolConverter.kt @@ -25,3 +25,30 @@ interface ToolConverter { /** Returns the name of the tool as it should be represented in the LLM schema. */ fun getToolName(tool: AppFunctionMetadata): String } + +/** + * Deduplicates [tools] by their converted tool name, prioritizing functions whose ID starts with + * their package name. + */ +fun ToolConverter<*>.selectDistinctTools(tools: List): List = + tools + .sortedByDescending { it.id.startsWith(it.packageName) } + .distinctBy { getToolName(it) } + +/** Finds the [AppFunctionMetadata] in [tools] matching a converted tool [name]. */ +fun ToolConverter<*>.findToolByName( + name: String, + tools: List, +): AppFunctionMetadata? = tools.find { getToolName(it) == name } + +/** + * Resolves the converted tool name for [functionId] from [tools], falling back to [functionId] if + * no matching tool metadata is present. + */ +fun ToolConverter<*>.resolveToolName( + functionId: String, + tools: List, +): String { + val matchingTool = tools.find { it.id == functionId } ?: return functionId + return getToolName(matchingTool) +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt index eaebf3ad..3e228ad2 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt @@ -23,17 +23,20 @@ import androidx.appfunctions.metadata.AppFunctionResponseMetadata import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata import com.example.appfunctions.agent.domain.LiveLlmEvent import com.example.appfunctions.agent.domain.ToolOutput +import com.example.appfunctions.agent.domain.withSession import io.ktor.client.HttpClient +import io.ktor.websocket.CloseReason import io.ktor.websocket.Frame import io.ktor.websocket.WebSocketSession +import io.ktor.websocket.readReason import io.ktor.websocket.readText import io.mockk.coEvery import io.mockk.every import io.mockk.mockk import io.mockk.mockkStatic +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.flow.toList import kotlinx.coroutines.test.runTest import kotlinx.serialization.json.Json @@ -47,92 +50,109 @@ import org.junit.Assert.assertTrue import org.junit.Assert.fail import org.junit.Before import org.junit.Test +import java.io.IOException import java.util.Base64 +import kotlin.coroutines.CoroutineContext class GeminiLiveProviderImplTest { - private lateinit var settingsRepository: SettingsRepository - private lateinit var provider: GeminiLiveProviderImpl + private lateinit var httpClient: HttpClient + private lateinit var toolConverter: GeminiLiveToolConverter @Before fun setUp() { mockkStatic(Log::class) every { Log.d(any(), any()) } returns 0 every { Log.w(any(), any()) } returns 0 + every { Log.w(any(), any(), any()) } returns 0 every { Log.e(any(), any(), any()) } returns 0 - settingsRepository = mockk(relaxed = true) - every { settingsRepository.geminiLiveVoice } returns flowOf(GeminiLiveVoice.DEFAULT) - - provider = - GeminiLiveProviderImpl( - httpClient = mockk(relaxed = true), - toolConverter = GeminiToolConverter(), - settingsRepository = settingsRepository, - ) + httpClient = mockk(relaxed = true) + toolConverter = GeminiLiveToolConverter(GeminiToolConverter()) } @Test - fun `buildSetupMessage configures gemini-3_8-live model, audio modality, transcriptions, and tools`() { - val tool = createSampleNoteTool() - - val setupEnvelope = provider.buildSetupMessage(listOf(tool)) - val setup = setupEnvelope["setup"]!!.jsonObject - - assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content) - - val modalities = - setup["generationConfig"]!! - .jsonObject["responseModalities"]!! - .jsonArray - .map { it.jsonPrimitive.content } - assertEquals(listOf("AUDIO"), modalities) - - val voiceName = - setup["generationConfig"]!! - .jsonObject["speechConfig"]!! - .jsonObject["voiceConfig"]!! - .jsonObject["prebuiltVoiceConfig"]!! - .jsonObject["voiceName"]!! - .jsonPrimitive - .content - assertEquals("Zephyr", voiceName) - - assertTrue(setup.containsKey("inputAudioTranscription")) - assertTrue(setup.containsKey("outputAudioTranscription")) - val systemInstructionText = - setup["systemInstruction"]!! - .jsonObject["parts"]!! - .jsonArray - .first() - .jsonObject["text"]!! - .jsonPrimitive - .content - assertTrue( - systemInstructionText.contains( - "Default to English for understanding and replying unless the user clearly " + - "speaks in or asks to switch to another language.", - ), - ) - assertTrue( - systemInstructionText.contains( - "Execute user-requested actions directly without claiming an on-screen " + - "confirmation card or button is shown", - ), - ) - assertFalse(systemInstructionText.contains("confirmation_required")) - - val declarations = - setup["tools"]!! - .jsonArray - .first() - .jsonObject["functionDeclarations"]!! - .jsonArray - assertEquals(1, declarations.size) - assertEquals( - "com_example_notes_createNote", - declarations.first().jsonObject["name"]!!.jsonPrimitive.content, - ) - } + fun `connect sends setup message with model, audio modality, voice, transcriptions, and tools`() = + runTest { + val tool = createSampleNoteTool() + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.close() + + val sentFrames = mutableListOf() + var capturedWsUrl: String? = null + val provider = + createProvider { wsUrl -> + capturedWsUrl = wsUrl + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ) + } + + provider.withSession( + apiKey = "secret-key", + tools = listOf(tool), + voice = GeminiLiveVoice.AOEDE, + ) {} + + assertTrue(capturedWsUrl!!.endsWith("?key=secret-key")) + val setup = Json.parseToJsonElement(sentFrames.first()).jsonObject["setup"]!!.jsonObject + assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content) + + val modalities = + setup["generationConfig"]!! + .jsonObject["responseModalities"]!! + .jsonArray + .map { it.jsonPrimitive.content } + assertEquals(listOf("AUDIO"), modalities) + + val voiceName = + setup["generationConfig"]!! + .jsonObject["speechConfig"]!! + .jsonObject["voiceConfig"]!! + .jsonObject["prebuiltVoiceConfig"]!! + .jsonObject["voiceName"]!! + .jsonPrimitive + .content + assertEquals("Aoede", voiceName) + + assertTrue(setup.containsKey("inputAudioTranscription")) + assertTrue(setup.containsKey("outputAudioTranscription")) + val systemInstructionText = + setup["systemInstruction"]!! + .jsonObject["parts"]!! + .jsonArray + .first() + .jsonObject["text"]!! + .jsonPrimitive + .content + assertTrue( + systemInstructionText.contains( + "Default to English for understanding and replying unless the user clearly " + + "speaks in or asks to switch to another language.", + ), + ) + assertTrue( + systemInstructionText.contains( + "Execute user-requested actions directly without claiming an on-screen " + + "confirmation card or button is shown", + ), + ) + assertFalse(systemInstructionText.contains("confirmation_required")) + + val declarations = + setup["tools"]!! + .jsonArray + .first() + .jsonObject["functionDeclarations"]!! + .jsonArray + assertEquals(1, declarations.size) + assertEquals( + "com_example_notes_createNote", + declarations.first().jsonObject["name"]!!.jsonPrimitive.content, + ) + } @Test fun `connect throws IllegalStateException when WebSocket closes before setupComplete`() = @@ -140,14 +160,13 @@ class GeminiLiveProviderImplTest { val incomingFrames = Channel(Channel.UNLIMITED) incomingFrames.close() - provider.webSocketConnector = { _, block -> - coroutineScope { - createMockWebSocketSession(coroutineContext, incomingFrames).block() + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) } - } try { - provider.connect(apiKey = "key-123", tools = emptyList()) {} + provider.connect(apiKey = "key-123", tools = emptyList()) fail("Expected IllegalStateException") } catch (e: IllegalStateException) { assertTrue(e.message!!.contains("setupComplete")) @@ -155,60 +174,201 @@ class GeminiLiveProviderImplTest { } @Test - fun `connect sends selected GeminiLiveVoice from SettingsRepository in setup message`() = + fun `connect throws IllegalStateException when first frame is missing setupComplete`() = runTest { - every { settingsRepository.geminiLiveVoice } returns flowOf(GeminiLiveVoice.AOEDE) + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"error":{"message":"Invalid argument"}}""")) + incomingFrames.close() + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + try { + provider.connect(apiKey = "key-123", tools = emptyList()) + fail("Expected IllegalStateException") + } catch (e: IllegalStateException) { + assertTrue(e.message!!.startsWith("Unexpected setup response:")) + } + } + + @Test + fun `connect rethrows CancellationException during handshake without wrapping as IllegalStateException`() = + runTest { + val wsSession = mockk(relaxed = true) + every { wsSession.coroutineContext } returns coroutineContext + every { wsSession.incoming } returns + mockk { + coEvery { receive() } throws CancellationException("Handshake cancelled") + } + + val provider = createProvider { wsSession } + + try { + provider.connect(apiKey = "key-123", tools = emptyList()) + fail("Expected CancellationException") + } catch (e: CancellationException) { + assertEquals("Handshake cancelled", e.message) + } + } + + @Test + fun `withSession closes WebSocket with NORMAL CloseReason when block completes normally or throws`() = + runTest { + val incomingFramesNormal = Channel(Channel.UNLIMITED) + incomingFramesNormal.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFramesNormal.close() + val normalCloseReasons = mutableListOf() + + val providerNormal = + createProvider { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFramesNormal, + onSendClose = { normalCloseReasons.add(it) }, + ) + } + + providerNormal.withSession(apiKey = "key-123", tools = emptyList()) {} + + assertEquals( + listOf(CloseReason(CloseReason.Codes.NORMAL, "Session ended by client")), + normalCloseReasons, + ) + + val incomingFramesFailing = Channel(Channel.UNLIMITED) + incomingFramesFailing.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFramesFailing.close() + val failingCloseReasons = mutableListOf() + + val providerFailing = + createProvider { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFramesFailing, + onSendClose = { failingCloseReasons.add(it) }, + ) + } + + try { + providerFailing.withSession(apiKey = "key-123", tools = emptyList()) { + throw IllegalStateException("Caller failed") + } + fail("Expected IllegalStateException") + } catch (e: IllegalStateException) { + assertEquals("Caller failed", e.message) + } + + assertEquals( + listOf(CloseReason(CloseReason.Codes.NORMAL, "Session ended by client")), + failingCloseReasons, + ) + } + + @Test + fun `connect and events decode Frame_Binary server frames`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend( + Frame.Binary(fin = true, data = """{"setupComplete":{}}""".encodeToByteArray()), + ) + incomingFrames.trySend( + Frame.Binary( + fin = true, + data = """{"serverContent":{"turnComplete":true}}""".encodeToByteArray(), + ), + ) + incomingFrames.close() + + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + val receivedEvents = mutableListOf() + provider.withSession(apiKey = "key-123", tools = emptyList()) { session -> + receivedEvents.addAll(session.events.toList()) + } + + assertEquals(listOf(LiveLlmEvent.TurnComplete), receivedEvents) + } + + @Test + fun `sendAudio encodes PCM bytes with specified sampleRateHz in realtimeInput frame`() = + runTest { val incomingFrames = Channel(Channel.UNLIMITED) incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) incomingFrames.close() val sentFrames = mutableListOf() - provider.webSocketConnector = { _, block -> - coroutineScope { + val provider = + createProvider { createMockWebSocketSession( context = coroutineContext, incomingFrames = incomingFrames, onSendText = { sentFrames.add(it) }, - ).block() + ) } + + provider.withSession(apiKey = "key-123", tools = emptyList()) { session -> + session.sendAudio(pcmBytes = byteArrayOf(1, 2, 3), sampleRateHz = 16_000) } - provider.connect(apiKey = "key-123", tools = emptyList()) {} + assertEquals(2, sentFrames.size) + val realtimeAudio = + Json.parseToJsonElement(sentFrames[1]) + .jsonObject["realtimeInput"]!! + .jsonObject["audio"]!! + .jsonObject + assertEquals("audio/pcm;rate=16000", realtimeAudio["mimeType"]!!.jsonPrimitive.content) + assertArrayEquals( + byteArrayOf(1, 2, 3), + Base64.getDecoder().decode(realtimeAudio["data"]!!.jsonPrimitive.content), + ) + } + + @Test + fun `events emits Interrupted when serverContent has interrupted true`() = + runTest { + val events = + collectSingleServerFrame( + """{"serverContent":{"interrupted":true}}""", + ) + assertEquals(listOf(LiveLlmEvent.Interrupted), events) + } - val setupVoiceName = - Json.parseToJsonElement(sentFrames.first()) - .jsonObject["setup"]!! - .jsonObject["generationConfig"]!! - .jsonObject["speechConfig"]!! - .jsonObject["voiceConfig"]!! - .jsonObject["prebuiltVoiceConfig"]!! - .jsonObject["voiceName"]!! - .jsonPrimitive - .content - assertEquals("Aoede", setupVoiceName) + @Test + fun `events emits InputTranscription when serverContent has inputTranscription`() = + runTest { + val events = + collectSingleServerFrame( + """{"serverContent":{"inputTranscription":{"text":"Create a note"}}}""", + ) + assertEquals(listOf(LiveLlmEvent.InputTranscription("Create a note")), events) } @Test - fun `connect exchanges realtimeInput audio, parses server events, and sends mapped toolResponse`() = + fun `events emits OutputTranscription when serverContent has outputTranscription`() = + runTest { + val events = + collectSingleServerFrame( + """{"serverContent":{"outputTranscription":{"text":"Creating note"}}}""", + ) + assertEquals(listOf(LiveLlmEvent.OutputTranscription("Creating note")), events) + } + + @Test + fun `events emits AudioChunk with sampleRateHz when serverContent has modelTurn inlineData`() = runTest { - val tool = createSampleNoteTool() val expectedPcm = byteArrayOf(10, 20, 30, 40) val encodedPcm = Base64.getEncoder().encodeToString(expectedPcm) - val incomingFrames = Channel(Channel.UNLIMITED) - val sentFrames = mutableListOf() - var capturedWsUrl: String? = null - - incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) - incomingFrames.trySend( - Frame.Text( + val events = + collectSingleServerFrame( """ { "serverContent": { - "interrupted": true, - "inputTranscription": { "text": "Create a note" }, - "outputTranscription": { "text": "Creating note" }, "modelTurn": { "parts": [ { @@ -218,13 +378,34 @@ class GeminiLiveProviderImplTest { } } ] - }, - "turnComplete": true + } } } """.trimIndent(), - ), + ) + + assertEquals( + listOf(LiveLlmEvent.AudioChunk(pcmBytes = expectedPcm, sampleRateHz = 24_000)), + events, ) + } + + @Test + fun `events emits TurnComplete when serverContent has turnComplete true`() = + runTest { + val events = + collectSingleServerFrame( + """{"serverContent":{"turnComplete":true}}""", + ) + assertEquals(listOf(LiveLlmEvent.TurnComplete), events) + } + + @Test + fun `events emits ToolCallRequest and sendToolResponses sends mapped functionResponses`() = + runTest { + val tool = createSampleNoteTool() + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) incomingFrames.trySend( Frame.Text( """ @@ -238,11 +419,6 @@ class GeminiLiveProviderImplTest { "title": "Shopping", "meta": { "pinned": true, "priority": 2 } } - }, - { - "id": "call-unknown", - "name": "hallucinated_tool", - "args": {} } ] } @@ -252,22 +428,18 @@ class GeminiLiveProviderImplTest { ) incomingFrames.close() - provider.webSocketConnector = { wsUrl, block -> - capturedWsUrl = wsUrl - coroutineScope { - val ws = - createMockWebSocketSession( - context = coroutineContext, - incomingFrames = incomingFrames, - onSendText = { sentFrames.add(it) }, - ) - ws.block() + val sentFrames = mutableListOf() + val provider = + createProvider { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ) } - } val receivedEvents = mutableListOf() - provider.connect(apiKey = "secret-key", tools = listOf(tool)) { session -> - session.sendAudio(byteArrayOf(1, 2, 3)) + provider.withSession(apiKey = "key-123", tools = listOf(tool)) { session -> receivedEvents.addAll(session.events.toList()) session.sendToolResponses( listOf( @@ -276,61 +448,28 @@ class GeminiLiveProviderImplTest { callId = "call-1", result = """{"noteId":"n-1"}""", ), - ToolOutput( - functionId = "hallucinated_tool", - callId = "call-unknown", - result = "Tool not found: hallucinated_tool", - ), ), ) } - assertTrue(capturedWsUrl!!.endsWith("?key=secret-key")) - assertEquals(3, sentFrames.size) - - // Frame 0: setup - assertTrue(Json.parseToJsonElement(sentFrames[0]).jsonObject.containsKey("setup")) - - // Frame 1: realtimeInput audio - val realtimeAudio = - Json.parseToJsonElement(sentFrames[1]) - .jsonObject["realtimeInput"]!! - .jsonObject["audio"]!! - .jsonObject - assertEquals("audio/pcm;rate=16000", realtimeAudio["mimeType"]!!.jsonPrimitive.content) - assertArrayEquals( - byteArrayOf(1, 2, 3), - Base64.getDecoder().decode(realtimeAudio["data"]!!.jsonPrimitive.content), - ) - - // Received events - assertEquals(6, receivedEvents.size) - assertEquals(LiveLlmEvent.Interrupted, receivedEvents[0]) - assertEquals(LiveLlmEvent.InputTranscription("Create a note"), receivedEvents[1]) - assertEquals(LiveLlmEvent.OutputTranscription("Creating note"), receivedEvents[2]) - assertArrayEquals(expectedPcm, (receivedEvents[3] as LiveLlmEvent.AudioChunk).pcmBytes) - assertEquals(LiveLlmEvent.TurnComplete, receivedEvents[4]) - - val toolCallReq = receivedEvents[5] as LiveLlmEvent.ToolCallRequest - assertEquals(2, toolCallReq.calls.size) + assertEquals(1, receivedEvents.size) + val toolCallReq = receivedEvents.single() as LiveLlmEvent.ToolCallRequest + assertEquals(1, toolCallReq.calls.size) assertEquals("com.example.notes", toolCallReq.calls[0].packageName) assertEquals("com.example.notes.createNote", toolCallReq.calls[0].functionId) + assertEquals("call-1", toolCallReq.calls[0].callId) assertEquals("Shopping", toolCallReq.calls[0].arguments["title"]) @Suppress("UNCHECKED_CAST") val meta = toolCallReq.calls[0].arguments["meta"] as Map assertEquals(true, meta["pinned"]) assertEquals(2L, meta["priority"]) - assertEquals("", toolCallReq.calls[1].packageName) - assertEquals("hallucinated_tool", toolCallReq.calls[1].functionId) - - // Frame 2: toolResponse val responses = - Json.parseToJsonElement(sentFrames[2]) + Json.parseToJsonElement(sentFrames.last()) .jsonObject["toolResponse"]!! .jsonObject["functionResponses"]!! .jsonArray - assertEquals(2, responses.size) + assertEquals(1, responses.size) assertEquals( "com_example_notes_createNote", responses[0].jsonObject["name"]!!.jsonPrimitive.content, @@ -344,16 +483,154 @@ class GeminiLiveProviderImplTest { .jsonPrimitive .content, ) - assertEquals( - "hallucinated_tool", - responses[1].jsonObject["name"]!!.jsonPrimitive.content, + } + + @Test + fun `events emits Error when toolCall references unknown function or is missing id or name`() = + runTest { + val tool = createSampleNoteTool() + val events = + collectSingleServerFrame( + frameJson = + """ + { + "toolCall": { + "functionCalls": [ + { + "id": "call-unknown", + "name": "hallucinated_tool", + "args": {} + }, + { + "name": "com_example_notes_createNote", + "args": {} + }, + { + "id": "call-no-name", + "args": {} + } + ] + } + } + """.trimIndent(), + tools = listOf(tool), + ) + + assertEquals(3, events.size) + assertTrue((events[0] as LiveLlmEvent.Error).message.contains("hallucinated_tool")) + assertTrue((events[1] as LiveLlmEvent.Error).message.contains("id")) + assertTrue((events[2] as LiveLlmEvent.Error).message.contains("name")) + } + + @Test + fun `events recovers from malformed JSON frame mid-session and emits subsequent valid events`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.trySend(Frame.Text("not-valid-json")) + incomingFrames.trySend(Frame.Text("""{"serverContent":{"turnComplete":true}}""")) + incomingFrames.close() + + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + val receivedEvents = mutableListOf() + provider.withSession(apiKey = "key-123", tools = emptyList()) { session -> + receivedEvents.addAll(session.events.toList()) + } + + assertEquals(2, receivedEvents.size) + assertTrue(receivedEvents[0] is LiveLlmEvent.Error) + assertEquals(LiveLlmEvent.TurnComplete, receivedEvents[1]) + } + + @Test + fun `events emits Error when server closes WebSocket with non-normal CloseReason or transport exception`() = + runTest { + val incomingCloseFrames = Channel(Channel.UNLIMITED) + incomingCloseFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingCloseFrames.trySend( + Frame.Close(CloseReason(CloseReason.Codes.VIOLATED_POLICY, "Invalid API key")), + ) + incomingCloseFrames.close() + + val providerClose = + createProvider { + createMockWebSocketSession(coroutineContext, incomingCloseFrames) + } + + val closeEvents = mutableListOf() + providerClose.withSession(apiKey = "bad-key", tools = emptyList()) { session -> + closeEvents.addAll(session.events.toList()) + } + + assertEquals(1, closeEvents.size) + assertTrue((closeEvents.single() as LiveLlmEvent.Error).message.contains("Invalid API key")) + + var receiveCount = 0 + val wsSession = mockk(relaxed = true) + every { wsSession.coroutineContext } returns coroutineContext + every { wsSession.incoming } returns + mockk { + coEvery { receive() } coAnswers { + receiveCount++ + if (receiveCount == 1) { + Frame.Text("""{"setupComplete":{}}""") + } else { + throw IOException("Connection reset by peer") + } + } + } + + val providerError = createProvider { wsSession } + val transportEvents = mutableListOf() + providerError.withSession(apiKey = "key-123", tools = emptyList()) { session -> + transportEvents.addAll(session.events.toList()) + } + + assertEquals(1, transportEvents.size) + assertTrue( + (transportEvents.single() as LiveLlmEvent.Error) + .message + .contains("Connection reset by peer"), ) } + private suspend fun CoroutineScope.collectSingleServerFrame( + frameJson: String, + tools: List = emptyList(), + ): List { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.trySend(Frame.Text(frameJson)) + incomingFrames.close() + + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + val receivedEvents = mutableListOf() + provider.withSession(apiKey = "key-123", tools = tools) { session -> + receivedEvents.addAll(session.events.toList()) + } + return receivedEvents + } + + private fun createProvider(connector: suspend (String) -> WebSocketSession): GeminiLiveProviderImpl = + GeminiLiveProviderImpl( + httpClient = httpClient, + toolConverter = toolConverter, + webSocketConnector = connector, + ) + private fun createMockWebSocketSession( - context: kotlin.coroutines.CoroutineContext, + context: CoroutineContext, incomingFrames: Channel, onSendText: (String) -> Unit = {}, + onSendClose: (CloseReason?) -> Unit = {}, ): WebSocketSession { val outgoingFrames = Channel(Channel.UNLIMITED) return mockk(relaxed = true).apply { @@ -361,9 +638,10 @@ class GeminiLiveProviderImplTest { every { incoming } returns incomingFrames every { outgoing } returns outgoingFrames coEvery { send(any()) } coAnswers { - val frame = firstArg() - if (frame is Frame.Text) { - onSendText(frame.readText()) + when (val frame = firstArg()) { + is Frame.Text -> onSendText(frame.readText()) + is Frame.Close -> onSendClose(frame.readReason()) + else -> {} } } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt new file mode 100644 index 00000000..7c75f8df --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt @@ -0,0 +1,207 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionIntTypeMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import kotlinx.serialization.json.Json +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class GeminiLiveToolConverterTest { + private lateinit var converter: GeminiLiveToolConverter + + @Before + fun setUp() { + converter = GeminiLiveToolConverter(GeminiToolConverter()) + } + + @Test + fun convert_normalizesStringEnumsAndStripsFileReferenceFormat() { + val enumParam = + AppFunctionParameterMetadata( + name = "mode", + isRequired = true, + dataType = + AppFunctionStringTypeMetadata(isNullable = false, enumValues = setOf("A", "B")), + description = "Mode parameter", + ) + val uriParam = + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = false, + dataType = AppFunctionStringTypeMetadata(isNullable = true), + description = "Wallpaper URI", + ) + val tool = createTool(listOf(enumParam, uriParam)) + + val liveSchema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["A", "B"], + "description": "Mode parameter" + }, + "wallpaperUri": { + "type": "string", + "description": "Wallpaper URI" + } + }, + "required": ["mode"] + } + } + """, + ) + + assertEquals(expectedJson, liveSchema) + } + + @Test + fun convert_omitsEnumForIntParameterWithEnumValues() { + val intEnumParam = + AppFunctionParameterMetadata( + name = "priority", + isRequired = true, + dataType = + AppFunctionIntTypeMetadata(isNullable = false, enumValues = setOf(1, 2, 3)), + description = "Priority level", + ) + val tool = createTool(listOf(intEnumParam)) + + val liveSchema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "priority": { + "type": "integer", + "description": "Priority level" + } + }, + "required": ["priority"] + } + } + """, + ) + + assertEquals(expectedJson, liveSchema) + } + + @Test + fun convert_omitsFileReferenceFormatForUriObjectAndReferenceParameters() { + val uriObjectParam = + AppFunctionParameterMetadata( + name = "directUri", + isRequired = true, + dataType = + AppFunctionObjectTypeMetadata( + properties = emptyMap(), + required = emptyList(), + qualifiedName = "android.net.Uri", + isNullable = false, + ), + description = "Direct Uri object parameter", + ) + val uriRefParam = + AppFunctionParameterMetadata( + name = "refUri", + isRequired = false, + dataType = + AppFunctionReferenceTypeMetadata( + referenceDataType = "android.net.Uri", + isNullable = true, + ), + description = "Reference Uri parameter", + ) + val tool = createTool(listOf(uriObjectParam, uriRefParam)) + + val liveSchema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "directUri": { + "type": "string", + "description": "Direct Uri object parameter" + }, + "refUri": { + "type": "string", + "description": "Reference Uri parameter" + } + }, + "required": ["directUri"] + } + } + """, + ) + + assertEquals(expectedJson, liveSchema) + } + + @Test + fun getToolName_delegatesToGeminiToolConverter() { + val tool = createTool(emptyList()) + assertEquals("com_example_my_function", converter.getToolName(tool)) + } + + private fun createTool(parameters: List): AppFunctionMetadata = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = parameters, + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt index 8388855d..62cb16ab 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt @@ -520,68 +520,4 @@ class GeminiToolConverterTest { assertEquals(expectedJson, schema) } - - @Test - fun convertForLiveApi_normalizesEnumsAndStripsFileReferenceFormat() { - val enumParam = - AppFunctionParameterMetadata( - name = "mode", - isRequired = true, - dataType = - AppFunctionStringTypeMetadata(isNullable = false, enumValues = setOf("A", "B")), - description = "Mode parameter", - ) - val uriParam = - AppFunctionParameterMetadata( - name = "wallpaperUri", - isRequired = false, - dataType = AppFunctionStringTypeMetadata(isNullable = true), - description = "Wallpaper URI", - ) - val tool = - AppFunctionMetadata( - id = "com.example.my_function", - packageName = "com.example", - isEnabled = true, - schema = null, - parameters = listOf(enumParam, uriParam), - response = - AppFunctionResponseMetadata( - valueType = AppFunctionStringTypeMetadata(isNullable = false), - description = "", - ), - components = AppFunctionComponentsMetadata(emptyMap()), - description = "Test function", - deprecation = null, - ) - - val liveSchema = converter.convertForLiveApi(tool) - - val expectedJson = - Json.parseToJsonElement( - """ - { - "name": "com_example_my_function", - "description": "Test function", - "parameters": { - "type": "object", - "properties": { - "mode": { - "type": "string", - "enum": ["A", "B"], - "description": "Mode parameter" - }, - "wallpaperUri": { - "type": "string", - "description": "Wallpaper URI" - } - }, - "required": ["mode"] - } - } - """, - ) - - assertEquals(expectedJson, liveSchema) - } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt index b18b913c..6d6e7ca3 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt @@ -18,6 +18,8 @@ package com.example.appfunctions.agent.data import androidx.datastore.core.DataStore import androidx.datastore.preferences.core.PreferenceDataStoreFactory import androidx.datastore.preferences.core.Preferences +import androidx.datastore.preferences.core.edit +import androidx.datastore.preferences.core.stringPreferencesKey import com.example.appfunctions.agent.BuildConfig import kotlinx.coroutines.flow.first import kotlinx.coroutines.test.runTest @@ -92,4 +94,15 @@ class SettingsRepositoryTest { val value = repository.geminiLiveVoice.first() assertEquals(GeminiLiveVoice.AOEDE, value) } + + @Test + fun geminiLiveVoice_unrecognizedStoredValue_fallsBackToDefault() = + runTest { + dataStore.edit { preferences -> + preferences[stringPreferencesKey("gemini_live_voice")] = "UNRECOGNIZED_VOICE" + } + + val value = repository.geminiLiveVoice.first() + assertEquals(GeminiLiveVoice.DEFAULT, value) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ToolConverterTest.kt new file mode 100644 index 00000000..dbaf963a --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ToolConverterTest.kt @@ -0,0 +1,90 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ToolConverterTest { + private val fakeConverter = + object : ToolConverter { + override fun convert(tool: AppFunctionMetadata): String = getToolName(tool) + + override fun getToolName(tool: AppFunctionMetadata): String = tool.id.substringAfterLast('.') + } + + @Test + fun selectDistinctTools_prioritizesIdStartingWithPackageNameAndDeduplicatesByToolName() { + val legacyTool = createTool(packageName = "com.example.notes", id = "legacy.createNote") + val canonicalTool = + createTool(packageName = "com.example.notes", id = "com.example.notes.createNote") + val otherTool = + createTool(packageName = "com.example.notes", id = "com.example.notes.deleteNote") + + val distinct = + fakeConverter.selectDistinctTools(listOf(legacyTool, canonicalTool, otherTool)) + + assertEquals(listOf(canonicalTool, otherTool), distinct) + } + + @Test + fun findToolByName_returnsMatchingToolOrNull() { + val createNote = + createTool(packageName = "com.example.notes", id = "com.example.notes.createNote") + val deleteNote = + createTool(packageName = "com.example.notes", id = "com.example.notes.deleteNote") + val tools = listOf(createNote, deleteNote) + + assertEquals(createNote, fakeConverter.findToolByName("createNote", tools)) + assertEquals(deleteNote, fakeConverter.findToolByName("deleteNote", tools)) + assertNull(fakeConverter.findToolByName("unknownTool", tools)) + } + + @Test + fun resolveToolName_returnsConvertedNameWhenMatchedOrFallsBackToFunctionId() { + val createNote = + createTool(packageName = "com.example.notes", id = "com.example.notes.createNote") + val tools = listOf(createNote) + + assertEquals("createNote", fakeConverter.resolveToolName(createNote.id, tools)) + assertEquals("unregistered.id", fakeConverter.resolveToolName("unregistered.id", tools)) + } + + private fun createTool( + packageName: String, + id: String, + ): AppFunctionMetadata = + AppFunctionMetadata( + id = id, + packageName = packageName, + isEnabled = true, + schema = null, + parameters = emptyList(), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test tool", + deprecation = null, + ) +} From 4c8a94ed159e08271c0593783f4fada6b0509c26 Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Thu, 5 Nov 2026 15:53:02 +0100 Subject: [PATCH 3/3] refactor(agent): decouple LiveLlmProvider from Gemini defaults and add live integration tests --- .../GeminiLiveProviderImplIntegrationTest.kt | 36 ++ .../LiveLlmProviderIntegrationTestBase.kt | 449 ++++++++++++++++++ .../agent/data/GeminiLiveProviderImpl.kt | 38 +- .../agent/domain/LiveLlmProvider.kt | 24 +- .../agent/data/GeminiLiveProviderImplTest.kt | 6 +- .../agent/data/GeminiLiveToolConverterTest.kt | 278 +++++------ 6 files changed, 644 insertions(+), 187 deletions(-) create mode 100644 agent/app/src/androidTest/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplIntegrationTest.kt create mode 100644 agent/app/src/androidTest/java/com/example/appfunctions/agent/data/LiveLlmProviderIntegrationTestBase.kt diff --git a/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplIntegrationTest.kt b/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplIntegrationTest.kt new file mode 100644 index 00000000..94be3a7a --- /dev/null +++ b/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplIntegrationTest.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.example.appfunctions.agent.domain.LiveLlmProvider +import io.ktor.client.HttpClient +import org.junit.runner.RunWith + +/** Integration tests for [GeminiLiveProviderImpl]. */ +@RunWith(AndroidJUnit4::class) +class GeminiLiveProviderImplIntegrationTest : LiveLlmProviderIntegrationTestBase() { + override val apiKeyArgumentKey: String = "gemini_api_key" + override val inputSampleRateHz: Int = GeminiLiveProviderImpl.DEFAULT_INPUT_SAMPLE_RATE_HZ + override val voiceName: String = GeminiLiveVoice.DEFAULT.apiName + + override fun createProvider(httpClient: HttpClient): LiveLlmProvider { + return GeminiLiveProviderImpl( + httpClient = httpClient, + toolConverter = GeminiLiveToolConverter(GeminiToolConverter()), + ) + } +} diff --git a/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/LiveLlmProviderIntegrationTestBase.kt b/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/LiveLlmProviderIntegrationTestBase.kt new file mode 100644 index 00000000..ad55c002 --- /dev/null +++ b/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/LiveLlmProviderIntegrationTestBase.kt @@ -0,0 +1,449 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import androidx.test.platform.app.InstrumentationRegistry +import com.example.appfunctions.agent.domain.LiveLlmEvent +import com.example.appfunctions.agent.domain.LiveLlmProvider +import com.example.appfunctions.agent.domain.LiveLlmSession +import com.example.appfunctions.agent.domain.ToolOutput +import com.example.appfunctions.agent.domain.withSession +import io.ktor.client.HttpClient +import io.ktor.client.engine.okhttp.OkHttp +import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.websocket.WebSockets +import io.ktor.serialization.kotlinx.json.json +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.json.Json +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Assume.assumeTrue +import org.junit.Before +import org.junit.Test +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.Locale + +/** + * Base class for integration tests for [LiveLlmProvider] implementations. + * + * Subclasses must provide the specific live provider instance, instrumentation API key argument + * name, and input PCM sample rate. + */ +abstract class LiveLlmProviderIntegrationTestBase { + protected lateinit var provider: LiveLlmProvider + protected lateinit var apiKey: String + private var httpClient: HttpClient? = null + + /** Argument key used to retrieve the API key from instrumentation arguments. */ + protected abstract val apiKeyArgumentKey: String + + /** Input PCM sample rate in Hz expected by the live provider. */ + protected abstract val inputSampleRateHz: Int + + /** Optional voice identifier to request during live session setup. */ + protected open val voiceName: String? = null + + /** Creates the specific [LiveLlmProvider] instance under test. */ + protected abstract fun createProvider(httpClient: HttpClient): LiveLlmProvider + + @Before + fun setUp() { + val arguments = InstrumentationRegistry.getArguments() + apiKey = arguments.getString(apiKeyArgumentKey) ?: "" + assumeTrue( + "API key is required. Please provide it via instrumentation arguments.", + apiKey.isNotEmpty() && apiKey != "PLACEHOLDER", + ) + + val client = + HttpClient(OkHttp) { + install(ContentNegotiation) { + json( + Json { + ignoreUnknownKeys = true + prettyPrint = true + isLenient = true + }, + ) + } + install(HttpTimeout) { socketTimeoutMillis = SOCKET_TIMEOUT_MS } + install(WebSockets) { pingIntervalMillis = WEBSOCKET_PING_INTERVAL_MS } + } + httpClient = client + provider = createProvider(client) + } + + @After + fun tearDown() { + httpClient?.close() + } + + @Test + fun connect_withToolsAndVoice_completesHandshakeAndClosesCleanly() = + runBlocking { + val weatherTool = createWeatherTool() + var sessionEstablished = false + + provider.withSession( + apiKey = apiKey, + tools = listOf(weatherTool), + voiceName = voiceName, + ) { + sessionEstablished = true + } + + assertTrue(sessionEstablished) + } + + @Test + fun connect_invalidApiKey_throwsException() = + runBlocking { + try { + val session = provider.connect(apiKey = "INVALID_KEY", tools = emptyList()) + session.close() + fail("Expected exception when connecting with an invalid API key") + } catch (_: IllegalStateException) { + // Expected when server rejects handshake or closes before setupComplete + } + } + + @Test + fun session_audioPrompt_receivesAudioResponseAndTurnComplete() = + runBlocking { + val speechPcm = synthesizeSpeechPcm("Hello! Please say hi back in one short sentence.") + + provider.withSession( + apiKey = apiKey, + tools = emptyList(), + voiceName = voiceName, + ) { session -> + withTimeout(LIVE_TURN_TIMEOUT_MS) { + coroutineScope { + val eventsChannel = Channel(Channel.UNLIMITED) + val collectorJob = + launch { + session.events.collect { eventsChannel.send(it) } + } + val senderJob = + launch { + streamPcmWithTrailingSilence(session, speechPcm) + } + + val audioChunks = mutableListOf() + try { + while (true) { + when (val event = eventsChannel.receive()) { + is LiveLlmEvent.AudioChunk -> audioChunks.add(event) + is LiveLlmEvent.Error -> + fail("Unexpected LiveLlmEvent.Error: ${event.message}") + is LiveLlmEvent.TurnComplete -> break + else -> {} + } + } + } finally { + senderJob.cancel() + collectorJob.cancel() + } + + assertTrue("Expected non-empty audio chunks from model", audioChunks.isNotEmpty()) + assertTrue( + audioChunks.all { it.pcmBytes.isNotEmpty() && it.sampleRateHz > 0 }, + ) + } + } + } + } + + @Test + fun session_toolCallAndResponse_completesTurn() = + runBlocking { + val weatherTool = createWeatherTool() + val speechPcm = + synthesizeSpeechPcm("Use the weather tool to check the weather in Mountain View.") + + provider.withSession( + apiKey = apiKey, + tools = listOf(weatherTool), + voiceName = voiceName, + ) { session -> + withTimeout(LIVE_TURN_TIMEOUT_MS) { + coroutineScope { + val eventsChannel = Channel(Channel.UNLIMITED) + val collectorJob = + launch { + session.events.collect { eventsChannel.send(it) } + } + val senderJob = + launch { + streamPcmWithTrailingSilence(session, speechPcm) + } + + try { + var toolCallRequest: LiveLlmEvent.ToolCallRequest? = null + while (toolCallRequest == null) { + when (val event = eventsChannel.receive()) { + is LiveLlmEvent.ToolCallRequest -> toolCallRequest = event + is LiveLlmEvent.Error -> + fail("Unexpected LiveLlmEvent.Error: ${event.message}") + is LiveLlmEvent.TurnComplete -> + fail("Turn completed before expected tool call was requested") + else -> {} + } + } + senderJob.cancelAndJoin() + + assertTrue(toolCallRequest.calls.isNotEmpty()) + val toolCall = toolCallRequest.calls.first() + assertEquals("com.example.appfunctions.agent", toolCall.packageName) + assertEquals(weatherTool.id, toolCall.functionId) + + session.sendToolResponses( + listOf( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = """{"weather":"Sunny and 72 degrees in Mountain View"}""", + ), + ), + ) + + val postToolAudioChunks = mutableListOf() + while (true) { + when (val event = eventsChannel.receive()) { + is LiveLlmEvent.AudioChunk -> postToolAudioChunks.add(event) + is LiveLlmEvent.Error -> + fail("Unexpected LiveLlmEvent.Error: ${event.message}") + is LiveLlmEvent.TurnComplete -> { + if (postToolAudioChunks.isNotEmpty()) break + } + else -> {} + } + } + assertTrue( + "Expected model audio response after tool output", + postToolAudioChunks.isNotEmpty(), + ) + } finally { + senderJob.cancel() + collectorJob.cancel() + } + } + } + } + } + + private suspend fun streamPcmWithTrailingSilence( + session: LiveLlmSession, + speechPcm: ByteArray, + ) { + val bytesPerChunk = (inputSampleRateHz * BYTES_PER_SAMPLE * CHUNK_DURATION_MS) / 1000 + var offset = 0 + while (offset < speechPcm.size) { + val end = (offset + bytesPerChunk).coerceAtMost(speechPcm.size) + session.sendAudio(speechPcm.copyOfRange(offset, end), inputSampleRateHz) + offset = end + delay(STREAM_PACE_MS) + } + + val silenceChunk = ByteArray(bytesPerChunk) + repeat(TRAILING_SILENCE_CHUNKS) { + session.sendAudio(silenceChunk, inputSampleRateHz) + delay(STREAM_PACE_MS) + } + } + + private suspend fun synthesizeSpeechPcm(text: String): ByteArray { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val wavFile = File.createTempFile("live_llm_tts_", ".wav", context.cacheDir) + var tts: TextToSpeech? = null + try { + val initResult = CompletableDeferred() + tts = + withContext(Dispatchers.Main) { + TextToSpeech(context) { status -> initResult.complete(status) } + } + val status = withTimeout(TTS_TIMEOUT_MS) { initResult.await() } + assumeTrue("Android TextToSpeech initialization failed", status == TextToSpeech.SUCCESS) + + val langSupport = tts.setLanguage(Locale.US) + assumeTrue( + "US English TextToSpeech voice is unavailable on this device", + langSupport >= TextToSpeech.LANG_AVAILABLE, + ) + + val synthesisDone = CompletableDeferred() + val utteranceId = "live_test_utterance" + tts.setOnUtteranceProgressListener( + object : UtteranceProgressListener() { + override fun onStart(id: String?) {} + + override fun onDone(id: String?) { + synthesisDone.complete(true) + } + + @Deprecated("Deprecated in Java") + override fun onError(id: String?) { + synthesisDone.complete(false) + } + + override fun onError( + id: String?, + errorCode: Int, + ) { + synthesisDone.complete(false) + } + }, + ) + + val enqueueStatus = tts.synthesizeToFile(text, Bundle.EMPTY, wavFile, utteranceId) + assumeTrue( + "TextToSpeech.synthesizeToFile failed to enqueue", + enqueueStatus == TextToSpeech.SUCCESS, + ) + val completed = withTimeout(TTS_TIMEOUT_MS) { synthesisDone.await() } + assumeTrue("TextToSpeech synthesis failed to complete", completed) + + return decodeWavAndResampleTo16BitMonoPcm(wavFile.readBytes(), inputSampleRateHz) + } finally { + tts?.shutdown() + wavFile.delete() + } + } + + private fun decodeWavAndResampleTo16BitMonoPcm( + wavBytes: ByteArray, + targetSampleRateHz: Int, + ): ByteArray { + val buffer = ByteBuffer.wrap(wavBytes).order(ByteOrder.LITTLE_ENDIAN) + var offset = WAV_HEADER_PREAMBLE_BYTES + var channels = 1 + var sourceSampleRateHz = targetSampleRateHz + var pcmDataOffset = -1 + var pcmDataSize = 0 + + while (offset + WAV_CHUNK_HEADER_BYTES <= wavBytes.size) { + val chunkId = String(wavBytes, offset, 4, Charsets.US_ASCII) + val chunkSize = buffer.getInt(offset + 4) + val payloadStart = offset + WAV_CHUNK_HEADER_BYTES + if (chunkId == "fmt " && chunkSize >= 16) { + channels = buffer.getShort(payloadStart + 2).toInt().coerceAtLeast(1) + sourceSampleRateHz = buffer.getInt(payloadStart + 4) + } else if (chunkId == "data") { + pcmDataOffset = payloadStart + pcmDataSize = chunkSize.coerceAtMost(wavBytes.size - payloadStart) + break + } + offset = payloadStart + chunkSize + (chunkSize and 1) + } + check(pcmDataOffset >= 0 && pcmDataSize > 0) { "WAV file missing PCM data chunk" } + + val frameCount = pcmDataSize / (BYTES_PER_SAMPLE * channels) + val monoSamples = ShortArray(frameCount) + for (frame in 0 until frameCount) { + var sum = 0 + for (ch in 0 until channels) { + val byteIdx = pcmDataOffset + (frame * channels + ch) * BYTES_PER_SAMPLE + sum += buffer.getShort(byteIdx).toInt() + } + monoSamples[frame] = (sum / channels).toShort() + } + + if (sourceSampleRateHz == targetSampleRateHz) { + val out = ByteBuffer.allocate(monoSamples.size * BYTES_PER_SAMPLE).order(ByteOrder.LITTLE_ENDIAN) + for (sample in monoSamples) { + out.putShort(sample) + } + return out.array() + } + + val outFrameCount = + ((monoSamples.size.toLong() * targetSampleRateHz) / sourceSampleRateHz).toInt() + val resampled = + ByteBuffer.allocate(outFrameCount * BYTES_PER_SAMPLE).order(ByteOrder.LITTLE_ENDIAN) + for (i in 0 until outFrameCount) { + val srcPos = i.toDouble() * sourceSampleRateHz / targetSampleRateHz + val idx = srcPos.toInt().coerceIn(0, monoSamples.lastIndex) + val nextIdx = (idx + 1).coerceAtMost(monoSamples.lastIndex) + val frac = srcPos - idx + val interpolated = + (monoSamples[idx] * (1.0 - frac) + monoSamples[nextIdx] * frac).toInt().toShort() + resampled.putShort(interpolated) + } + return resampled.array() + } + + private fun createWeatherTool(): AppFunctionMetadata { + val stringType = AppFunctionStringTypeMetadata(isNullable = false) + return AppFunctionMetadata( + id = "com.example.appfunctions.agent.get_weather", + packageName = "com.example.appfunctions.agent", + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "location", + isRequired = true, + dataType = stringType, + description = "The city and state or location to get weather for", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = stringType, + description = "Returns the current weather description", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Gets the current weather for a given location.", + deprecation = null, + ) + } + + companion object { + private const val SOCKET_TIMEOUT_MS = 30_000L + private const val WEBSOCKET_PING_INTERVAL_MS = 20_000L + private const val TTS_TIMEOUT_MS = 15_000L + private const val LIVE_TURN_TIMEOUT_MS = 45_000L + private const val BYTES_PER_SAMPLE = 2 + private const val CHUNK_DURATION_MS = 100 + private const val STREAM_PACE_MS = 40L + private const val TRAILING_SILENCE_CHUNKS = 30 + private const val WAV_HEADER_PREAMBLE_BYTES = 12 + private const val WAV_CHUNK_HEADER_BYTES = 8 + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt index d4c67a8e..9e8b26d6 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt @@ -18,7 +18,6 @@ package com.example.appfunctions.agent.data import android.util.Log import androidx.annotation.VisibleForTesting import androidx.appfunctions.metadata.AppFunctionMetadata -import com.example.appfunctions.agent.domain.DEFAULT_LIVE_OUTPUT_SAMPLE_RATE_HZ import com.example.appfunctions.agent.domain.LiveLlmEvent import com.example.appfunctions.agent.domain.LiveLlmProvider import com.example.appfunctions.agent.domain.LiveLlmSession @@ -68,39 +67,34 @@ import javax.inject.Singleton */ @Singleton class GeminiLiveProviderImpl - @Inject - constructor( - private val httpClient: HttpClient, + @VisibleForTesting + internal constructor( private val toolConverter: GeminiLiveToolConverter, + private val webSocketConnector: suspend (wsUrl: String) -> WebSocketSession, ) : LiveLlmProvider { - private var webSocketConnector: suspend (wsUrl: String) -> WebSocketSession = { wsUrl -> - httpClient.webSocketSession(urlString = wsUrl) - } - - @VisibleForTesting - internal constructor( + @Inject + constructor( httpClient: HttpClient, toolConverter: GeminiLiveToolConverter, - webSocketConnector: suspend (wsUrl: String) -> WebSocketSession, ) : this( - httpClient = httpClient, toolConverter = toolConverter, - ) { - this.webSocketConnector = webSocketConnector - } + webSocketConnector = { wsUrl -> httpClient.webSocketSession(urlString = wsUrl) }, + ) override suspend fun connect( apiKey: String, tools: List, - voice: GeminiLiveVoice, + voiceName: String?, ): LiveLlmSession { + val resolvedVoiceName = + voiceName?.takeIf { it.isNotBlank() } ?: GeminiLiveVoice.DEFAULT.apiName val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" val webSocketSession = webSocketConnector(wsUrl) try { - val setupMessage = buildSetupMessage(tools = tools, voice = voice) + val setupMessage = buildSetupMessage(tools = tools, voiceName = resolvedVoiceName) Log.d( TAG, - "Sending Gemini Live setup for model $LIVE_MODEL_NAME (voice=${voice.apiName})", + "Sending Gemini Live setup for model $LIVE_MODEL_NAME (voice=$resolvedVoiceName)", ) webSocketSession.send(Frame.Text(setupMessage.toString())) @@ -387,7 +381,7 @@ class GeminiLiveProviderImpl .map { it.trim() } .firstOrNull { it.startsWith(MIME_TYPE_RATE_PARAM, ignoreCase = true) } val parsedRate = rateParam?.substringAfter('=')?.toIntOrNull() - return parsedRate ?: DEFAULT_LIVE_OUTPUT_SAMPLE_RATE_HZ + return parsedRate ?: DEFAULT_OUTPUT_SAMPLE_RATE_HZ } private fun parseToolCall( @@ -452,7 +446,7 @@ class GeminiLiveProviderImpl private fun buildSetupMessage( tools: List, - voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + voiceName: String, ): JsonObject { val functionDeclarations = toolConverter @@ -487,7 +481,7 @@ class GeminiLiveProviderImpl put( KEY_PREBUILT_VOICE_CONFIG, buildJsonObject { - put(KEY_VOICE_NAME, voice.apiName) + put(KEY_VOICE_NAME, voiceName) }, ) }, @@ -622,6 +616,8 @@ class GeminiLiveProviderImpl companion object { private const val TAG = "GeminiLiveProvider" const val LIVE_MODEL_NAME = "models/gemini-3.8-live" + const val DEFAULT_INPUT_SAMPLE_RATE_HZ = 16_000 + const val DEFAULT_OUTPUT_SAMPLE_RATE_HZ = 24_000 private const val GEMINI_LIVE_WS_URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" private const val CLOSE_REASON_CLIENT_ENDED = "Session ended by client" diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt index f04dc24f..8b00d472 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt @@ -16,28 +16,22 @@ package com.example.appfunctions.agent.domain import androidx.appfunctions.metadata.AppFunctionMetadata -import com.example.appfunctions.agent.data.GeminiLiveVoice import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.withContext -/** Default microphone PCM input sample rate in Hz for live audio sessions. */ -const val DEFAULT_LIVE_INPUT_SAMPLE_RATE_HZ = 16_000 - -/** Default model PCM output sample rate in Hz for live audio sessions. */ -const val DEFAULT_LIVE_OUTPUT_SAMPLE_RATE_HZ = 24_000 - /** Interface for LLM providers supporting bidirectional realtime audio and tool sessions. */ interface LiveLlmProvider { /** - * Connects to the live LLM backend, completes session initialization with [tools] and [voice], - * and returns the active [LiveLlmSession]. The caller is responsible for closing the returned - * session via [LiveLlmSession.close] or using [withSession]. + * Connects to the live LLM backend, completes session initialization with [tools] and optional + * provider-specific [voiceName], and returns the active [LiveLlmSession]. The caller is + * responsible for closing the returned session via [LiveLlmSession.close] or using + * [withSession]. */ suspend fun connect( apiKey: String, tools: List, - voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + voiceName: String? = null, ): LiveLlmSession } @@ -48,10 +42,10 @@ interface LiveLlmProvider { suspend inline fun LiveLlmProvider.withSession( apiKey: String, tools: List, - voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + voiceName: String? = null, block: (LiveLlmSession) -> Unit, ) { - val session = connect(apiKey = apiKey, tools = tools, voice = voice) + val session = connect(apiKey = apiKey, tools = tools, voiceName = voiceName) try { block(session) } finally { @@ -71,7 +65,7 @@ interface LiveLlmSession { */ suspend fun sendAudio( pcmBytes: ByteArray, - sampleRateHz: Int = DEFAULT_LIVE_INPUT_SAMPLE_RATE_HZ, + sampleRateHz: Int, ) /** @@ -98,7 +92,7 @@ sealed class LiveLlmEvent { /** Raw 16-bit mono PCM audio chunk generated by the model for playback at [sampleRateHz]. */ data class AudioChunk( val pcmBytes: ByteArray, - val sampleRateHz: Int = DEFAULT_LIVE_OUTPUT_SAMPLE_RATE_HZ, + val sampleRateHz: Int, ) : LiveLlmEvent() { override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt index 3e228ad2..8770e753 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt @@ -24,7 +24,6 @@ import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata import com.example.appfunctions.agent.domain.LiveLlmEvent import com.example.appfunctions.agent.domain.ToolOutput import com.example.appfunctions.agent.domain.withSession -import io.ktor.client.HttpClient import io.ktor.websocket.CloseReason import io.ktor.websocket.Frame import io.ktor.websocket.WebSocketSession @@ -55,7 +54,6 @@ import java.util.Base64 import kotlin.coroutines.CoroutineContext class GeminiLiveProviderImplTest { - private lateinit var httpClient: HttpClient private lateinit var toolConverter: GeminiLiveToolConverter @Before @@ -66,7 +64,6 @@ class GeminiLiveProviderImplTest { every { Log.w(any(), any(), any()) } returns 0 every { Log.e(any(), any(), any()) } returns 0 - httpClient = mockk(relaxed = true) toolConverter = GeminiLiveToolConverter(GeminiToolConverter()) } @@ -93,7 +90,7 @@ class GeminiLiveProviderImplTest { provider.withSession( apiKey = "secret-key", tools = listOf(tool), - voice = GeminiLiveVoice.AOEDE, + voiceName = GeminiLiveVoice.AOEDE.apiName, ) {} assertTrue(capturedWsUrl!!.endsWith("?key=secret-key")) @@ -621,7 +618,6 @@ class GeminiLiveProviderImplTest { private fun createProvider(connector: suspend (String) -> WebSocketSession): GeminiLiveProviderImpl = GeminiLiveProviderImpl( - httpClient = httpClient, toolConverter = toolConverter, webSocketConnector = connector, ) diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt index 7c75f8df..6e9e9528 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt @@ -15,193 +15,179 @@ */ package com.example.appfunctions.agent.data -import androidx.appfunctions.metadata.AppFunctionComponentsMetadata -import androidx.appfunctions.metadata.AppFunctionIntTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata -import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata -import androidx.appfunctions.metadata.AppFunctionParameterMetadata -import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata -import androidx.appfunctions.metadata.AppFunctionResponseMetadata -import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject import org.junit.Assert.assertEquals import org.junit.Before import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -@RunWith(RobolectricTestRunner::class) class GeminiLiveToolConverterTest { + private lateinit var geminiToolConverter: GeminiToolConverter private lateinit var converter: GeminiLiveToolConverter + private lateinit var tool: AppFunctionMetadata @Before fun setUp() { - converter = GeminiLiveToolConverter(GeminiToolConverter()) + geminiToolConverter = mockk() + converter = GeminiLiveToolConverter(geminiToolConverter) + tool = mockk() } @Test - fun convert_normalizesStringEnumsAndStripsFileReferenceFormat() { - val enumParam = - AppFunctionParameterMetadata( - name = "mode", - isRequired = true, - dataType = - AppFunctionStringTypeMetadata(isNullable = false, enumValues = setOf("A", "B")), - description = "Mode parameter", - ) - val uriParam = - AppFunctionParameterMetadata( - name = "wallpaperUri", - isRequired = false, - dataType = AppFunctionStringTypeMetadata(isNullable = true), - description = "Wallpaper URI", - ) - val tool = createTool(listOf(enumParam, uriParam)) - - val liveSchema = converter.convert(tool) + fun convert_stringEnum_replacesEnumsKeyWithEnum() { + every { geminiToolConverter.convert(tool) } returns + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enums": ["A", "B"] + } + } + } + } + """.trimIndent(), + ).jsonObject - val expectedJson = + val expected = Json.parseToJsonElement( """ - { - "name": "com_example_my_function", - "description": "Test function", - "parameters": { - "type": "object", - "properties": { - "mode": { - "type": "string", - "enum": ["A", "B"], - "description": "Mode parameter" - }, - "wallpaperUri": { - "type": "string", - "description": "Wallpaper URI" + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["A", "B"] + } } - }, - "required": ["mode"] + } } - } - """, - ) + """.trimIndent(), + ).jsonObject - assertEquals(expectedJson, liveSchema) + assertEquals(expected, converter.convert(tool)) } @Test - fun convert_omitsEnumForIntParameterWithEnumValues() { - val intEnumParam = - AppFunctionParameterMetadata( - name = "priority", - isRequired = true, - dataType = - AppFunctionIntTypeMetadata(isNullable = false, enumValues = setOf(1, 2, 3)), - description = "Priority level", - ) - val tool = createTool(listOf(intEnumParam)) - - val liveSchema = converter.convert(tool) + fun convert_nonStringEnum_omitsEnumsKey() { + every { geminiToolConverter.convert(tool) } returns + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "priority": { + "type": "integer", + "enums": [1, 2, 3] + } + } + } + } + """.trimIndent(), + ).jsonObject - val expectedJson = + val expected = Json.parseToJsonElement( """ - { - "name": "com_example_my_function", - "description": "Test function", - "parameters": { - "type": "object", - "properties": { - "priority": { - "type": "integer", - "description": "Priority level" + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "priority": { + "type": "integer" + } } - }, - "required": ["priority"] + } } - } - """, - ) + """.trimIndent(), + ).jsonObject - assertEquals(expectedJson, liveSchema) + assertEquals(expected, converter.convert(tool)) } @Test - fun convert_omitsFileReferenceFormatForUriObjectAndReferenceParameters() { - val uriObjectParam = - AppFunctionParameterMetadata( - name = "directUri", - isRequired = true, - dataType = - AppFunctionObjectTypeMetadata( - properties = emptyMap(), - required = emptyList(), - qualifiedName = "android.net.Uri", - isNullable = false, - ), - description = "Direct Uri object parameter", - ) - val uriRefParam = - AppFunctionParameterMetadata( - name = "refUri", - isRequired = false, - dataType = - AppFunctionReferenceTypeMetadata( - referenceDataType = "android.net.Uri", - isNullable = true, - ), - description = "Reference Uri parameter", - ) - val tool = createTool(listOf(uriObjectParam, uriRefParam)) - - val liveSchema = converter.convert(tool) + fun convert_fileReferenceFormat_stripsFileReferenceRecursivelyAndKeepsOtherFormats() { + every { geminiToolConverter.convert(tool) } returns + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "wallpaperUri": { + "type": "string", + "format": "file_reference" + }, + "nested": { + "type": "object", + "properties": { + "attachmentUri": { + "type": "string", + "format": "file_reference" + }, + "createdAt": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + """.trimIndent(), + ).jsonObject - val expectedJson = + val expected = Json.parseToJsonElement( """ - { - "name": "com_example_my_function", - "description": "Test function", - "parameters": { - "type": "object", - "properties": { - "directUri": { - "type": "string", - "description": "Direct Uri object parameter" - }, - "refUri": { - "type": "string", - "description": "Reference Uri parameter" + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "wallpaperUri": { + "type": "string" + }, + "nested": { + "type": "object", + "properties": { + "attachmentUri": { + "type": "string" + }, + "createdAt": { + "type": "string", + "format": "date-time" + } + } + } } - }, - "required": ["directUri"] + } } - } - """, - ) + """.trimIndent(), + ).jsonObject - assertEquals(expectedJson, liveSchema) + assertEquals(expected, converter.convert(tool)) } @Test fun getToolName_delegatesToGeminiToolConverter() { - val tool = createTool(emptyList()) - assertEquals("com_example_my_function", converter.getToolName(tool)) - } + every { geminiToolConverter.getToolName(tool) } returns "delegated_tool_name" - private fun createTool(parameters: List): AppFunctionMetadata = - AppFunctionMetadata( - id = "com.example.my_function", - packageName = "com.example", - isEnabled = true, - schema = null, - parameters = parameters, - response = - AppFunctionResponseMetadata( - valueType = AppFunctionStringTypeMetadata(isNullable = false), - description = "", - ), - components = AppFunctionComponentsMetadata(emptyMap()), - description = "Test function", - deprecation = null, - ) + assertEquals("delegated_tool_name", converter.getToolName(tool)) + verify(exactly = 1) { geminiToolConverter.getToolName(tool) } + } }