diff --git a/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplIntegrationTest.kt b/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplIntegrationTest.kt new file mode 100644 index 0000000..94be3a7 --- /dev/null +++ b/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplIntegrationTest.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.example.appfunctions.agent.domain.LiveLlmProvider +import io.ktor.client.HttpClient +import org.junit.runner.RunWith + +/** Integration tests for [GeminiLiveProviderImpl]. */ +@RunWith(AndroidJUnit4::class) +class GeminiLiveProviderImplIntegrationTest : LiveLlmProviderIntegrationTestBase() { + override val apiKeyArgumentKey: String = "gemini_api_key" + override val inputSampleRateHz: Int = GeminiLiveProviderImpl.DEFAULT_INPUT_SAMPLE_RATE_HZ + override val voiceName: String = GeminiLiveVoice.DEFAULT.apiName + + override fun createProvider(httpClient: HttpClient): LiveLlmProvider { + return GeminiLiveProviderImpl( + httpClient = httpClient, + toolConverter = GeminiLiveToolConverter(GeminiToolConverter()), + ) + } +} diff --git a/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/LiveLlmProviderIntegrationTestBase.kt b/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/LiveLlmProviderIntegrationTestBase.kt new file mode 100644 index 0000000..ad55c00 --- /dev/null +++ b/agent/app/src/androidTest/java/com/example/appfunctions/agent/data/LiveLlmProviderIntegrationTestBase.kt @@ -0,0 +1,449 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import androidx.test.platform.app.InstrumentationRegistry +import com.example.appfunctions.agent.domain.LiveLlmEvent +import com.example.appfunctions.agent.domain.LiveLlmProvider +import com.example.appfunctions.agent.domain.LiveLlmSession +import com.example.appfunctions.agent.domain.ToolOutput +import com.example.appfunctions.agent.domain.withSession +import io.ktor.client.HttpClient +import io.ktor.client.engine.okhttp.OkHttp +import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.websocket.WebSockets +import io.ktor.serialization.kotlinx.json.json +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.json.Json +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Assume.assumeTrue +import org.junit.Before +import org.junit.Test +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.Locale + +/** + * Base class for integration tests for [LiveLlmProvider] implementations. + * + * Subclasses must provide the specific live provider instance, instrumentation API key argument + * name, and input PCM sample rate. + */ +abstract class LiveLlmProviderIntegrationTestBase { + protected lateinit var provider: LiveLlmProvider + protected lateinit var apiKey: String + private var httpClient: HttpClient? = null + + /** Argument key used to retrieve the API key from instrumentation arguments. */ + protected abstract val apiKeyArgumentKey: String + + /** Input PCM sample rate in Hz expected by the live provider. */ + protected abstract val inputSampleRateHz: Int + + /** Optional voice identifier to request during live session setup. */ + protected open val voiceName: String? = null + + /** Creates the specific [LiveLlmProvider] instance under test. */ + protected abstract fun createProvider(httpClient: HttpClient): LiveLlmProvider + + @Before + fun setUp() { + val arguments = InstrumentationRegistry.getArguments() + apiKey = arguments.getString(apiKeyArgumentKey) ?: "" + assumeTrue( + "API key is required. Please provide it via instrumentation arguments.", + apiKey.isNotEmpty() && apiKey != "PLACEHOLDER", + ) + + val client = + HttpClient(OkHttp) { + install(ContentNegotiation) { + json( + Json { + ignoreUnknownKeys = true + prettyPrint = true + isLenient = true + }, + ) + } + install(HttpTimeout) { socketTimeoutMillis = SOCKET_TIMEOUT_MS } + install(WebSockets) { pingIntervalMillis = WEBSOCKET_PING_INTERVAL_MS } + } + httpClient = client + provider = createProvider(client) + } + + @After + fun tearDown() { + httpClient?.close() + } + + @Test + fun connect_withToolsAndVoice_completesHandshakeAndClosesCleanly() = + runBlocking { + val weatherTool = createWeatherTool() + var sessionEstablished = false + + provider.withSession( + apiKey = apiKey, + tools = listOf(weatherTool), + voiceName = voiceName, + ) { + sessionEstablished = true + } + + assertTrue(sessionEstablished) + } + + @Test + fun connect_invalidApiKey_throwsException() = + runBlocking { + try { + val session = provider.connect(apiKey = "INVALID_KEY", tools = emptyList()) + session.close() + fail("Expected exception when connecting with an invalid API key") + } catch (_: IllegalStateException) { + // Expected when server rejects handshake or closes before setupComplete + } + } + + @Test + fun session_audioPrompt_receivesAudioResponseAndTurnComplete() = + runBlocking { + val speechPcm = synthesizeSpeechPcm("Hello! Please say hi back in one short sentence.") + + provider.withSession( + apiKey = apiKey, + tools = emptyList(), + voiceName = voiceName, + ) { session -> + withTimeout(LIVE_TURN_TIMEOUT_MS) { + coroutineScope { + val eventsChannel = Channel(Channel.UNLIMITED) + val collectorJob = + launch { + session.events.collect { eventsChannel.send(it) } + } + val senderJob = + launch { + streamPcmWithTrailingSilence(session, speechPcm) + } + + val audioChunks = mutableListOf() + try { + while (true) { + when (val event = eventsChannel.receive()) { + is LiveLlmEvent.AudioChunk -> audioChunks.add(event) + is LiveLlmEvent.Error -> + fail("Unexpected LiveLlmEvent.Error: ${event.message}") + is LiveLlmEvent.TurnComplete -> break + else -> {} + } + } + } finally { + senderJob.cancel() + collectorJob.cancel() + } + + assertTrue("Expected non-empty audio chunks from model", audioChunks.isNotEmpty()) + assertTrue( + audioChunks.all { it.pcmBytes.isNotEmpty() && it.sampleRateHz > 0 }, + ) + } + } + } + } + + @Test + fun session_toolCallAndResponse_completesTurn() = + runBlocking { + val weatherTool = createWeatherTool() + val speechPcm = + synthesizeSpeechPcm("Use the weather tool to check the weather in Mountain View.") + + provider.withSession( + apiKey = apiKey, + tools = listOf(weatherTool), + voiceName = voiceName, + ) { session -> + withTimeout(LIVE_TURN_TIMEOUT_MS) { + coroutineScope { + val eventsChannel = Channel(Channel.UNLIMITED) + val collectorJob = + launch { + session.events.collect { eventsChannel.send(it) } + } + val senderJob = + launch { + streamPcmWithTrailingSilence(session, speechPcm) + } + + try { + var toolCallRequest: LiveLlmEvent.ToolCallRequest? = null + while (toolCallRequest == null) { + when (val event = eventsChannel.receive()) { + is LiveLlmEvent.ToolCallRequest -> toolCallRequest = event + is LiveLlmEvent.Error -> + fail("Unexpected LiveLlmEvent.Error: ${event.message}") + is LiveLlmEvent.TurnComplete -> + fail("Turn completed before expected tool call was requested") + else -> {} + } + } + senderJob.cancelAndJoin() + + assertTrue(toolCallRequest.calls.isNotEmpty()) + val toolCall = toolCallRequest.calls.first() + assertEquals("com.example.appfunctions.agent", toolCall.packageName) + assertEquals(weatherTool.id, toolCall.functionId) + + session.sendToolResponses( + listOf( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = """{"weather":"Sunny and 72 degrees in Mountain View"}""", + ), + ), + ) + + val postToolAudioChunks = mutableListOf() + while (true) { + when (val event = eventsChannel.receive()) { + is LiveLlmEvent.AudioChunk -> postToolAudioChunks.add(event) + is LiveLlmEvent.Error -> + fail("Unexpected LiveLlmEvent.Error: ${event.message}") + is LiveLlmEvent.TurnComplete -> { + if (postToolAudioChunks.isNotEmpty()) break + } + else -> {} + } + } + assertTrue( + "Expected model audio response after tool output", + postToolAudioChunks.isNotEmpty(), + ) + } finally { + senderJob.cancel() + collectorJob.cancel() + } + } + } + } + } + + private suspend fun streamPcmWithTrailingSilence( + session: LiveLlmSession, + speechPcm: ByteArray, + ) { + val bytesPerChunk = (inputSampleRateHz * BYTES_PER_SAMPLE * CHUNK_DURATION_MS) / 1000 + var offset = 0 + while (offset < speechPcm.size) { + val end = (offset + bytesPerChunk).coerceAtMost(speechPcm.size) + session.sendAudio(speechPcm.copyOfRange(offset, end), inputSampleRateHz) + offset = end + delay(STREAM_PACE_MS) + } + + val silenceChunk = ByteArray(bytesPerChunk) + repeat(TRAILING_SILENCE_CHUNKS) { + session.sendAudio(silenceChunk, inputSampleRateHz) + delay(STREAM_PACE_MS) + } + } + + private suspend fun synthesizeSpeechPcm(text: String): ByteArray { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val wavFile = File.createTempFile("live_llm_tts_", ".wav", context.cacheDir) + var tts: TextToSpeech? = null + try { + val initResult = CompletableDeferred() + tts = + withContext(Dispatchers.Main) { + TextToSpeech(context) { status -> initResult.complete(status) } + } + val status = withTimeout(TTS_TIMEOUT_MS) { initResult.await() } + assumeTrue("Android TextToSpeech initialization failed", status == TextToSpeech.SUCCESS) + + val langSupport = tts.setLanguage(Locale.US) + assumeTrue( + "US English TextToSpeech voice is unavailable on this device", + langSupport >= TextToSpeech.LANG_AVAILABLE, + ) + + val synthesisDone = CompletableDeferred() + val utteranceId = "live_test_utterance" + tts.setOnUtteranceProgressListener( + object : UtteranceProgressListener() { + override fun onStart(id: String?) {} + + override fun onDone(id: String?) { + synthesisDone.complete(true) + } + + @Deprecated("Deprecated in Java") + override fun onError(id: String?) { + synthesisDone.complete(false) + } + + override fun onError( + id: String?, + errorCode: Int, + ) { + synthesisDone.complete(false) + } + }, + ) + + val enqueueStatus = tts.synthesizeToFile(text, Bundle.EMPTY, wavFile, utteranceId) + assumeTrue( + "TextToSpeech.synthesizeToFile failed to enqueue", + enqueueStatus == TextToSpeech.SUCCESS, + ) + val completed = withTimeout(TTS_TIMEOUT_MS) { synthesisDone.await() } + assumeTrue("TextToSpeech synthesis failed to complete", completed) + + return decodeWavAndResampleTo16BitMonoPcm(wavFile.readBytes(), inputSampleRateHz) + } finally { + tts?.shutdown() + wavFile.delete() + } + } + + private fun decodeWavAndResampleTo16BitMonoPcm( + wavBytes: ByteArray, + targetSampleRateHz: Int, + ): ByteArray { + val buffer = ByteBuffer.wrap(wavBytes).order(ByteOrder.LITTLE_ENDIAN) + var offset = WAV_HEADER_PREAMBLE_BYTES + var channels = 1 + var sourceSampleRateHz = targetSampleRateHz + var pcmDataOffset = -1 + var pcmDataSize = 0 + + while (offset + WAV_CHUNK_HEADER_BYTES <= wavBytes.size) { + val chunkId = String(wavBytes, offset, 4, Charsets.US_ASCII) + val chunkSize = buffer.getInt(offset + 4) + val payloadStart = offset + WAV_CHUNK_HEADER_BYTES + if (chunkId == "fmt " && chunkSize >= 16) { + channels = buffer.getShort(payloadStart + 2).toInt().coerceAtLeast(1) + sourceSampleRateHz = buffer.getInt(payloadStart + 4) + } else if (chunkId == "data") { + pcmDataOffset = payloadStart + pcmDataSize = chunkSize.coerceAtMost(wavBytes.size - payloadStart) + break + } + offset = payloadStart + chunkSize + (chunkSize and 1) + } + check(pcmDataOffset >= 0 && pcmDataSize > 0) { "WAV file missing PCM data chunk" } + + val frameCount = pcmDataSize / (BYTES_PER_SAMPLE * channels) + val monoSamples = ShortArray(frameCount) + for (frame in 0 until frameCount) { + var sum = 0 + for (ch in 0 until channels) { + val byteIdx = pcmDataOffset + (frame * channels + ch) * BYTES_PER_SAMPLE + sum += buffer.getShort(byteIdx).toInt() + } + monoSamples[frame] = (sum / channels).toShort() + } + + if (sourceSampleRateHz == targetSampleRateHz) { + val out = ByteBuffer.allocate(monoSamples.size * BYTES_PER_SAMPLE).order(ByteOrder.LITTLE_ENDIAN) + for (sample in monoSamples) { + out.putShort(sample) + } + return out.array() + } + + val outFrameCount = + ((monoSamples.size.toLong() * targetSampleRateHz) / sourceSampleRateHz).toInt() + val resampled = + ByteBuffer.allocate(outFrameCount * BYTES_PER_SAMPLE).order(ByteOrder.LITTLE_ENDIAN) + for (i in 0 until outFrameCount) { + val srcPos = i.toDouble() * sourceSampleRateHz / targetSampleRateHz + val idx = srcPos.toInt().coerceIn(0, monoSamples.lastIndex) + val nextIdx = (idx + 1).coerceAtMost(monoSamples.lastIndex) + val frac = srcPos - idx + val interpolated = + (monoSamples[idx] * (1.0 - frac) + monoSamples[nextIdx] * frac).toInt().toShort() + resampled.putShort(interpolated) + } + return resampled.array() + } + + private fun createWeatherTool(): AppFunctionMetadata { + val stringType = AppFunctionStringTypeMetadata(isNullable = false) + return AppFunctionMetadata( + id = "com.example.appfunctions.agent.get_weather", + packageName = "com.example.appfunctions.agent", + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "location", + isRequired = true, + dataType = stringType, + description = "The city and state or location to get weather for", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = stringType, + description = "Returns the current weather description", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Gets the current weather for a given location.", + deprecation = null, + ) + } + + companion object { + private const val SOCKET_TIMEOUT_MS = 30_000L + private const val WEBSOCKET_PING_INTERVAL_MS = 20_000L + private const val TTS_TIMEOUT_MS = 15_000L + private const val LIVE_TURN_TIMEOUT_MS = 45_000L + private const val BYTES_PER_SAMPLE = 2 + private const val CHUNK_DURATION_MS = 100 + private const val STREAM_PACE_MS = 40L + private const val TRAILING_SILENCE_CHUNKS = 30 + private const val WAV_HEADER_PREAMBLE_BYTES = 12 + private const val WAV_CHUNK_HEADER_BYTES = 8 + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt index aac24d4..329fe35 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt @@ -37,6 +37,7 @@ class DataStoreSettingsRepository val GEMINI_API_KEY = stringPreferencesKey("gemini_api_key") val SELECTED_PROVIDER = stringPreferencesKey("selected_provider") val SERVICE_TIER = stringPreferencesKey("service_tier") + val GEMINI_LIVE_VOICE = stringPreferencesKey("gemini_live_voice") val PINNED_APPS = stringSetPreferencesKey("pinned_apps") val DISCONNECTED_APPS = stringSetPreferencesKey("disconnected_apps") } @@ -65,6 +66,11 @@ class DataStoreSettingsRepository } ?: ServiceTier.PRIORITY } + override val geminiLiveVoice: Flow = + dataStore.data.map { preferences -> + GeminiLiveVoice.fromStoredValue(preferences[PreferencesKeys.GEMINI_LIVE_VOICE]) + } + override val pinnedApps: Flow> = dataStore.data.map { preferences -> preferences[PreferencesKeys.PINNED_APPS] ?: emptySet() } @@ -87,6 +93,12 @@ class DataStoreSettingsRepository dataStore.edit { preferences -> preferences[PreferencesKeys.SERVICE_TIER] = tier.name } } + override suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) { + dataStore.edit { preferences -> + preferences[PreferencesKeys.GEMINI_LIVE_VOICE] = voice.name + } + } + override suspend fun setAppPinned( packageName: String, pinned: Boolean, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt new file mode 100644 index 0000000..9e8b26d --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt @@ -0,0 +1,670 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.util.Log +import androidx.annotation.VisibleForTesting +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.domain.LiveLlmEvent +import com.example.appfunctions.agent.domain.LiveLlmProvider +import com.example.appfunctions.agent.domain.LiveLlmSession +import com.example.appfunctions.agent.domain.LlmResponsePart +import com.example.appfunctions.agent.domain.ToolOutput +import com.example.appfunctions.agent.domain.appfunction.findToolByName +import com.example.appfunctions.agent.domain.appfunction.resolveToolName +import com.example.appfunctions.agent.domain.appfunction.selectDistinctTools +import com.example.appfunctions.agent.domain.toArgumentsMap +import io.ktor.client.HttpClient +import io.ktor.client.plugins.websocket.webSocketSession +import io.ktor.websocket.CloseReason +import io.ktor.websocket.Frame +import io.ktor.websocket.WebSocketSession +import io.ktor.websocket.close +import io.ktor.websocket.readBytes +import io.ktor.websocket.readReason +import io.ktor.websocket.readText +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.channels.ClosedReceiveChannelException +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.isActive +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put +import java.time.LocalDate +import java.util.Base64 +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Gemini implementation of [LiveLlmProvider] over the `BidiGenerateContent` WebSocket API + * (`models/gemini-3.8-live`). + */ +@Singleton +class GeminiLiveProviderImpl + @VisibleForTesting + internal constructor( + private val toolConverter: GeminiLiveToolConverter, + private val webSocketConnector: suspend (wsUrl: String) -> WebSocketSession, + ) : LiveLlmProvider { + @Inject + constructor( + httpClient: HttpClient, + toolConverter: GeminiLiveToolConverter, + ) : this( + toolConverter = toolConverter, + webSocketConnector = { wsUrl -> httpClient.webSocketSession(urlString = wsUrl) }, + ) + + override suspend fun connect( + apiKey: String, + tools: List, + voiceName: String?, + ): LiveLlmSession { + val resolvedVoiceName = + voiceName?.takeIf { it.isNotBlank() } ?: GeminiLiveVoice.DEFAULT.apiName + val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" + val webSocketSession = webSocketConnector(wsUrl) + try { + val setupMessage = buildSetupMessage(tools = tools, voiceName = resolvedVoiceName) + Log.d( + TAG, + "Sending Gemini Live setup for model $LIVE_MODEL_NAME (voice=$resolvedVoiceName)", + ) + webSocketSession.send(Frame.Text(setupMessage.toString())) + + val firstFrameText = + when (val firstFrame = webSocketSession.receiveNextFrame()) { + is FrameReadResult.JsonText -> firstFrame.text + is FrameReadResult.NormalClose -> + throw IllegalStateException( + "WebSocket closed before $KEY_SETUP_COMPLETE was received", + ) + is FrameReadResult.ErrorClose -> + throw IllegalStateException( + "WebSocket failed before $KEY_SETUP_COMPLETE was received: ${firstFrame.message}", + firstFrame.cause, + ) + } + val firstJson = Json.parseToJsonElement(firstFrameText).jsonObject + if (!firstJson.containsKey(KEY_SETUP_COMPLETE)) { + throw IllegalStateException("Unexpected setup response: $firstFrameText") + } + + return createSession(webSocketSession = webSocketSession, tools = tools) + } catch (t: Throwable) { + withContext(NonCancellable) { + closeWebSocketQuietly(webSocketSession) + } + throw t + } + } + + private fun createSession( + webSocketSession: WebSocketSession, + tools: List, + ): LiveLlmSession = + object : LiveLlmSession { + override val events: Flow = + flow { + while (currentCoroutineContext().isActive && webSocketSession.isActive) { + val messageText = + when (val frameResult = webSocketSession.receiveNextFrame()) { + is FrameReadResult.JsonText -> frameResult.text + is FrameReadResult.NormalClose -> break + is FrameReadResult.ErrorClose -> { + emit( + LiveLlmEvent.Error( + message = frameResult.message, + cause = frameResult.cause, + ), + ) + break + } + } + val serverJson = + try { + Json.parseToJsonElement(messageText).jsonObject + } catch (e: Exception) { + Log.w(TAG, "Failed to parse Gemini Live server frame", e) + emit( + LiveLlmEvent.Error( + message = "Failed to parse Gemini Live server frame", + cause = e, + ), + ) + continue + } + + for (event in parseServerMessage(serverJson, tools)) { + emit(event) + } + } + } + + override suspend fun sendAudio( + pcmBytes: ByteArray, + sampleRateHz: Int, + ) { + val realtimeInput = + buildRealtimeAudioMessage( + pcmBytes = pcmBytes, + sampleRateHz = sampleRateHz, + ) + webSocketSession.send(Frame.Text(realtimeInput.toString())) + } + + override suspend fun sendToolResponses(outputs: List) { + val toolResponseMessage = buildToolResponseMessage(outputs, tools) + Log.d(TAG, "Sending Gemini Live toolResponse: $toolResponseMessage") + webSocketSession.send(Frame.Text(toolResponseMessage.toString())) + } + + override suspend fun close() { + closeWebSocketQuietly(webSocketSession) + } + } + + private suspend fun closeWebSocketQuietly(webSocketSession: WebSocketSession) { + try { + webSocketSession.close( + CloseReason( + CloseReason.Codes.NORMAL, + CLOSE_REASON_CLIENT_ENDED, + ), + ) + } catch (e: Exception) { + Log.w(TAG, "Failed to close Gemini Live WebSocket cleanly", e) + } + } + + private suspend fun WebSocketSession.receiveNextFrame(): FrameReadResult { + while (isActive) { + val frame = + try { + incoming.receive() + } catch (_: ClosedReceiveChannelException) { + return FrameReadResult.NormalClose + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Log.w(TAG, "Gemini Live WebSocket transport error", e) + return FrameReadResult.ErrorClose( + message = "WebSocket transport error: ${e.message}", + cause = e, + ) + } + when (frame) { + is Frame.Text -> return FrameReadResult.JsonText(frame.readText()) + is Frame.Binary -> return FrameReadResult.JsonText(frame.readBytes().decodeToString()) + is Frame.Close -> { + val reason = frame.readReason() + return if (reason != null && reason.code != CloseReason.Codes.NORMAL.code) { + Log.w( + TAG, + "Gemini Live WebSocket closed by server: code=${reason.code}, message=${reason.message}", + ) + FrameReadResult.ErrorClose( + message = + "WebSocket closed by server (code=${reason.code}): ${reason.message}", + ) + } else { + FrameReadResult.NormalClose + } + } + else -> continue + } + } + return FrameReadResult.NormalClose + } + + private fun parseServerMessage( + serverJson: JsonObject, + tools: List, + ): List { + return when { + KEY_SERVER_CONTENT in serverJson -> { + val serverContent = serverJson[KEY_SERVER_CONTENT]?.jsonObject + if (serverContent != null) { + parseServerContent(serverContent) + } else { + listOf( + LiveLlmEvent.Error( + "Invalid '$KEY_SERVER_CONTENT' payload in server message: $serverJson", + ), + ) + } + } + KEY_TOOL_CALL in serverJson -> { + val toolCallObj = serverJson[KEY_TOOL_CALL]?.jsonObject + if (toolCallObj != null) { + parseToolCall(toolCallObj, tools) + } else { + listOf( + LiveLlmEvent.Error( + "Invalid '$KEY_TOOL_CALL' payload in server message: $serverJson", + ), + ) + } + } + else -> emptyList() + } + } + + private fun parseServerContent(serverContent: JsonObject): List { + return when { + serverContent[KEY_INTERRUPTED]?.jsonPrimitive?.booleanOrNull == true -> { + listOf(LiveLlmEvent.Interrupted) + } + serverContent[KEY_TURN_COMPLETE]?.jsonPrimitive?.booleanOrNull == true -> { + listOf(LiveLlmEvent.TurnComplete) + } + KEY_INPUT_TRANSCRIPTION in serverContent -> { + val text = + serverContent[KEY_INPUT_TRANSCRIPTION] + ?.jsonObject + ?.get(KEY_TEXT) + ?.jsonPrimitive + ?.contentOrNull + if (!text.isNullOrEmpty()) { + listOf(LiveLlmEvent.InputTranscription(text)) + } else { + emptyList() + } + } + KEY_OUTPUT_TRANSCRIPTION in serverContent -> { + val text = + serverContent[KEY_OUTPUT_TRANSCRIPTION] + ?.jsonObject + ?.get(KEY_TEXT) + ?.jsonPrimitive + ?.contentOrNull + if (!text.isNullOrEmpty()) { + listOf(LiveLlmEvent.OutputTranscription(text)) + } else { + emptyList() + } + } + KEY_MODEL_TURN in serverContent -> { + val modelTurn = serverContent[KEY_MODEL_TURN]?.jsonObject + if (modelTurn != null) { + parseModelTurn(modelTurn) + } else { + listOf( + LiveLlmEvent.Error( + "Invalid '$KEY_MODEL_TURN' payload in serverContent: $serverContent", + ), + ) + } + } + else -> emptyList() + } + } + + private fun parseModelTurn(modelTurn: JsonObject): List { + val parts = + modelTurn[KEY_PARTS]?.jsonArray + ?: return listOf( + LiveLlmEvent.Error("modelTurn is missing '$KEY_PARTS' array: $modelTurn"), + ) + val events = mutableListOf() + for (part in parts) { + val partObj = part.jsonObject + val inlineData = partObj[KEY_INLINE_DATA]?.jsonObject ?: continue + val mimeType = inlineData[KEY_MIME_TYPE]?.jsonPrimitive?.contentOrNull + if (mimeType == null || !mimeType.startsWith(MIME_TYPE_AUDIO_PCM_PREFIX)) { + events.add( + LiveLlmEvent.Error( + "Unsupported inlineData mimeType in modelTurn: $mimeType", + ), + ) + continue + } + val base64Data = inlineData[KEY_DATA]?.jsonPrimitive?.contentOrNull + if (base64Data.isNullOrEmpty()) { + events.add( + LiveLlmEvent.Error("inlineData is missing audio '$KEY_DATA' in modelTurn"), + ) + continue + } + try { + val pcmBytes = Base64.getMimeDecoder().decode(base64Data) + val sampleRateHz = extractSampleRateHz(mimeType) + events.add( + LiveLlmEvent.AudioChunk( + pcmBytes = pcmBytes, + sampleRateHz = sampleRateHz, + ), + ) + } catch (e: IllegalArgumentException) { + Log.w(TAG, "Failed to decode base64 PCM audio chunk", e) + events.add( + LiveLlmEvent.Error( + message = "Failed to decode base64 PCM audio chunk", + cause = e, + ), + ) + } + } + return events + } + + private fun extractSampleRateHz(mimeType: String): Int { + val rateParam = + mimeType + .split(';') + .map { it.trim() } + .firstOrNull { it.startsWith(MIME_TYPE_RATE_PARAM, ignoreCase = true) } + val parsedRate = rateParam?.substringAfter('=')?.toIntOrNull() + return parsedRate ?: DEFAULT_OUTPUT_SAMPLE_RATE_HZ + } + + private fun parseToolCall( + toolCallObj: JsonObject, + tools: List, + ): List { + val functionCalls = + toolCallObj[KEY_FUNCTION_CALLS]?.jsonArray + ?: return listOf( + LiveLlmEvent.Error( + "toolCall is missing '$KEY_FUNCTION_CALLS' array: $toolCallObj", + ), + ) + if (functionCalls.isEmpty()) { + return listOf( + LiveLlmEvent.Error("toolCall '$KEY_FUNCTION_CALLS' array is empty"), + ) + } + + val events = mutableListOf() + val validCalls = mutableListOf() + + for (callElement in functionCalls) { + val callObj = callElement.jsonObject + val callId = callObj[KEY_ID]?.jsonPrimitive?.contentOrNull + if (callId.isNullOrBlank()) { + events.add( + LiveLlmEvent.Error("Function call missing '$KEY_ID' in response: $callObj"), + ) + continue + } + val name = callObj[KEY_NAME]?.jsonPrimitive?.contentOrNull + if (name.isNullOrBlank()) { + events.add( + LiveLlmEvent.Error("Called function without a '$KEY_NAME': $callObj"), + ) + continue + } + val matchingTool = toolConverter.findToolByName(name, tools) + if (matchingTool == null) { + events.add( + LiveLlmEvent.Error("Called unknown function: $name"), + ) + continue + } + val argumentsMap = callObj[KEY_ARGS]?.jsonObject.toArgumentsMap() + validCalls.add( + LlmResponsePart.ToolCall( + packageName = matchingTool.packageName, + functionId = matchingTool.id, + arguments = argumentsMap, + callId = callId, + ), + ) + } + + if (validCalls.isNotEmpty()) { + events.add(LiveLlmEvent.ToolCallRequest(validCalls)) + } + return events + } + + private fun buildSetupMessage( + tools: List, + voiceName: String, + ): JsonObject { + val functionDeclarations = + toolConverter + .selectDistinctTools(tools) + .mapNotNull { tool -> + try { + toolConverter.convert(tool) + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null + } + } + + return buildJsonObject { + put( + KEY_SETUP, + buildJsonObject { + put(KEY_MODEL, LIVE_MODEL_NAME) + put( + KEY_GENERATION_CONFIG, + buildJsonObject { + put( + KEY_RESPONSE_MODALITIES, + buildJsonArray { add(JsonPrimitive(VALUE_MODALITY_AUDIO)) }, + ) + put( + KEY_SPEECH_CONFIG, + buildJsonObject { + put( + KEY_VOICE_CONFIG, + buildJsonObject { + put( + KEY_PREBUILT_VOICE_CONFIG, + buildJsonObject { + put(KEY_VOICE_NAME, voiceName) + }, + ) + }, + ) + }, + ) + }, + ) + put( + KEY_SYSTEM_INSTRUCTION, + buildJsonObject { + put( + KEY_PARTS, + buildJsonArray { + add( + buildJsonObject { + put(KEY_TEXT, getSystemInstruction()) + }, + ) + }, + ) + }, + ) + if (functionDeclarations.isNotEmpty()) { + put( + KEY_TOOLS, + buildJsonArray { + add( + buildJsonObject { + put( + KEY_FUNCTION_DECLARATIONS, + JsonArray(functionDeclarations), + ) + }, + ) + }, + ) + } + put(KEY_INPUT_AUDIO_TRANSCRIPTION, buildJsonObject {}) + put(KEY_OUTPUT_AUDIO_TRANSCRIPTION, buildJsonObject {}) + }, + ) + } + } + + private fun buildRealtimeAudioMessage( + pcmBytes: ByteArray, + sampleRateHz: Int, + ): JsonObject { + val base64Audio = Base64.getEncoder().encodeToString(pcmBytes) + return buildJsonObject { + put( + KEY_REALTIME_INPUT, + buildJsonObject { + put( + KEY_AUDIO, + buildJsonObject { + put(KEY_MIME_TYPE, "$MIME_TYPE_AUDIO_PCM_PREFIX;$MIME_TYPE_RATE_PARAM$sampleRateHz") + put(KEY_DATA, base64Audio) + }, + ) + }, + ) + } + } + + private fun buildToolResponseMessage( + toolOutputs: List, + tools: List, + ): JsonObject { + return buildJsonObject { + put( + KEY_TOOL_RESPONSE, + buildJsonObject { + put( + KEY_FUNCTION_RESPONSES, + buildJsonArray { + for (output in toolOutputs) { + val mappedName = + toolConverter.resolveToolName(output.functionId, tools) + val parsedResult = + try { + Json.parseToJsonElement(output.result) + } catch (_: Exception) { + JsonPrimitive(output.result) + } + add( + buildJsonObject { + if (output.callId.isNotEmpty()) { + put(KEY_ID, output.callId) + } + put(KEY_NAME, mappedName) + put( + KEY_RESPONSE, + buildJsonObject { + put(KEY_RESULT, parsedResult) + }, + ) + }, + ) + } + }, + ) + }, + ) + } + } + + private fun getSystemInstruction(): String { + val currentDate = LocalDate.now().toString() + return """ + You are an AI voice assistant running on Android using Gemini Live. Today's date is $currentDate. + Default to English for understanding and replying unless the user clearly speaks in or asks to switch to another language. + Always reply to the user using concise, friendly, natural spoken language. + You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device. + Execute user-requested actions directly without claiming an on-screen confirmation card or button is shown; if you ever need confirmation before a sensitive action, ask the user verbally. + When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence. + """.trimIndent() + } + + private sealed class FrameReadResult { + data class JsonText(val text: String) : FrameReadResult() + + object NormalClose : FrameReadResult() + + data class ErrorClose( + val message: String, + val cause: Throwable? = null, + ) : FrameReadResult() + } + + companion object { + private const val TAG = "GeminiLiveProvider" + const val LIVE_MODEL_NAME = "models/gemini-3.8-live" + const val DEFAULT_INPUT_SAMPLE_RATE_HZ = 16_000 + const val DEFAULT_OUTPUT_SAMPLE_RATE_HZ = 24_000 + private const val GEMINI_LIVE_WS_URL = + "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" + private const val CLOSE_REASON_CLIENT_ENDED = "Session ended by client" + + private const val KEY_SETUP = "setup" + private const val KEY_SETUP_COMPLETE = "setupComplete" + private const val KEY_MODEL = "model" + private const val KEY_GENERATION_CONFIG = "generationConfig" + private const val KEY_RESPONSE_MODALITIES = "responseModalities" + private const val VALUE_MODALITY_AUDIO = "AUDIO" + private const val KEY_SPEECH_CONFIG = "speechConfig" + private const val KEY_VOICE_CONFIG = "voiceConfig" + private const val KEY_PREBUILT_VOICE_CONFIG = "prebuiltVoiceConfig" + private const val KEY_VOICE_NAME = "voiceName" + private const val KEY_SYSTEM_INSTRUCTION = "systemInstruction" + private const val KEY_PARTS = "parts" + private const val KEY_TEXT = "text" + private const val KEY_TOOLS = "tools" + private const val KEY_FUNCTION_DECLARATIONS = "functionDeclarations" + private const val KEY_INPUT_AUDIO_TRANSCRIPTION = "inputAudioTranscription" + private const val KEY_OUTPUT_AUDIO_TRANSCRIPTION = "outputAudioTranscription" + + private const val KEY_SERVER_CONTENT = "serverContent" + private const val KEY_INTERRUPTED = "interrupted" + private const val KEY_INPUT_TRANSCRIPTION = "inputTranscription" + private const val KEY_OUTPUT_TRANSCRIPTION = "outputTranscription" + private const val KEY_MODEL_TURN = "modelTurn" + private const val KEY_INLINE_DATA = "inlineData" + private const val KEY_MIME_TYPE = "mimeType" + private const val KEY_DATA = "data" + private const val KEY_TURN_COMPLETE = "turnComplete" + + private const val KEY_TOOL_CALL = "toolCall" + private const val KEY_FUNCTION_CALLS = "functionCalls" + private const val KEY_ID = "id" + private const val KEY_NAME = "name" + private const val KEY_ARGS = "args" + + private const val KEY_REALTIME_INPUT = "realtimeInput" + private const val KEY_AUDIO = "audio" + + private const val KEY_TOOL_RESPONSE = "toolResponse" + private const val KEY_FUNCTION_RESPONSES = "functionResponses" + private const val KEY_RESPONSE = "response" + private const val KEY_RESULT = "result" + + private const val MIME_TYPE_AUDIO_PCM_PREFIX = "audio/pcm" + private const val MIME_TYPE_RATE_PARAM = "rate=" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveToolConverter.kt new file mode 100644 index 0000000..3c8ef63 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveToolConverter.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.domain.appfunction.ToolConverter +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Implementation of [ToolConverter] for Gemini Live (`BidiGenerateContent`) `FunctionDeclaration` + * schemas, normalizing OpenAPI schema fields produced by [GeminiToolConverter]. + */ +@Singleton +class GeminiLiveToolConverter + @Inject + constructor( + private val geminiToolConverter: GeminiToolConverter, + ) : ToolConverter { + override fun convert(tool: AppFunctionMetadata): JsonObject { + val baseSchema = geminiToolConverter.convert(tool) + return normalizeForLiveApi(baseSchema) + } + + override fun getToolName(tool: AppFunctionMetadata): String { + return geminiToolConverter.getToolName(tool) + } + + private fun normalizeForLiveApi(obj: JsonObject): JsonObject { + val typeValue = (obj[KEY_TYPE] as? JsonPrimitive)?.content + return buildJsonObject { + for ((key, value) in obj) { + when { + key == KEY_FORMAT && + (value as? JsonPrimitive)?.content == VALUE_FILE_REFERENCE -> { + // Omit Interactions-API-specific file_reference format in BidiGenerateContent + } + key == KEY_ENUMS -> { + if (typeValue == VALUE_STRING) { + put(KEY_ENUM_LIVE, value) + } + } + value is JsonObject -> { + put(key, normalizeForLiveApi(value)) + } + else -> { + put(key, value) + } + } + } + } + } + + companion object { + private const val KEY_TYPE = "type" + private const val KEY_FORMAT = "format" + private const val VALUE_FILE_REFERENCE = "file_reference" + private const val VALUE_STRING = "string" + private const val KEY_ENUMS = "enums" + private const val KEY_ENUM_LIVE = "enum" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt new file mode 100644 index 0000000..b9ea6c1 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +/** + * Prebuilt studio voices supported by Gemini Live (`models/gemini-3.8-live`) via + * `speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName`. + */ +enum class GeminiLiveVoice( + /** Voice identifier sent in `prebuiltVoiceConfig.voiceName` and shown in Settings. */ + val apiName: String, + /** Short vocal style descriptor for the voice. */ + val style: String, +) { + ZEPHYR("Zephyr", "Bright"), + PUCK("Puck", "Upbeat"), + CHARON("Charon", "Informative"), + KORE("Kore", "Firm"), + FENRIR("Fenrir", "Excitable"), + LEDA("Leda", "Youthful"), + ORUS("Orus", "Firm"), + AOEDE("Aoede", "Breezy"), + CALLIRRHOE("Callirrhoe", "Easy-going"), + AUTONOE("Autonoe", "Bright"), + ENCELADUS("Enceladus", "Breathy"), + IAPETUS("Iapetus", "Clear"), + UMBRIEL("Umbriel", "Easy-going"), + ALGIEBA("Algieba", "Smooth"), + DESPINA("Despina", "Smooth"), + ERINOME("Erinome", "Clear"), + ALGENIB("Algenib", "Gravelly"), + RASALGETHI("Rasalgethi", "Informative"), + LAOMEDEIA("Laomedeia", "Upbeat"), + ACHERNAR("Achernar", "Soft"), + ALNILAM("Alnilam", "Firm"), + SCHEDAR("Schedar", "Even"), + GACRUX("Gacrux", "Mature"), + PULCHERRIMA("Pulcherrima", "Forward"), + ACHIRD("Achird", "Friendly"), + ZUBENELGENUBI("Zubenelgenubi", "Casual"), + VINDEMIATRIX("Vindemiatrix", "Gentle"), + SADACHBIA("Sadachbia", "Lively"), + SADALTAGER("Sadaltager", "Knowledgeable"), + SULAFAT("Sulafat", "Warm"), + ; + + companion object { + val DEFAULT: GeminiLiveVoice = ZEPHYR + + /** Resolves a persisted preference string back to a [GeminiLiveVoice], falling back to [DEFAULT]. */ + fun fromStoredValue(value: String?): GeminiLiveVoice { + if (value == null) return DEFAULT + return entries.find { + it.name.equals(value, ignoreCase = true) || + it.apiName.equals(value, ignoreCase = true) + } ?: DEFAULT + } + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index a6da325..b7eaf5f 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -21,6 +21,10 @@ import com.example.appfunctions.agent.domain.LlmInput import com.example.appfunctions.agent.domain.LlmProvider import com.example.appfunctions.agent.domain.LlmResponse import com.example.appfunctions.agent.domain.LlmResponsePart +import com.example.appfunctions.agent.domain.appfunction.findToolByName +import com.example.appfunctions.agent.domain.appfunction.resolveToolName +import com.example.appfunctions.agent.domain.appfunction.selectDistinctTools +import com.example.appfunctions.agent.domain.toArgumentsMap import io.ktor.client.HttpClient import io.ktor.client.request.get import io.ktor.client.request.header @@ -32,9 +36,8 @@ import io.ktor.http.ContentType import io.ktor.http.contentType import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonArray @@ -119,9 +122,8 @@ class GeminiProviderImpl serviceTier: ServiceTier, ): LlmResponse { val convertedTools = - tools - .sortedByDescending { it.id.startsWith(it.packageName) } - .distinctBy { toolConverter.getToolName(it) } + toolConverter + .selectDistinctTools(tools) .mapNotNull { tool -> try { buildJsonObject { @@ -148,15 +150,10 @@ class GeminiProviderImpl when (input) { is LlmInput.ToolResponse -> { val inputElement = - kotlinx.serialization.json.buildJsonArray { + buildJsonArray { input.outputs.forEach { output -> - val matchingTool = tools.find { it.id == output.functionId } val mappedName = - if (matchingTool != null) { - toolConverter.getToolName(matchingTool) - } else { - output.functionId - } + toolConverter.resolveToolName(output.functionId, tools) add( buildJsonObject { @@ -178,7 +175,7 @@ class GeminiProviderImpl } if (convertedTools.isNotEmpty()) { - put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) + put(KEY_TOOLS, JsonArray(convertedTools)) } if (previousInteractionId != null) { @@ -259,19 +256,13 @@ class GeminiProviderImpl stepObj[KEY_NAME]?.jsonPrimitive?.content ?: return LlmResponse.Error("Called function without a name") val matchingTool = - tools.find { tool -> toolConverter.getToolName(tool) == name } + toolConverter.findToolByName(name, tools) ?: return LlmResponse.Error("Called unknown function: $name") val packageName = matchingTool.packageName val functionId = matchingTool.id - val args = stepObj[KEY_ARGUMENTS]?.jsonObject - val argumentsMap = mutableMapOf() - if (args != null) { - for ((key, value) in args) { - argumentsMap[key] = value.toPrimitive() - } - } + val argumentsMap = stepObj[KEY_ARGUMENTS]?.jsonObject.toArgumentsMap() val callId = stepObj["id"]?.jsonPrimitive?.content ?: return LlmResponse.Error( @@ -297,19 +288,6 @@ class GeminiProviderImpl ) } - private fun JsonElement.toPrimitive(): Any? { - return when (this) { - is JsonPrimitive -> { - if (this.isString) return this.content - return this.content.toBooleanStrictOrNull() - ?: this.content.toLongOrNull() - ?: this.content.toDoubleOrNull() - } - is JsonObject -> this.mapValues { it.value.toPrimitive() } - is JsonArray -> this.map { it.toPrimitive() } - } - } - private fun getSystemInstruction(): String { val currentDate = LocalDate.now().toString() return """ diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt index 0475b8c..8d543c3 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt @@ -15,21 +15,29 @@ */ package com.example.appfunctions.agent.data +import com.example.appfunctions.agent.domain.LiveLlmProvider import com.example.appfunctions.agent.domain.LlmProvider import com.example.appfunctions.agent.domain.LlmProviderFactory import javax.inject.Inject import javax.inject.Singleton -/** Implementation of LlmProviderFactory. */ +/** Implementation of [LlmProviderFactory]. */ @Singleton class LlmProviderFactoryImpl @Inject constructor( private val geminiProvider: GeminiProviderImpl, + private val geminiLiveProvider: GeminiLiveProviderImpl, ) : LlmProviderFactory { override fun getProvider(name: LlmProviderName): LlmProvider { return when (name) { LlmProviderName.GEMINI -> geminiProvider } } + + override fun getLiveProvider(name: LlmProviderName): LiveLlmProvider { + return when (name) { + LlmProviderName.GEMINI -> geminiLiveProvider + } + } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt index 0feefb9..92a2ca1 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt @@ -37,6 +37,12 @@ interface SettingsRepository { /** Sets the service tier requested for LLM API calls. */ suspend fun setServiceTier(tier: ServiceTier) + /** Flow of the prebuilt voice selected for Gemini Live sessions. */ + val geminiLiveVoice: Flow + + /** Sets the prebuilt voice selected for Gemini Live sessions. */ + suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) + /** Flow of pinned app package names. */ val pinnedApps: Flow> diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index f92fae7..d384df2 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -25,11 +25,13 @@ import com.example.appfunctions.agent.data.DataStoreSettingsRepository import com.example.appfunctions.agent.data.FileDownloadHelper import com.example.appfunctions.agent.data.FileDownloadHelperImpl import com.example.appfunctions.agent.data.InMemoryPendingIntentRepository +import com.example.appfunctions.agent.data.LlmProviderFactoryImpl import com.example.appfunctions.agent.data.PendingIntentRepository import com.example.appfunctions.agent.data.RoomChatRepository import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.AppDatabase import com.example.appfunctions.agent.data.db.dao.ChatDao +import com.example.appfunctions.agent.domain.LlmProviderFactory import dagger.Binds import dagger.Module import dagger.Provides @@ -40,6 +42,7 @@ import io.ktor.client.HttpClient import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.websocket.WebSockets import io.ktor.serialization.kotlinx.json.json import kotlinx.serialization.json.Json import javax.inject.Singleton @@ -64,9 +67,7 @@ abstract class DataModule { @Binds @Singleton - abstract fun bindLlmProviderFactory( - impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl, - ): com.example.appfunctions.agent.domain.LlmProviderFactory + abstract fun bindLlmProviderFactory(impl: LlmProviderFactoryImpl): LlmProviderFactory @Binds @Singleton @@ -114,8 +115,12 @@ abstract class DataModule { }, ) } - install(HttpTimeout) { socketTimeoutMillis = 30000 } + install(HttpTimeout) { socketTimeoutMillis = SOCKET_TIMEOUT_MS } + install(WebSockets) { pingIntervalMillis = WEBSOCKET_PING_INTERVAL_MS } } } + + private const val SOCKET_TIMEOUT_MS = 30_000L + private const val WEBSOCKET_PING_INTERVAL_MS = 20_000L } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/JsonExtensions.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/JsonExtensions.kt new file mode 100644 index 0000000..c49c1f4 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/JsonExtensions.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive + +/** Converts a [JsonElement] recursively into standard Kotlin primitive, [Map], or [List] values. */ +internal fun JsonElement.toPrimitive(): Any? { + return when (this) { + JsonNull -> null + is JsonPrimitive -> { + if (this.isString) return this.content + return this.content.toBooleanStrictOrNull() + ?: this.content.toLongOrNull() + ?: this.content.toDoubleOrNull() + } + is JsonObject -> this.mapValues { it.value.toPrimitive() } + is JsonArray -> this.map { it.toPrimitive() } + } +} + +/** Converts a nullable JSON arguments object into a Kotlin arguments map. */ +internal fun JsonObject?.toArgumentsMap(): Map { + if (this == null) return emptyMap() + return mapValues { (_, value) -> value.toPrimitive() } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt new file mode 100644 index 0000000..8b00d47 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import androidx.appfunctions.metadata.AppFunctionMetadata +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.withContext + +/** Interface for LLM providers supporting bidirectional realtime audio and tool sessions. */ +interface LiveLlmProvider { + /** + * Connects to the live LLM backend, completes session initialization with [tools] and optional + * provider-specific [voiceName], and returns the active [LiveLlmSession]. The caller is + * responsible for closing the returned session via [LiveLlmSession.close] or using + * [withSession]. + */ + suspend fun connect( + apiKey: String, + tools: List, + voiceName: String? = null, + ): LiveLlmSession +} + +/** + * Connects to the [LiveLlmProvider], invokes [block] with the active [LiveLlmSession], and ensures + * [LiveLlmSession.close] is called when [block] completes or is cancelled. + */ +suspend inline fun LiveLlmProvider.withSession( + apiKey: String, + tools: List, + voiceName: String? = null, + block: (LiveLlmSession) -> Unit, +) { + val session = connect(apiKey = apiKey, tools = tools, voiceName = voiceName) + try { + block(session) + } finally { + withContext(NonCancellable) { + session.close() + } + } +} + +/** Represents an active bidirectional voice and tool session with an LLM provider. */ +interface LiveLlmSession { + /** Sequential stream of server-originated events for this live session. */ + val events: Flow + + /** + * Sends a chunk of raw 16-bit mono PCM user microphone audio at [sampleRateHz] to the model. + */ + suspend fun sendAudio( + pcmBytes: ByteArray, + sampleRateHz: Int, + ) + + /** + * Sends tool execution [outputs] back to the model in response to a + * [LiveLlmEvent.ToolCallRequest]. + */ + suspend fun sendToolResponses(outputs: List) + + /** Closes the live session and releases the underlying transport connection. */ + suspend fun close() +} + +/** Provider-agnostic events emitted by a [LiveLlmSession]. */ +sealed class LiveLlmEvent { + /** Indicates the user interrupted the model's current audio response. */ + object Interrupted : LiveLlmEvent() + + /** Incremental speech-to-text transcription of the user's audio input. */ + data class InputTranscription(val text: String) : LiveLlmEvent() + + /** Incremental transcription of the model's spoken response. */ + data class OutputTranscription(val text: String) : LiveLlmEvent() + + /** Raw 16-bit mono PCM audio chunk generated by the model for playback at [sampleRateHz]. */ + data class AudioChunk( + val pcmBytes: ByteArray, + val sampleRateHz: Int, + ) : LiveLlmEvent() { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is AudioChunk) return false + return pcmBytes.contentEquals(other.pcmBytes) && sampleRateHz == other.sampleRateHz + } + + override fun hashCode(): Int { + var result = pcmBytes.contentHashCode() + result = 31 * result + sampleRateHz + return result + } + } + + /** Indicates the model finished its current conversational turn. */ + object TurnComplete : LiveLlmEvent() + + /** Requests execution of one or more [calls] before continuing the live conversation. */ + data class ToolCallRequest(val calls: List) : LiveLlmEvent() + + /** Indicates an error occurred during the live session. */ + data class Error( + val message: String, + val cause: Throwable? = null, + ) : LiveLlmEvent() +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt index 054c287..c9ae94f 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt @@ -17,8 +17,11 @@ package com.example.appfunctions.agent.domain import com.example.appfunctions.agent.data.LlmProviderName -/** Factory interface to get the correct LlmProvider. */ +/** Factory interface to get the correct [LlmProvider] or [LiveLlmProvider]. */ interface LlmProviderFactory { - /** Returns the LlmProvider instance for the given name. */ + /** Returns the [LlmProvider] instance for the given [name]. */ fun getProvider(name: LlmProviderName): LlmProvider + + /** Returns the [LiveLlmProvider] instance for the given [name]. */ + fun getLiveProvider(name: LlmProviderName): LiveLlmProvider } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ToolConverter.kt index 022524f..338be8c 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ToolConverter.kt @@ -25,3 +25,30 @@ interface ToolConverter { /** Returns the name of the tool as it should be represented in the LLM schema. */ fun getToolName(tool: AppFunctionMetadata): String } + +/** + * Deduplicates [tools] by their converted tool name, prioritizing functions whose ID starts with + * their package name. + */ +fun ToolConverter<*>.selectDistinctTools(tools: List): List = + tools + .sortedByDescending { it.id.startsWith(it.packageName) } + .distinctBy { getToolName(it) } + +/** Finds the [AppFunctionMetadata] in [tools] matching a converted tool [name]. */ +fun ToolConverter<*>.findToolByName( + name: String, + tools: List, +): AppFunctionMetadata? = tools.find { getToolName(it) == name } + +/** + * Resolves the converted tool name for [functionId] from [tools], falling back to [functionId] if + * no matching tool metadata is present. + */ +fun ToolConverter<*>.resolveToolName( + functionId: String, + tools: List, +): String { + val matchingTool = tools.find { it.id == functionId } ?: return functionId + return getToolName(matchingTool) +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt new file mode 100644 index 0000000..8770e75 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt @@ -0,0 +1,670 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import com.example.appfunctions.agent.domain.LiveLlmEvent +import com.example.appfunctions.agent.domain.ToolOutput +import com.example.appfunctions.agent.domain.withSession +import io.ktor.websocket.CloseReason +import io.ktor.websocket.Frame +import io.ktor.websocket.WebSocketSession +import io.ktor.websocket.readReason +import io.ktor.websocket.readText +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkStatic +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Before +import org.junit.Test +import java.io.IOException +import java.util.Base64 +import kotlin.coroutines.CoroutineContext + +class GeminiLiveProviderImplTest { + private lateinit var toolConverter: GeminiLiveToolConverter + + @Before + fun setUp() { + mockkStatic(Log::class) + every { Log.d(any(), any()) } returns 0 + every { Log.w(any(), any()) } returns 0 + every { Log.w(any(), any(), any()) } returns 0 + every { Log.e(any(), any(), any()) } returns 0 + + toolConverter = GeminiLiveToolConverter(GeminiToolConverter()) + } + + @Test + fun `connect sends setup message with model, audio modality, voice, transcriptions, and tools`() = + runTest { + val tool = createSampleNoteTool() + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.close() + + val sentFrames = mutableListOf() + var capturedWsUrl: String? = null + val provider = + createProvider { wsUrl -> + capturedWsUrl = wsUrl + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ) + } + + provider.withSession( + apiKey = "secret-key", + tools = listOf(tool), + voiceName = GeminiLiveVoice.AOEDE.apiName, + ) {} + + assertTrue(capturedWsUrl!!.endsWith("?key=secret-key")) + val setup = Json.parseToJsonElement(sentFrames.first()).jsonObject["setup"]!!.jsonObject + assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content) + + val modalities = + setup["generationConfig"]!! + .jsonObject["responseModalities"]!! + .jsonArray + .map { it.jsonPrimitive.content } + assertEquals(listOf("AUDIO"), modalities) + + val voiceName = + setup["generationConfig"]!! + .jsonObject["speechConfig"]!! + .jsonObject["voiceConfig"]!! + .jsonObject["prebuiltVoiceConfig"]!! + .jsonObject["voiceName"]!! + .jsonPrimitive + .content + assertEquals("Aoede", voiceName) + + assertTrue(setup.containsKey("inputAudioTranscription")) + assertTrue(setup.containsKey("outputAudioTranscription")) + val systemInstructionText = + setup["systemInstruction"]!! + .jsonObject["parts"]!! + .jsonArray + .first() + .jsonObject["text"]!! + .jsonPrimitive + .content + assertTrue( + systemInstructionText.contains( + "Default to English for understanding and replying unless the user clearly " + + "speaks in or asks to switch to another language.", + ), + ) + assertTrue( + systemInstructionText.contains( + "Execute user-requested actions directly without claiming an on-screen " + + "confirmation card or button is shown", + ), + ) + assertFalse(systemInstructionText.contains("confirmation_required")) + + val declarations = + setup["tools"]!! + .jsonArray + .first() + .jsonObject["functionDeclarations"]!! + .jsonArray + assertEquals(1, declarations.size) + assertEquals( + "com_example_notes_createNote", + declarations.first().jsonObject["name"]!!.jsonPrimitive.content, + ) + } + + @Test + fun `connect throws IllegalStateException when WebSocket closes before setupComplete`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.close() + + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + try { + provider.connect(apiKey = "key-123", tools = emptyList()) + fail("Expected IllegalStateException") + } catch (e: IllegalStateException) { + assertTrue(e.message!!.contains("setupComplete")) + } + } + + @Test + fun `connect throws IllegalStateException when first frame is missing setupComplete`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"error":{"message":"Invalid argument"}}""")) + incomingFrames.close() + + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + try { + provider.connect(apiKey = "key-123", tools = emptyList()) + fail("Expected IllegalStateException") + } catch (e: IllegalStateException) { + assertTrue(e.message!!.startsWith("Unexpected setup response:")) + } + } + + @Test + fun `connect rethrows CancellationException during handshake without wrapping as IllegalStateException`() = + runTest { + val wsSession = mockk(relaxed = true) + every { wsSession.coroutineContext } returns coroutineContext + every { wsSession.incoming } returns + mockk { + coEvery { receive() } throws CancellationException("Handshake cancelled") + } + + val provider = createProvider { wsSession } + + try { + provider.connect(apiKey = "key-123", tools = emptyList()) + fail("Expected CancellationException") + } catch (e: CancellationException) { + assertEquals("Handshake cancelled", e.message) + } + } + + @Test + fun `withSession closes WebSocket with NORMAL CloseReason when block completes normally or throws`() = + runTest { + val incomingFramesNormal = Channel(Channel.UNLIMITED) + incomingFramesNormal.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFramesNormal.close() + val normalCloseReasons = mutableListOf() + + val providerNormal = + createProvider { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFramesNormal, + onSendClose = { normalCloseReasons.add(it) }, + ) + } + + providerNormal.withSession(apiKey = "key-123", tools = emptyList()) {} + + assertEquals( + listOf(CloseReason(CloseReason.Codes.NORMAL, "Session ended by client")), + normalCloseReasons, + ) + + val incomingFramesFailing = Channel(Channel.UNLIMITED) + incomingFramesFailing.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFramesFailing.close() + val failingCloseReasons = mutableListOf() + + val providerFailing = + createProvider { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFramesFailing, + onSendClose = { failingCloseReasons.add(it) }, + ) + } + + try { + providerFailing.withSession(apiKey = "key-123", tools = emptyList()) { + throw IllegalStateException("Caller failed") + } + fail("Expected IllegalStateException") + } catch (e: IllegalStateException) { + assertEquals("Caller failed", e.message) + } + + assertEquals( + listOf(CloseReason(CloseReason.Codes.NORMAL, "Session ended by client")), + failingCloseReasons, + ) + } + + @Test + fun `connect and events decode Frame_Binary server frames`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend( + Frame.Binary(fin = true, data = """{"setupComplete":{}}""".encodeToByteArray()), + ) + incomingFrames.trySend( + Frame.Binary( + fin = true, + data = """{"serverContent":{"turnComplete":true}}""".encodeToByteArray(), + ), + ) + incomingFrames.close() + + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + val receivedEvents = mutableListOf() + provider.withSession(apiKey = "key-123", tools = emptyList()) { session -> + receivedEvents.addAll(session.events.toList()) + } + + assertEquals(listOf(LiveLlmEvent.TurnComplete), receivedEvents) + } + + @Test + fun `sendAudio encodes PCM bytes with specified sampleRateHz in realtimeInput frame`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.close() + + val sentFrames = mutableListOf() + val provider = + createProvider { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ) + } + + provider.withSession(apiKey = "key-123", tools = emptyList()) { session -> + session.sendAudio(pcmBytes = byteArrayOf(1, 2, 3), sampleRateHz = 16_000) + } + + assertEquals(2, sentFrames.size) + val realtimeAudio = + Json.parseToJsonElement(sentFrames[1]) + .jsonObject["realtimeInput"]!! + .jsonObject["audio"]!! + .jsonObject + assertEquals("audio/pcm;rate=16000", realtimeAudio["mimeType"]!!.jsonPrimitive.content) + assertArrayEquals( + byteArrayOf(1, 2, 3), + Base64.getDecoder().decode(realtimeAudio["data"]!!.jsonPrimitive.content), + ) + } + + @Test + fun `events emits Interrupted when serverContent has interrupted true`() = + runTest { + val events = + collectSingleServerFrame( + """{"serverContent":{"interrupted":true}}""", + ) + assertEquals(listOf(LiveLlmEvent.Interrupted), events) + } + + @Test + fun `events emits InputTranscription when serverContent has inputTranscription`() = + runTest { + val events = + collectSingleServerFrame( + """{"serverContent":{"inputTranscription":{"text":"Create a note"}}}""", + ) + assertEquals(listOf(LiveLlmEvent.InputTranscription("Create a note")), events) + } + + @Test + fun `events emits OutputTranscription when serverContent has outputTranscription`() = + runTest { + val events = + collectSingleServerFrame( + """{"serverContent":{"outputTranscription":{"text":"Creating note"}}}""", + ) + assertEquals(listOf(LiveLlmEvent.OutputTranscription("Creating note")), events) + } + + @Test + fun `events emits AudioChunk with sampleRateHz when serverContent has modelTurn inlineData`() = + runTest { + val expectedPcm = byteArrayOf(10, 20, 30, 40) + val encodedPcm = Base64.getEncoder().encodeToString(expectedPcm) + + val events = + collectSingleServerFrame( + """ + { + "serverContent": { + "modelTurn": { + "parts": [ + { + "inlineData": { + "mimeType": "audio/pcm;rate=24000", + "data": "$encodedPcm" + } + } + ] + } + } + } + """.trimIndent(), + ) + + assertEquals( + listOf(LiveLlmEvent.AudioChunk(pcmBytes = expectedPcm, sampleRateHz = 24_000)), + events, + ) + } + + @Test + fun `events emits TurnComplete when serverContent has turnComplete true`() = + runTest { + val events = + collectSingleServerFrame( + """{"serverContent":{"turnComplete":true}}""", + ) + assertEquals(listOf(LiveLlmEvent.TurnComplete), events) + } + + @Test + fun `events emits ToolCallRequest and sendToolResponses sends mapped functionResponses`() = + runTest { + val tool = createSampleNoteTool() + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.trySend( + Frame.Text( + """ + { + "toolCall": { + "functionCalls": [ + { + "id": "call-1", + "name": "com_example_notes_createNote", + "args": { + "title": "Shopping", + "meta": { "pinned": true, "priority": 2 } + } + } + ] + } + } + """.trimIndent(), + ), + ) + incomingFrames.close() + + val sentFrames = mutableListOf() + val provider = + createProvider { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ) + } + + val receivedEvents = mutableListOf() + provider.withSession(apiKey = "key-123", tools = listOf(tool)) { session -> + receivedEvents.addAll(session.events.toList()) + session.sendToolResponses( + listOf( + ToolOutput( + functionId = tool.id, + callId = "call-1", + result = """{"noteId":"n-1"}""", + ), + ), + ) + } + + assertEquals(1, receivedEvents.size) + val toolCallReq = receivedEvents.single() as LiveLlmEvent.ToolCallRequest + assertEquals(1, toolCallReq.calls.size) + assertEquals("com.example.notes", toolCallReq.calls[0].packageName) + assertEquals("com.example.notes.createNote", toolCallReq.calls[0].functionId) + assertEquals("call-1", toolCallReq.calls[0].callId) + assertEquals("Shopping", toolCallReq.calls[0].arguments["title"]) + @Suppress("UNCHECKED_CAST") + val meta = toolCallReq.calls[0].arguments["meta"] as Map + assertEquals(true, meta["pinned"]) + assertEquals(2L, meta["priority"]) + + val responses = + Json.parseToJsonElement(sentFrames.last()) + .jsonObject["toolResponse"]!! + .jsonObject["functionResponses"]!! + .jsonArray + assertEquals(1, responses.size) + assertEquals( + "com_example_notes_createNote", + responses[0].jsonObject["name"]!!.jsonPrimitive.content, + ) + assertEquals( + "n-1", + responses[0] + .jsonObject["response"]!! + .jsonObject["result"]!! + .jsonObject["noteId"]!! + .jsonPrimitive + .content, + ) + } + + @Test + fun `events emits Error when toolCall references unknown function or is missing id or name`() = + runTest { + val tool = createSampleNoteTool() + val events = + collectSingleServerFrame( + frameJson = + """ + { + "toolCall": { + "functionCalls": [ + { + "id": "call-unknown", + "name": "hallucinated_tool", + "args": {} + }, + { + "name": "com_example_notes_createNote", + "args": {} + }, + { + "id": "call-no-name", + "args": {} + } + ] + } + } + """.trimIndent(), + tools = listOf(tool), + ) + + assertEquals(3, events.size) + assertTrue((events[0] as LiveLlmEvent.Error).message.contains("hallucinated_tool")) + assertTrue((events[1] as LiveLlmEvent.Error).message.contains("id")) + assertTrue((events[2] as LiveLlmEvent.Error).message.contains("name")) + } + + @Test + fun `events recovers from malformed JSON frame mid-session and emits subsequent valid events`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.trySend(Frame.Text("not-valid-json")) + incomingFrames.trySend(Frame.Text("""{"serverContent":{"turnComplete":true}}""")) + incomingFrames.close() + + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + val receivedEvents = mutableListOf() + provider.withSession(apiKey = "key-123", tools = emptyList()) { session -> + receivedEvents.addAll(session.events.toList()) + } + + assertEquals(2, receivedEvents.size) + assertTrue(receivedEvents[0] is LiveLlmEvent.Error) + assertEquals(LiveLlmEvent.TurnComplete, receivedEvents[1]) + } + + @Test + fun `events emits Error when server closes WebSocket with non-normal CloseReason or transport exception`() = + runTest { + val incomingCloseFrames = Channel(Channel.UNLIMITED) + incomingCloseFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingCloseFrames.trySend( + Frame.Close(CloseReason(CloseReason.Codes.VIOLATED_POLICY, "Invalid API key")), + ) + incomingCloseFrames.close() + + val providerClose = + createProvider { + createMockWebSocketSession(coroutineContext, incomingCloseFrames) + } + + val closeEvents = mutableListOf() + providerClose.withSession(apiKey = "bad-key", tools = emptyList()) { session -> + closeEvents.addAll(session.events.toList()) + } + + assertEquals(1, closeEvents.size) + assertTrue((closeEvents.single() as LiveLlmEvent.Error).message.contains("Invalid API key")) + + var receiveCount = 0 + val wsSession = mockk(relaxed = true) + every { wsSession.coroutineContext } returns coroutineContext + every { wsSession.incoming } returns + mockk { + coEvery { receive() } coAnswers { + receiveCount++ + if (receiveCount == 1) { + Frame.Text("""{"setupComplete":{}}""") + } else { + throw IOException("Connection reset by peer") + } + } + } + + val providerError = createProvider { wsSession } + val transportEvents = mutableListOf() + providerError.withSession(apiKey = "key-123", tools = emptyList()) { session -> + transportEvents.addAll(session.events.toList()) + } + + assertEquals(1, transportEvents.size) + assertTrue( + (transportEvents.single() as LiveLlmEvent.Error) + .message + .contains("Connection reset by peer"), + ) + } + + private suspend fun CoroutineScope.collectSingleServerFrame( + frameJson: String, + tools: List = emptyList(), + ): List { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.trySend(Frame.Text(frameJson)) + incomingFrames.close() + + val provider = + createProvider { + createMockWebSocketSession(coroutineContext, incomingFrames) + } + + val receivedEvents = mutableListOf() + provider.withSession(apiKey = "key-123", tools = tools) { session -> + receivedEvents.addAll(session.events.toList()) + } + return receivedEvents + } + + private fun createProvider(connector: suspend (String) -> WebSocketSession): GeminiLiveProviderImpl = + GeminiLiveProviderImpl( + toolConverter = toolConverter, + webSocketConnector = connector, + ) + + private fun createMockWebSocketSession( + context: CoroutineContext, + incomingFrames: Channel, + onSendText: (String) -> Unit = {}, + onSendClose: (CloseReason?) -> Unit = {}, + ): WebSocketSession { + val outgoingFrames = Channel(Channel.UNLIMITED) + return mockk(relaxed = true).apply { + every { coroutineContext } returns context + every { incoming } returns incomingFrames + every { outgoing } returns outgoingFrames + coEvery { send(any()) } coAnswers { + when (val frame = firstArg()) { + is Frame.Text -> onSendText(frame.readText()) + is Frame.Close -> onSendClose(frame.readReason()) + else -> {} + } + } + } + } + + private fun createSampleNoteTool(): AppFunctionMetadata = + AppFunctionMetadata( + id = "com.example.notes.createNote", + packageName = "com.example.notes", + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "title", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "Note title", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Creates a note", + deprecation = null, + ) +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt new file mode 100644 index 0000000..6e9e952 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveToolConverterTest.kt @@ -0,0 +1,193 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import androidx.appfunctions.metadata.AppFunctionMetadata +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test + +class GeminiLiveToolConverterTest { + private lateinit var geminiToolConverter: GeminiToolConverter + private lateinit var converter: GeminiLiveToolConverter + private lateinit var tool: AppFunctionMetadata + + @Before + fun setUp() { + geminiToolConverter = mockk() + converter = GeminiLiveToolConverter(geminiToolConverter) + tool = mockk() + } + + @Test + fun convert_stringEnum_replacesEnumsKeyWithEnum() { + every { geminiToolConverter.convert(tool) } returns + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enums": ["A", "B"] + } + } + } + } + """.trimIndent(), + ).jsonObject + + val expected = + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["A", "B"] + } + } + } + } + """.trimIndent(), + ).jsonObject + + assertEquals(expected, converter.convert(tool)) + } + + @Test + fun convert_nonStringEnum_omitsEnumsKey() { + every { geminiToolConverter.convert(tool) } returns + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "priority": { + "type": "integer", + "enums": [1, 2, 3] + } + } + } + } + """.trimIndent(), + ).jsonObject + + val expected = + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "priority": { + "type": "integer" + } + } + } + } + """.trimIndent(), + ).jsonObject + + assertEquals(expected, converter.convert(tool)) + } + + @Test + fun convert_fileReferenceFormat_stripsFileReferenceRecursivelyAndKeepsOtherFormats() { + every { geminiToolConverter.convert(tool) } returns + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "wallpaperUri": { + "type": "string", + "format": "file_reference" + }, + "nested": { + "type": "object", + "properties": { + "attachmentUri": { + "type": "string", + "format": "file_reference" + }, + "createdAt": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + """.trimIndent(), + ).jsonObject + + val expected = + Json.parseToJsonElement( + """ + { + "name": "com_example_fn", + "parameters": { + "type": "object", + "properties": { + "wallpaperUri": { + "type": "string" + }, + "nested": { + "type": "object", + "properties": { + "attachmentUri": { + "type": "string" + }, + "createdAt": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + """.trimIndent(), + ).jsonObject + + assertEquals(expected, converter.convert(tool)) + } + + @Test + fun getToolName_delegatesToGeminiToolConverter() { + every { geminiToolConverter.getToolName(tool) } returns "delegated_tool_name" + + assertEquals("delegated_tool_name", converter.getToolName(tool)) + verify(exactly = 1) { geminiToolConverter.getToolName(tool) } + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt index ea05a1f..6d6e7ca 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt @@ -18,6 +18,8 @@ package com.example.appfunctions.agent.data import androidx.datastore.core.DataStore import androidx.datastore.preferences.core.PreferenceDataStoreFactory import androidx.datastore.preferences.core.Preferences +import androidx.datastore.preferences.core.edit +import androidx.datastore.preferences.core.stringPreferencesKey import com.example.appfunctions.agent.BuildConfig import kotlinx.coroutines.flow.first import kotlinx.coroutines.test.runTest @@ -77,4 +79,30 @@ class SettingsRepositoryTest { val value = repository.serviceTier.first() assertEquals(ServiceTier.PRIORITY, value) } + + @Test + fun geminiLiveVoice_defaultsToZephyr() = + runTest { + val value = repository.geminiLiveVoice.first() + assertEquals(GeminiLiveVoice.ZEPHYR, value) + } + + @Test + fun setGeminiLiveVoice_persistsValue() = + runTest { + repository.setGeminiLiveVoice(GeminiLiveVoice.AOEDE) + val value = repository.geminiLiveVoice.first() + assertEquals(GeminiLiveVoice.AOEDE, value) + } + + @Test + fun geminiLiveVoice_unrecognizedStoredValue_fallsBackToDefault() = + runTest { + dataStore.edit { preferences -> + preferences[stringPreferencesKey("gemini_live_voice")] = "UNRECOGNIZED_VOICE" + } + + val value = repository.geminiLiveVoice.first() + assertEquals(GeminiLiveVoice.DEFAULT, value) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ToolConverterTest.kt new file mode 100644 index 0000000..dbaf963 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ToolConverterTest.kt @@ -0,0 +1,90 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ToolConverterTest { + private val fakeConverter = + object : ToolConverter { + override fun convert(tool: AppFunctionMetadata): String = getToolName(tool) + + override fun getToolName(tool: AppFunctionMetadata): String = tool.id.substringAfterLast('.') + } + + @Test + fun selectDistinctTools_prioritizesIdStartingWithPackageNameAndDeduplicatesByToolName() { + val legacyTool = createTool(packageName = "com.example.notes", id = "legacy.createNote") + val canonicalTool = + createTool(packageName = "com.example.notes", id = "com.example.notes.createNote") + val otherTool = + createTool(packageName = "com.example.notes", id = "com.example.notes.deleteNote") + + val distinct = + fakeConverter.selectDistinctTools(listOf(legacyTool, canonicalTool, otherTool)) + + assertEquals(listOf(canonicalTool, otherTool), distinct) + } + + @Test + fun findToolByName_returnsMatchingToolOrNull() { + val createNote = + createTool(packageName = "com.example.notes", id = "com.example.notes.createNote") + val deleteNote = + createTool(packageName = "com.example.notes", id = "com.example.notes.deleteNote") + val tools = listOf(createNote, deleteNote) + + assertEquals(createNote, fakeConverter.findToolByName("createNote", tools)) + assertEquals(deleteNote, fakeConverter.findToolByName("deleteNote", tools)) + assertNull(fakeConverter.findToolByName("unknownTool", tools)) + } + + @Test + fun resolveToolName_returnsConvertedNameWhenMatchedOrFallsBackToFunctionId() { + val createNote = + createTool(packageName = "com.example.notes", id = "com.example.notes.createNote") + val tools = listOf(createNote) + + assertEquals("createNote", fakeConverter.resolveToolName(createNote.id, tools)) + assertEquals("unregistered.id", fakeConverter.resolveToolName("unregistered.id", tools)) + } + + private fun createTool( + packageName: String, + id: String, + ): AppFunctionMetadata = + AppFunctionMetadata( + id = id, + packageName = packageName, + isEnabled = true, + schema = null, + parameters = emptyList(), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test tool", + deprecation = null, + ) +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt index 7ea3b0b..e36b55b 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt @@ -15,6 +15,7 @@ */ package com.example.appfunctions.agent.ui.screens.agentdemo +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.ServiceTier import com.example.appfunctions.agent.data.SettingsRepository @@ -31,6 +32,9 @@ class FakeSettingsRepository : SettingsRepository { private val _serviceTier = MutableStateFlow(ServiceTier.STANDARD) override val serviceTier: Flow = _serviceTier + private val _geminiLiveVoice = MutableStateFlow(GeminiLiveVoice.DEFAULT) + override val geminiLiveVoice: Flow = _geminiLiveVoice + private val _pinnedApps = MutableStateFlow>(emptySet()) override val pinnedApps: Flow> = _pinnedApps @@ -46,6 +50,10 @@ class FakeSettingsRepository : SettingsRepository { _serviceTier.value = tier } + override suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) { + _geminiLiveVoice.value = voice + } + override suspend fun setAppPinned( packageName: String, pinned: Boolean,