diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e07ed8cfbf..a7e3e04427 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,12 +17,14 @@ permissions: contents: read concurrency: - # A new PR push supersedes that PR's older checks. Every main commit gets - # its own group and finishes: when main pushes cancelled each other, no - # main run completed for a day and a release shipped with no verdict - # (release.yml now waits for this workflow's `CI` check on its commit). - # Merge-queue runs stay protected on their own refs. - group: ci-${{ github.event_name == 'merge_group' && github.event.merge_group.head_ref || github.event_name == 'push' && github.sha || github.ref }} + # A new PR push supersedes that PR's older checks. Main keeps one run going + # and one waiting: each merge replaces the waiting run, so a burst of + # merges costs two full runs, not one per merge (fifteen queued behind each + # other in October 2026). The running one always finishes. A release runs + # CI on its own commit in a release-ci/v lane when main's run for + # it was replaced (scripts/release-ci.mjs). Merge-queue runs stay on their + # own refs. + group: ci-${{ github.event_name == 'merge_group' && github.event.merge_group.head_ref || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: @@ -91,10 +93,9 @@ jobs: shard: [1, 2, 3, 4] runs-on: ${{ matrix.os }} # vitest splits by file count, not by duration, so the shard holding the - # heavy e2e files ran 13 minutes on macOS/Ubuntu and 19 on Windows with - # three shards; four keeps the heaviest near 10 and 15. The cap catches a - # hang, not normal growth: raise it with the suite, or add a shard. - timeout-minutes: ${{ matrix.os == 'windows-latest' && 25 || 20 }} + # heavy e2e files exceeds 20 minutes on macOS and 25 on Windows with + # tests still progressing. The cap catches a hang, not normal suite growth. + timeout-minutes: ${{ matrix.os == 'ubuntu-latest' && 20 || 35 }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: pnpm/action-setup@ff378ebe6b225b0680b81c1ad4498ae0d1d3a5e3 # v6.0.10 @@ -352,16 +353,32 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # A repo without the secret (a fork, or before the token is created) + # skips the deploy with a warning instead of failing every main push. + - id: token + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + run: | + if [ -n "$CLOUDFLARE_API_TOKEN" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "::warning title=Composio broker not deployed::CLOUDFLARE_API_TOKEN is not set, so the broker Worker was not deployed." + fi + - if: steps.token.outputs.present == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: pnpm/action-setup@ff378ebe6b225b0680b81c1ad4498ae0d1d3a5e3 # v6.0.10 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + - if: steps.token.outputs.present == 'true' + uses: pnpm/action-setup@ff378ebe6b225b0680b81c1ad4498ae0d1d3a5e3 # v6.0.10 + - if: steps.token.outputs.present == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24 cache: pnpm - - run: pnpm install --frozen-lockfile + - if: steps.token.outputs.present == 'true' + run: pnpm install --frozen-lockfile - name: Deploy the composio broker Worker via Wrangler + if: steps.token.outputs.present == 'true' uses: cloudflare/wrangler-action@v3 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 778c11578e..2e469b302e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,48 +105,38 @@ jobs: ci: # Ship only a commit whose own `CI` check (ci.yml's gate) passed. The wait - # overlaps the platform builds, so a green commit costs no time; a red, - # cancelled or missing check stops the release before any draft exists. - # 0.1.79 and 0.1.90 shipped commits whose main runs never finished. + # overlaps the platform builds, so a green commit costs no time; a red or + # missing verdict stops the release before any draft exists. Main keeps + # one CI run going and one waiting, so a burst of merges can replace the + # release commit's own run: scripts/release-ci.mjs then runs CI on the + # commit in its own lane (a release-ci/v branch no merge can + # touch) and deletes that branch afterwards. name: CI passed on the release commit needs: prepare if: needs.prepare.outputs.should_release == 'true' runs-on: ubuntu-24.04 timeout-minutes: 180 permissions: - actions: read + # read CI runs and start CI in the release lane + actions: write + # create and delete the release-ci/v branch + contents: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: scripts/release-ci.mjs + sparse-checkout-cone-mode: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 - name: Wait for the CI check on the pinned commit if: ${{ !inputs.ship_without_ci }} env: GH_TOKEN: ${{ github.token }} SHA: ${{ needs.prepare.outputs.sha }} - run: | - # push = a merge on main; merge_group = the queue's commit that - # becomes main; workflow_dispatch = a manual CI run of this commit. - # A pull_request run tests a merge ref, not this exact commit. - started=$SECONDS - while :; do - run=$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?head_sha=$SHA&per_page=50" \ - --jq '[.workflow_runs[] | select(.event == "push" or .event == "merge_group" or .event == "workflow_dispatch")] | sort_by(.created_at) | last | select(. != null) | "\(.id) \(.status)"') - if [ -z "$run" ]; then - if [ $((SECONDS - started)) -gt 900 ]; then - echo "::error::No CI run exists for $SHA. Run CI on it (Actions → CI → Run workflow on main while main is at this commit), then re-run this job." - exit 1 - fi - else - set -- $run - if [ "$2" = completed ]; then - gate=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$1/jobs?per_page=100" --jq '.jobs[] | select(.name == "CI") | .conclusion') - echo "CI run $1 on $SHA: gate ${gate:-missing}" - [ "$gate" = success ] && exit 0 - echo "::error::The CI check on $SHA did not pass (${gate:-no gate job}). Fix main or re-run CI ($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$1), then re-run this job." - exit 1 - fi - echo "CI run $1 on $SHA is $2; waiting" - fi - sleep 60 - done + VERSION: ${{ needs.prepare.outputs.version }} + run: node scripts/release-ci.mjs mac: name: macOS arm64 + x64 (sign, notarize, staple) diff --git a/AGENTS.md b/AGENTS.md index 6b3ca0f266..5cc118fa84 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1118,8 +1118,8 @@ it after an upstream merge instead of renaming by hand. ## Upstream sync -Last sync: 2026-10-01, upstream `milind-soni/OpenMausBot` main at -`4ed952aa` (0.1.92) merged into Sagax; `baseVersion` follows it. To repeat: +Last sync: 2026-10-03, upstream `milind-soni/OpenMausBot` main at +`ff01be8a` (0.1.94) merged into Sagax; `baseVersion` follows it. To repeat: - Keep the `upstream` remote fetch-only (`git remote set-url --push upstream no_push`). Never push, open a pull request or file an issue diff --git a/NOTICE b/NOTICE index a01b02e316..e3c1f2ef8d 100644 --- a/NOTICE +++ b/NOTICE @@ -19,6 +19,16 @@ directory was removed and none of its source or license check is included; files across the tree were modified. These changes are licensed under the PolyForm Noncommercial License 1.0.0, not the Apache License 2.0. +The Android companion app bundles a prebuilt WebRTC library for Live calls +(io.github.webrtc-sdk:android), built from the WebRTC project, Copyright (c) +2011, The WebRTC project authors, used under the BSD 3-Clause License. WebRTC +includes third-party components under their own licenses, listed with the +library. + +Selected provider icon paths are from Simple Icons, used under CC0 1.0. +The Cohere provider icon path is from Lobe Icons, Copyright 2023 LobeHub, +used under the MIT License. + The OpenMausBot name and mascot are trademarks of Milind Soni. This product does not use those marks as its name. Apache License 2.0 section 6 does not grant trademark rights. @@ -31,3 +41,13 @@ quantized to int8 by Pulsatrix Technologies inc. Licenses are in third_party/onnxruntime-web, third_party/silero-vad, third_party/3d-speaker and third_party/sherpa-onnx; details and checksums in src/lib/voice-mode/models/NOTICE.txt. + +The browser desktop viewer bundles the unmodified noVNC 1.7.0 library, +Copyright the noVNC authors, under MPL-2.0 and its accompanying licenses. +Attribution, license terms and the corresponding source location are in +public/novnc-NOTICE.txt (novnc-NOTICE.txt in the built UI). + +The iOS companion app bundles a prebuilt WebRTC library for Live calls +(stasel/WebRTC), built from the WebRTC project, Copyright (c) 2011, The WebRTC +project authors, used under the BSD 3-Clause License. WebRTC includes +third-party components under their own licenses, listed with the library. diff --git a/README.md b/README.md index 6fa23efc91..1b40323769 100644 --- a/README.md +++ b/README.md @@ -278,13 +278,13 @@ server"). For a solo server on a VPS, see Most settings live in the app (Settings). The server also reads environment variables. Names start with `SAGAX_`; the old `OMB_*`, `OPENMAUSBOT_*` and -`OPENMAUS_*` names are still read and moved onto `SAGAX_*` at start, for one +`SAGAX_*` names are still read and moved onto `SAGAX_*` at start, for one release. Some deployment files (`.env.example`, `compose.yaml`, `deploy/`) -still use the `OMB_*` names on purpose. +still use the `SAGAX_*` names on purpose. | Variable | Default | What it does | |---|---|---| -| `SAGAX_DATA_DIR` | `~/.sagax` | Where bots, conversations and settings live. An old `~/.openmausbot` moves there on first start. | +| `SAGAX_DATA_DIR` | `~/.sagax` | Where bots, conversations and settings live. An old `~/.sagax` moves there on first start. | | `SAGAX_PORT` | `8799` | Port of the Sagax server (127.0.0.1). | | `SAGAX_WEBHOOK_PORT` | `SAGAX_PORT` + 1 | Port of the webhook receiver for routines. | | `SAGAX_PUBLIC_URL` | none | The server's public address, used in links and sign-in. | @@ -362,11 +362,11 @@ bundling it, needs a license from Pulsatrix Technologies inc. See Sagax is based on OpenMausBot, Copyright 2026 Milind Soni and OpenMausBot contributors, under the [Apache License 2.0](LICENSE-APACHE). The parts that -come unchanged from OpenMausBot stay under Apache 2.0; the changes and +come unchanged from Sagax stay under Apache 2.0; the changes and additions made by Pulsatrix Technologies inc. are under the PolyForm Noncommercial License only. Copies published before 2026-10-02 under Apache -2.0 keep that license. The OpenMausBot notices are kept in [NOTICE](NOTICE). -The OpenMausBot name and mascot are trademarks of Milind Soni and are not this +2.0 keep that license. The Sagax notices are kept in [NOTICE](NOTICE). +The Sagax name and mascot are trademarks of Milind Soni and are not this product's name. The source-available `enterprise/` directory of the original project is not part of this tree. diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 6e53605b00..9c592bea66 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -205,6 +205,12 @@ kotlin { dependencies { implementation(project(":core")) + // Live calls: a prebuilt libwebrtc (BSD-3), the phone's own audio path to + // OpenAI GPT-Live. Only `audio/WebRtcLiveCallTransport.kt` may import + // `org.webrtc` (LiveCallNativeIsolationTest pins it), so the JVM suite + // never loads libjingle_peerconnection_so. Measured 2026-09-25: +53 MB on + // the universal debug APK (four ABIs); +12 MB on an arm64 phone. + implementation("io.github.webrtc-sdk:android:150.7871.01") val composeBom = platform("androidx.compose:compose-bom:2025.10.01") implementation(composeBom) diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 1fd18feebb..02087b9078 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -30,11 +30,18 @@ android:name="android.hardware.camera.autofocus" android:required="false" /> + + diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/MainActivity.kt b/android/app/src/main/kotlin/com/openmausbot/companion/MainActivity.kt index 4edd8ea620..95e418f522 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/MainActivity.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/MainActivity.kt @@ -179,6 +179,7 @@ class MainActivity : ComponentActivity() { avatars = app.avatars, voicePreview = app.voicePreview, voiceNotes = app.voiceNotes, + liveCalls = app.liveCalls, dictation = dictation, chatDrafts = chatDrafts, requestPermissions = { permissions -> permissionRequests.request(permissions) }, diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/OpenMausApp.kt b/android/app/src/main/kotlin/com/openmausbot/companion/OpenMausApp.kt index 8d1011be2a..de232ed8f5 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/OpenMausApp.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/OpenMausApp.kt @@ -2,6 +2,7 @@ package com.openmausbot.companion import android.app.Application import androidx.lifecycle.ProcessLifecycleOwner +import com.openmausbot.companion.audio.LiveCallManager import com.openmausbot.companion.audio.VoicePreviewPlayer import com.openmausbot.companion.audio.VoiceNotePlayer import com.openmausbot.companion.avatar.AvatarImageStore @@ -57,6 +58,8 @@ class OpenMausApp : Application() { private set lateinit var voiceNotes: VoiceNotePlayer private set + lateinit var liveCalls: LiveCallManager + private set lateinit var linger: SessionLingerController private set lateinit var shareInbox: ShareInbox @@ -89,8 +92,18 @@ class OpenMausApp : Application() { notificationSink = notifications, ) avatars = AvatarImageStore(fetch = session::avatarData) - voicePreview = VoicePreviewPlayer(this) - voiceNotes = VoiceNotePlayer(this) + // Live calls hold media across screens, and end when the process + // leaves the foreground — not when a rotation recreates the Activity — + // so the manager is app-scoped and observes the process lifecycle, as + // the linger controller below does. + liveCalls = LiveCallManager(this, session, appScope) + ProcessLifecycleOwner.get().lifecycle.addObserver(liveCalls) + // A voice note or a voice preview asks for the audio focus, and the + // call ends when it loses that focus: both players refuse while the + // call holds the audio, whatever asks them to play. + val liveCallHoldsAudio = { liveCalls.state.value.holdsMedia } + voicePreview = VoicePreviewPlayer(this, liveCallHoldsAudio = liveCallHoldsAudio) + voiceNotes = VoiceNotePlayer(this, liveCallHoldsAudio = liveCallHoldsAudio) // iOS resets the avatar cache inside signOut. Observe Unpaired here so // the platform cache cannot outlive the pairing that minted its URLs. diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallApi.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallApi.kt new file mode 100644 index 0000000000..7beffaf042 --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallApi.kt @@ -0,0 +1,51 @@ +package com.openmausbot.companion.audio + +import com.openmausbot.companion.core.LiveCallStart +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.Session +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map + +/** The two calls and the reports a Live call needs from the computer. [Session] in production, a fake in tests. */ +interface LiveCallApi { + suspend fun start(botId: String, threadId: String, sdp: String): LiveCallStart + + suspend fun end(callId: String) + + /** The computer's current call as the `live.call` frames and `GET /api/live/call` report it. */ + val serverCall: Flow + + /** + * Which computer this phone follows, and whether its pairing still + * stands. A call belongs to the computer it started on: losing the + * pairing hangs it up, and switching computers leaves it behind. + */ + val link: Flow +} + +/** + * [computerId] is the paired computer's connection id, null while none is + * selected; [signedIn] is false once the pairing is gone or its token is + * refused (`Session.Status.Unpaired` or `Unauthorized`). + */ +data class LiveCallLink(val computerId: String?, val signedIn: Boolean) + +internal class SessionLiveCallApi(private val session: Session) : LiveCallApi { + override suspend fun start(botId: String, threadId: String, sdp: String): LiveCallStart = + session.startLiveCall(botId, threadId, sdp) + + override suspend fun end(callId: String) { + session.endLiveCall(callId) + } + + override val serverCall: Flow = session.state.map { it.liveCall }.distinctUntilChanged() + + override val link: Flow = combine(session.connection, session.status) { connection, status -> + LiveCallLink( + computerId = connection?.id, + signedIn = status !is Session.Status.Unpaired && status !is Session.Status.Unauthorized, + ) + }.distinctUntilChanged() +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallAudioRoute.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallAudioRoute.kt new file mode 100644 index 0000000000..3fc1454a4d --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallAudioRoute.kt @@ -0,0 +1,188 @@ +package com.openmausbot.companion.audio + +import android.annotation.SuppressLint +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioDeviceCallback +import android.media.AudioDeviceInfo +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.os.Build +import android.os.Handler +import android.os.Looper + +/** + * Where a Live call's audio goes and who owns it: MODE_IN_COMMUNICATION for + * the call's lifetime, voice-communication audio focus (voice notes and + * previews pause on it, as they already do for dictation), and the route. + * A headset the person has on (wired, USB, Bluetooth, a hearing aid) keeps + * the call off the loudspeaker, so replies are not played out loud to the room + * while earbuds are in; the speaker setting only chooses between the + * loudspeaker and the earpiece when no headset is connected. Behind an + * interface so the manager's tests need no AudioManager. Needs + * MODIFY_AUDIO_SETTINGS, declared in the manifest. + */ +interface LiveCallAudioRoute { + /** + * Take the mode, the focus and the first route. [onFocusLost] fires when + * another app takes the audio for good or for a while (a phone call); + * a chime that may duck the call is not reported. + */ + fun begin(speaker: Boolean, onFocusLost: () -> Unit) + + /** + * The loudspeaker (true) or the earpiece (false) while no headset is + * connected. Before [begin] it is only remembered. + */ + fun setSpeaker(speaker: Boolean) + + /** Give it all back. Safe without [begin]. */ + fun end() +} + +/** + * The routing rule, kept apart from AudioManager so a plain JVM test can pin + * it. Device types are `AudioDeviceInfo.TYPE_*` values: compile-time + * constants that are only compared, so the newer ones (`TYPE_HEARING_AID`, + * API 28; `TYPE_BLE_HEADSET`, API 31) are safe on older phones, which never + * list such a device. + */ +@SuppressLint("InlinedApi") +internal object LiveCallAudioRouting { + /** + * Devices the person wears or has plugged in, most preferred first. The + * order is close to Android's own call routing: a hearing aid, then a + * plug, then Bluetooth, which can stay connected while not being worn. + */ + val HEADSETS: List = listOf( + AudioDeviceInfo.TYPE_HEARING_AID, + AudioDeviceInfo.TYPE_WIRED_HEADSET, + AudioDeviceInfo.TYPE_WIRED_HEADPHONES, + AudioDeviceInfo.TYPE_USB_HEADSET, + AudioDeviceInfo.TYPE_BLE_HEADSET, + AudioDeviceInfo.TYPE_BLUETOOTH_SCO, + ) + + /** + * Connected outputs that keep private replies off the loudspeaker. + * ponytail: Below API 31 Bluetooth falls back to the earpiece; add SCO + * routing only after physical older-phone microphone/headset validation. + */ + private val PRIVATE_OUTPUTS: List = listOf( + AudioDeviceInfo.TYPE_HEARING_AID, + AudioDeviceInfo.TYPE_WIRED_HEADSET, + AudioDeviceInfo.TYPE_WIRED_HEADPHONES, + AudioDeviceInfo.TYPE_USB_HEADSET, + AudioDeviceInfo.TYPE_BLUETOOTH_SCO, + AudioDeviceInfo.TYPE_BLUETOOTH_A2DP, + ) + + /** + * API 31 and up: the type of the communication device to select from + * [available], or null to leave the choice to the system (for example a + * tablet that has no earpiece). + */ + fun communicationDevice(available: List, speaker: Boolean): Int? { + HEADSETS.firstOrNull { it in available }?.let { return it } + val builtIn = if (speaker) AudioDeviceInfo.TYPE_BUILTIN_SPEAKER else AudioDeviceInfo.TYPE_BUILTIN_EARPIECE + return builtIn.takeIf { it in available } + } + + /** + * API 26 to 30: whether to turn the speakerphone on. It stays off while a + * device from [PRIVATE_OUTPUTS] is among [outputs], including Bluetooth + * whose call route falls back to the earpiece on these older APIs. + */ + fun speakerphone(outputs: List, speaker: Boolean): Boolean = speaker && PRIVATE_OUTPUTS.none { it in outputs } +} + +internal class AndroidLiveCallAudioRoute(context: Context) : LiveCallAudioRoute { + private val audioManager = context.applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + private var focusRequest: AudioFocusRequest? = null + + /** The last speaker setting; applied whenever the route is recomputed. */ + private var speaker = true + + /** Between [begin] and [end]: the route is ours to set. */ + private var routing = false + + private val attributes: AudioAttributes = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + + /** + * Earbuds that connect mid-call take the call, and a pulled plug hands it + * back to the speaker setting. An explicitly selected device (the + * loudspeaker) would otherwise keep the call after a headset arrives. + */ + private val deviceChanges = object : AudioDeviceCallback() { + override fun onAudioDevicesAdded(addedDevices: Array) = route() + + override fun onAudioDevicesRemoved(removedDevices: Array) = route() + } + + override fun begin(speaker: Boolean, onFocusLost: () -> Unit) { + val request = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN) + .setAudioAttributes(attributes) + .setOnAudioFocusChangeListener { change -> + when (change) { + AudioManager.AUDIOFOCUS_LOSS, AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> onFocusLost() + } + } + .build() + focusRequest = request + // Refused while a phone or VoIP call holds the audio: this call must + // not share the microphone and route with it, and no loss will ever + // come for focus that was never granted. + if (audioManager.requestAudioFocus(request) != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + focusRequest = null + onFocusLost() + return + } + audioManager.mode = AudioManager.MODE_IN_COMMUNICATION + this.speaker = speaker + routing = true + route() + // The main looper, where the manager calls begin, setSpeaker and end, + // so a device change never runs in the middle of one of them. + audioManager.registerAudioDeviceCallback(deviceChanges, Handler(Looper.getMainLooper())) + } + + override fun setSpeaker(speaker: Boolean) { + this.speaker = speaker + route() + } + + override fun end() { + if (routing) { + audioManager.unregisterAudioDeviceCallback(deviceChanges) + routing = false + if (Build.VERSION.SDK_INT >= 31) audioManager.clearCommunicationDevice() + @Suppress("DEPRECATION") + audioManager.isSpeakerphoneOn = false + audioManager.mode = AudioManager.MODE_NORMAL + } + focusRequest?.let(audioManager::abandonAudioFocusRequest) + focusRequest = null + } + + /** + * Apply [LiveCallAudioRouting] to the devices connected now. A device + * change already queued when [end] ran finds [routing] false and does + * nothing. + */ + private fun route() { + if (!routing) return + if (Build.VERSION.SDK_INT >= 31) { + val available = audioManager.availableCommunicationDevices + val wanted = LiveCallAudioRouting.communicationDevice(available.map { it.type }, speaker) + val device = available.firstOrNull { it.type == wanted } + if (device != null) audioManager.setCommunicationDevice(device) else audioManager.clearCommunicationDevice() + return + } + val outputs = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS).map { it.type } + @Suppress("DEPRECATION") + audioManager.isSpeakerphoneOn = LiveCallAudioRouting.speakerphone(outputs, speaker) + } +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallManager.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallManager.kt new file mode 100644 index 0000000000..6afa288d61 --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallManager.kt @@ -0,0 +1,517 @@ +package com.openmausbot.companion.audio + +import android.content.Context +import androidx.lifecycle.DefaultLifecycleObserver +import androidx.lifecycle.LifecycleOwner +import com.openmausbot.companion.core.LiveCallStart +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.LiveCallStatus +import com.openmausbot.companion.core.Session +import com.openmausbot.companion.ui.LiveCallRules +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch + +/** Asks for the microphone; `MicPermissionController.ensure` in production. */ +fun interface MicrophoneAccess { + fun ensure(onResult: (Boolean) -> Unit) +} + +/** + * This phone's side of a Live call, app-scoped like [VoiceNotePlayer]: the + * chat screen is disposed when Computer is pushed or the roster returns, and + * a call must survive both. The computer owns the conversation (delegation, + * approvals, the idle hang-up); this owns media, captions and the local + * controls, and it is the one thing that sends `session.close` to OpenAI. + * + * The bar says "Connecting…" until the computer reports the call attached + * and this phone's data channel is open; the clock counts from that moment, + * on this phone's clock. After Hang up it says "Hanging up…" until the + * computer confirms the end. An ended call keeps its reason in the bar, with + * Try again only after a drop or a failed start. Losing the pairing hangs the + * call up at once; switching to another computer leaves it behind. + * + * Ends on process `ON_STOP` ([androidx.lifecycle.ProcessLifecycleOwner], + * 700 ms debounced), never the Activity's: rotation recreates MainActivity + * (the manifest declares no `configChanges`) and must not hang up. + * + * **Main-thread confined.** Lifecycle callbacks and the computer's frames + * arrive on [scope] (`Dispatchers.Main.immediate` in production); WebRTC's + * own threads deliver [LiveCallTransport.Listener] callbacks, which hop onto + * [scope] before touching anything. So the fields below need no locking, and + * a test drives the whole machine on a TestScope. + */ +class LiveCallManager internal constructor( + private val api: LiveCallApi, + private val scope: CoroutineScope, + private val transports: LiveCallTransportFactory, + private val audio: LiveCallAudioRoute, + private val preferences: LiveCallPreferences, + /** This phone's clock, epoch ms: the bar's clock counts from the moment the call went live on it. */ + private val clock: () -> Long = System::currentTimeMillis, +) : DefaultLifecycleObserver { + + constructor(context: Context, session: Session, scope: CoroutineScope) : this( + api = SessionLiveCallApi(session), + scope = scope, + transports = LiveCallTransportFactory { WebRtcLiveCallTransport(context) }, + audio = AndroidLiveCallAudioRoute(context), + preferences = SharedLiveCallPreferences(context), + ) + + private val _state = MutableStateFlow(LiveCallSnapshot(speaker = preferences.speaker)) + val state: StateFlow = _state.asStateFlow() + + /** The media of the call in progress; null between calls. Identity doubles as "is this attempt still wanted". */ + private var transport: LiveCallTransport? = null + + /** The computer has reported our call at least once; from then on "no call" means it is gone. */ + private var seenOnServer = false + + /** The computer has said its side of our call reached OpenAI: `live`, or any later word short of `ended`. */ + private var attached = false + + /** Our `oai-events` data channel is open. */ + private var channelOpen = false + + /** Bumped by every [start]: the generation guard for a microphone answer that outlives its attempt. */ + private var attempts = 0 + + /** + * The computer's latest report, whatever it was about. A frame about this + * phone's call can beat the 201 that names it; [connect] reads it again + * once the id is known. + */ + private var lastServerCall: LiveCallState? = null + + /** The id of the last call the computer created for this phone, kept across attempts: a 409 naming it is our own call winding down. */ + private var lastCallId: String? = null + + /** Ends the call if its audio never connects ([armConnectDeadline]); null when no call is waiting on its audio. */ + private var connectDeadline: Job? = null + + /** Stops "Hanging up…" waiting for a computer that never answers ([END_TIMEOUT_MS]). */ + private var endDeadline: Job? = null + + /** The computer this phone follows and whether its pairing stands; null before the first report. */ + private var link: LiveCallLink? = null + + init { + scope.launch { api.serverCall.collect { onServerCall(it) } } + scope.launch { api.link.collect { onLink(it) } } + } + + /** + * Before this phone's first Live call the chat says what a call sends to + * OpenAI, with Start call and Cancel: a phone has no Live switch, so its + * first call is where Live is turned on. Due until [acceptDisclosure]. + */ + val disclosureDue: Boolean get() = !preferences.disclosureShown + + /** The person chose Start call on the disclosure: not shown on this phone again. Cancel records nothing. */ + fun acceptDisclosure() { + preferences.disclosureShown = true + } + + /** Start a call on [threadId]. Ignored while one is starting, running or hanging up. */ + fun start(botId: String, threadId: String, botName: String, microphone: MicrophoneAccess) { + if (_state.value.active) return + val attempt = ++attempts + _state.value = LiveCallSnapshot( + phase = LiveCallPhase.STARTING, + botId = botId, + threadId = threadId, + botName = botName, + speaker = preferences.speaker, + ) + microphone.ensure { granted -> + // The system sheet can outlast the attempt: leaving the app under + // it ends the call, and a late answer must not start it again + // (Try again asks anew) nor connect a later attempt a second time. + if (attempt != attempts || _state.value.phase != LiveCallPhase.STARTING) return@ensure + if (!granted) { + // Settings has to change first: no Try again. + settle(LiveCallRules.MIC_DENIED_MESSAGE, canRetry = false) + return@ensure + } + scope.launch { connect() } + } + } + + /** Start again on the chat the last call was on, where the bar offers Try again. */ + fun retry(microphone: MicrophoneAccess) { + val last = _state.value + if (last.phase != LiveCallPhase.ENDED || !last.canRetry) return + val botId = last.botId ?: return + val threadId = last.threadId ?: return + _state.value = LiveCallSnapshot(speaker = preferences.speaker) + start(botId, threadId, last.botName, microphone) + } + + /** + * The person hangs up. OpenAI is told and the media released at once. If + * the computer has named the call, the bar says "Hanging up…" until it + * confirms the end (its answer, an `ended` frame, or [END_TIMEOUT_MS]), + * then goes; the call's id stays so the computer's `ending` echo of it + * still reads as this phone's call, not another device's. Before the + * computer named it there is nothing to wait for, and the bar goes now. + */ + fun hangUp() { + val current = _state.value + if (!current.holdsMedia) return + releaseMedia(tellOpenAi = true) + val callId = current.callId + if (callId == null) { + // A 201 that lands later is ended by [connect]. + _state.value = LiveCallSnapshot(speaker = preferences.speaker) + return + } + _state.value = current.copy(phase = LiveCallPhase.ENDING, caption = "", heard = "") + endDeadline = scope.launch { + delay(END_TIMEOUT_MS) + finishEnding(callId) + } + scope.launch { + api.end(callId) + finishEnding(callId) + } + } + + fun setMuted(muted: Boolean) { + transport?.setMuted(muted) + _state.update { it.copy(muted = muted) } + } + + /** Speaker or earpiece: this phone's own choice, remembered for the next call and the next launch. */ + fun setSpeaker(speaker: Boolean) { + preferences.speaker = speaker + if (_state.value.holdsMedia) audio.setSpeaker(speaker) + _state.update { it.copy(speaker = speaker) } + } + + /** Clear the notice of a call that ended, keeping its id for the same reason [hangUp] does. */ + fun dismiss() { + val current = _state.value + if (current.phase == LiveCallPhase.ENDED) { + _state.value = LiveCallSnapshot(callId = current.callId, speaker = preferences.speaker) + } + } + + /** The app left the foreground or the screen locked: the call ends (background calls are a later feature). Not a drop: no Try again. */ + override fun onStop(owner: LifecycleOwner) { + if (!_state.value.holdsMedia) return + endLocally(LiveCallRules.CALL_ENDED, tellComputer = true, canRetry = false) + } + + private suspend fun connect() { + val target = _state.value + val botId = target.botId ?: return + val threadId = target.threadId ?: return + val media = transports.create() + transport = media + seenOnServer = false + attached = false + channelOpen = false + audio.begin(speaker = target.speaker) { + // Another app took the audio: the call ended, it did not drop. + scope.launch { if (transport === media) endLocally(LiveCallRules.FOCUS_LOST_MESSAGE, tellComputer = true, canRetry = false) } + } + val offer = try { + media.offer(listenerFor(media)) + } catch (error: CancellationException) { + throw error + } catch (_: Exception) { + // A hang-up closes the transport under a pending offer, which then + // fails on purpose: that attempt is already over, quietly. Anything + // else is this phone's media, worded like a rejected answer. + if (transport === media) settle(LiveCallRules.AUDIO_FAILED_MESSAGE) + return + } + // Hung up while the offer was being built: nothing reached the computer. + if (transport !== media) return + val started = try { + api.start(botId, threadId, offer) + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + if (transport === media) settle(error.message?.takeIf { it.isNotBlank() } ?: LiveCallRules.START_FAILED_MESSAGE) + return + } + if (started is LiveCallStart.Started) lastCallId = started.call.callId + if (transport !== media) { + // Hung up, backgrounded or signed out while the computer was + // creating the session: release what it created, and leave the + // bar as that left it. The id is recorded first so the + // computer's echo of it reads as ours. + if (started is LiveCallStart.Started) { + _state.update { if (!it.active && it.callId == null) it.copy(callId = started.call.callId) else it } + scope.launch { api.end(started.call.callId) } + } + return + } + when (started) { + // Only the computer can take a key: no Try again. + is LiveCallStart.NeedsKey -> settle(LiveCallRules.NEEDS_KEY_MESSAGE, canRetry = false) + is LiveCallStart.Busy -> if (started.activeCall.callId == lastCallId) { + // This phone's previous call, which the computer is still + // winding down: not another device's, and in a moment Try + // again works. Its id goes back on the snapshot so the bar + // does not show it as a call on another phone either. + settle(LiveCallRules.LAST_CALL_ENDING) + _state.update { it.copy(callId = lastCallId) } + } else { + // Another device holds the line: it has to hang up first. + settle(LiveCallRules.busyMessage(started.activeCall), canRetry = false) + } + is LiveCallStart.Started -> { + _state.update { it.copy(callId = started.call.callId) } + // The computer created the session but could not attach its + // side, and says so in the 201 itself: there is no call to join. + if (started.call.status == LiveCallStatus.ENDED) { + settle(noticeFor(started.call), canRetry = started.call.dropped) + return + } + if (started.call.attached) attached = true + // A frame about this call that beat the 201 was passed over + // (no id yet); read the latest one again now that there is. + applyServerCall(lastServerCall) + if (transport !== media) return + // Armed as the answer goes in rather than after: CONNECTED can + // only follow the answer, so its report always finds the deadline. + armConnectDeadline(media) + try { + media.accept(started.answerSdp) + } catch (error: CancellationException) { + throw error + } catch (_: Exception) { + if (transport === media) { + // The computer holds a session this phone can never use: release it. + scope.launch { api.end(started.call.callId) } + settle(LiveCallRules.AUDIO_FAILED_MESSAGE) + } + return + } + if (transport !== media) return + media.setMuted(_state.value.muted) + goLiveIfReady() + } + } + } + + /** + * [media]'s callbacks. A WebRTC thread delivers them and [scope] runs them + * later, when a hang-up may have closed [media] and a new call begun: each + * acts only while [media] is still this phone's transport. + */ + private fun listenerFor(media: LiveCallTransport): LiveCallTransport.Listener = object : LiveCallTransport.Listener { + override fun onMessage(text: String) { + val event = LiveCaptions.parse(text) ?: return + scope.launch { + if (transport !== media) return@launch + if (event.type == "session.closed") { + // The computer's `ended` frame usually follows with its own + // reason; see the ENDED branch of [applyServerCall]. + val end = LiveCallRules.endNotice(event.reason) + endLocally(end.text, tellComputer = true, canRetry = end.dropped) + return@launch + } + val current = _state.value + val (caption, heard) = LiveCaptions.apply(current.caption, current.heard, event) + _state.update { it.copy(caption = caption, heard = heard) } + } + } + + override fun onConnected() { + scope.launch { if (transport === media) clearConnectDeadline() } + } + + override fun onChannelOpen() { + scope.launch { + if (transport !== media) return@launch + channelOpen = true + // The channel rides the call's own connection: its audio got through. + clearConnectDeadline() + goLiveIfReady() + } + } + + override fun onDropped() { + scope.launch { if (transport === media) endLocally(LiveCallRules.CALL_DROPPED, tellComputer = true, canRetry = true) } + } + } + + /** + * Ends the call as dropped unless [media] reports CONNECTED (or its + * channel opens) within [MEDIA_CONNECT_TIMEOUT_MS]. Every other end + * cancels it ([releaseMedia]), and it acts only while [media] is still + * this phone's transport, so it can end no call but the one it was armed for. + */ + private fun armConnectDeadline(media: LiveCallTransport) { + clearConnectDeadline() + connectDeadline = scope.launch { + delay(MEDIA_CONNECT_TIMEOUT_MS) + connectDeadline = null + if (transport === media) endLocally(LiveCallRules.AUDIO_TIMEOUT_MESSAGE, tellComputer = true, canRetry = true) + } + } + + private fun clearConnectDeadline() { + connectDeadline?.cancel() + connectDeadline = null + } + + private fun onServerCall(call: LiveCallState?) { + lastServerCall = call + applyServerCall(call) + } + + /** What the computer's word about the line means for this phone's call, phase by phase. */ + private fun applyServerCall(call: LiveCallState?) { + val current = _state.value + val ours = current.callId ?: return + when (current.phase) { + LiveCallPhase.STARTING, LiveCallPhase.LIVE -> when { + call?.callId == ours -> { + seenOnServer = true + if (call.status == LiveCallStatus.ENDED) { + endLocally(noticeFor(call), tellComputer = false, canRetry = call.dropped) + } else { + if (call.attached) attached = true + goLiveIfReady() + } + } + // The computer restarted, or forgot the call, while this phone + // was on the line. The desktop and the iPhone say "Call ended." + // too, and it is not a drop. + call == null && seenOnServer -> endLocally(LiveCallRules.CALL_ENDED, tellComputer = false, canRetry = false) + // Another call altogether: not ours to act on. + } + LiveCallPhase.ENDING -> + if (call == null || (call.callId == ours && call.status == LiveCallStatus.ENDED)) finishEnding(ours) + LiveCallPhase.ENDED -> + // Why the computer ended it, after this phone's data channel had + // already stopped it with a plain "Call ended." (the idle + // hang-up, say): the computer's reason replaces it, and says + // whether it was a drop. A drop the phone saw itself stays a + // drop, whatever its own end request made the computer record. + if (current.notice == LiveCallRules.CALL_ENDED && call?.callId == ours && call.status == LiveCallStatus.ENDED) { + _state.update { it.copy(notice = noticeFor(call), canRetry = call.dropped) } + } + LiveCallPhase.IDLE -> Unit + } + } + + /** Live once the computer's side is attached and the data channel is open; the clock starts now. */ + private fun goLiveIfReady() { + val current = _state.value + if (current.phase != LiveCallPhase.STARTING || current.callId == null || !attached || !channelOpen) return + _state.value = current.copy(phase = LiveCallPhase.LIVE, liveSince = clock()) + } + + /** The computer confirmed the hang-up of [callId] (or never answered): the bar goes. */ + private fun finishEnding(callId: String) { + val current = _state.value + if (current.phase != LiveCallPhase.ENDING || current.callId != callId) return + endDeadline?.cancel() + endDeadline = null + _state.value = LiveCallSnapshot(callId = callId, speaker = preferences.speaker) + } + + /** + * A call belongs to the computer and the pairing it started under. When + * the pairing goes (unpaired, signed out, the token refused), the call is + * hung up at once and the bar says why, with no Try again: the computer + * may no longer hear this phone, and a new call would be refused. When + * the phone moves to another computer, the call, or its notice, stays + * behind with the last one. + */ + private fun onLink(next: LiveCallLink) { + val previous = link + link = next + if (previous == null) return + when { + previous.signedIn && !next.signedIn -> + if (_state.value.holdsMedia) endLocally(LiveCallRules.SIGNED_OUT_MESSAGE, tellComputer = true, canRetry = false) + next.computerId != null && next.computerId != previous.computerId -> leaveComputer() + } + } + + /** Quietly, as a deliberate switch should: OpenAI is told, the new computer is not asked about the old one's call. */ + private fun leaveComputer() { + val current = _state.value + if (current.phase == LiveCallPhase.IDLE && current.callId == null) return + if (current.holdsMedia) releaseMedia(tellOpenAi = true) + endDeadline?.cancel() + endDeadline = null + _state.value = LiveCallSnapshot(speaker = preferences.speaker) + } + + /** The attempt is over before media flowed; keep the bar up with the reason. A failed start offers Try again unless [canRetry] says otherwise. */ + private fun settle(notice: String, canRetry: Boolean = true) { + releaseMedia(tellOpenAi = false) + _state.update { it.copy(phase = LiveCallPhase.ENDED, notice = notice, canRetry = canRetry, caption = "", heard = "") } + } + + /** + * The call is over: close the media, tell OpenAI, tell the computer unless + * it told us, keep the bar up with the reason. [canRetry]: only a drop + * offers Try again (the desktop's `canRetry: notice.dropped`). + */ + private fun endLocally(notice: String, tellComputer: Boolean, canRetry: Boolean) { + val callId = _state.value.callId + releaseMedia(tellOpenAi = true) + if (tellComputer && callId != null) scope.launch { api.end(callId) } + _state.update { it.copy(phase = LiveCallPhase.ENDED, notice = notice, canRetry = canRetry) } + } + + private fun releaseMedia(tellOpenAi: Boolean) { + clearConnectDeadline() + val media = transport + transport = null + if (media != null) { + if (tellOpenAi) media.sendClose() + media.close() + } + audio.end() + } + + /** The computer's own words when it sent some, its reason's words otherwise. */ + private fun noticeFor(call: LiveCallState): String = + call.error?.takeIf { it.isNotBlank() } ?: LiveCallRules.endNotice(call.endReason).text + + companion object { + /** + * How long the call's audio has to connect once the answer goes in. A + * reachable path connects within seconds; past this the call ends as + * dropped rather than staying up, and billing on the computer's + * OpenAI key, until the computer's idle hang-up minutes later. + */ + const val MEDIA_CONNECT_TIMEOUT_MS = 20_000L + + /** + * How long "Hanging up…" waits for the computer's word, as on the + * iPhone: longer than the harness's own close window (it gives OpenAI + * 5 s to confirm the close) plus a round trip, so a slow close still + * ends on the computer's answer and a lost one still clears the bar. + */ + const val END_TIMEOUT_MS = 8_000L + } +} + +/** + * The computer's side reached OpenAI: `live`, or any later word short of + * `ended` — `ending`, or a status this build does not know, which counts as + * a call still running. + */ +private val LiveCallState.attached: Boolean + get() = status != LiveCallStatus.CONNECTING && status != LiveCallStatus.ENDED + +/** The computer ended the call as a drop (its reason, as the desktop reads it): only then does the bar offer Try again. */ +private val LiveCallState.dropped: Boolean + get() = LiveCallRules.endNotice(endReason).dropped diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallPreferences.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallPreferences.kt new file mode 100644 index 0000000000..022744d527 --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallPreferences.kt @@ -0,0 +1,50 @@ +package com.openmausbot.companion.audio + +import android.content.Context +import android.content.SharedPreferences + +/** + * This phone's own Live call choices, kept across calls and launches. The + * computer's settings (voice, typed replies, idle minutes) are shared and + * live there; speaker or earpiece, and whether this phone already showed the + * first-call disclosure, are the phone's alone, as on the iPhone. + * Behind an interface so the manager's JVM tests need no SharedPreferences. + */ +interface LiveCallPreferences { + /** The loudspeaker (true) or the earpiece (false) while no headset is connected. */ + var speaker: Boolean + + /** + * This phone showed the first-call disclosure (what a Live call sends to + * OpenAI) and the person chose Start call: it is not shown here again. + */ + var disclosureShown: Boolean +} + +/** [LiveCallPreferences] in app-private SharedPreferences, the way `ChatPreferences` keeps its choices. */ +internal class SharedLiveCallPreferences(private val prefs: SharedPreferences) : LiveCallPreferences { + constructor(context: Context) : this( + context.applicationContext.getSharedPreferences(NAME, Context.MODE_PRIVATE), + ) + + override var speaker: Boolean + get() = prefs.getBoolean(SPEAKER, true) + set(value) { + if (prefs.contains(SPEAKER) && prefs.getBoolean(SPEAKER, true) == value) return + // Small and rare: commit, so the choice is on disk before the process can go. + prefs.edit().putBoolean(SPEAKER, value).commit() + } + + override var disclosureShown: Boolean + get() = prefs.getBoolean(DISCLOSURE_SHOWN, false) + set(value) { + if (prefs.getBoolean(DISCLOSURE_SHOWN, false) == value) return + prefs.edit().putBoolean(DISCLOSURE_SHOWN, value).commit() + } + + private companion object { + const val NAME = "live_call" + const val SPEAKER = "speaker" + const val DISCLOSURE_SHOWN = "disclosure_shown" + } +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallSnapshot.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallSnapshot.kt new file mode 100644 index 0000000000..8819560462 --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallSnapshot.kt @@ -0,0 +1,73 @@ +package com.openmausbot.companion.audio + +/** + * Where this phone's Live call is. IDLE has no bar. STARTING says + * "Connecting…" until the computer reports the call attached and the data + * channel is open; then LIVE. ENDING says "Hanging up…" until the computer + * confirms a hang-up. ENDED keeps the bar up with a notice. + */ +enum class LiveCallPhase { IDLE, STARTING, LIVE, ENDING, ENDED } + +/** + * This phone's side of a Live call — media, captions, mute — as the bar and + * the banner read it. The computer's side (`CompanionState.liveCall`) is a + * separate truth: a call started from another device shows up there, never here. + */ +data class LiveCallSnapshot( + val phase: LiveCallPhase = LiveCallPhase.IDLE, + val botId: String? = null, + val threadId: String? = null, + val botName: String = "", + /** + * The computer's id for this phone's latest call, from the 201. It stays + * after the call is over (ENDED, and the IDLE a hang-up or a dismiss + * leaves) because the computer goes on reporting the call while it winds + * down; the bar needs the id to know that call is this phone's, not + * another device's. The next start, or a switch to another computer, + * clears it. + */ + val callId: String? = null, + /** + * When the call went live, in epoch ms on this phone's clock: the moment + * the computer had reported it attached and the data channel was open. + * The bar's clock counts from it, never from the computer's `startedAt`. + */ + val liveSince: Long? = null, + /** The voice's words, last [CAPTION_CHARS] characters. */ + val caption: String = "", + /** The person's own words while they speak, last [HEARD_CHARS] characters; cleared when the voice answers. */ + val heard: String = "", + val muted: Boolean = false, + val speaker: Boolean = true, + /** Why the call ended, or what went wrong; shown in the bar. */ + val notice: String? = null, + /** + * Whether the ended bar offers Try again: only after a dropped call or a + * failed start, as on the desktop and the iPhone. False after every end + * that is not a drop (the phone button starts the next call), and where + * trying again cannot help until something else changes: no OpenAI key, + * a line busy with another device's call, a denied microphone, a lost + * pairing. + */ + val canRetry: Boolean = true, +) { + /** On a call or getting off one: the bar shows, the banner shows elsewhere, no new call starts. */ + val active: Boolean + get() = phase == LiveCallPhase.STARTING || phase == LiveCallPhase.LIVE || phase == LiveCallPhase.ENDING + + /** + * The microphone, the audio focus and the route are held. Anything that + * would take the audio (dictation, a voice note, a voice preview) stays + * off while this is true: losing the focus ends the call. + */ + val holdsMedia: Boolean + get() = phase == LiveCallPhase.STARTING || phase == LiveCallPhase.LIVE + + /** This chat owns the bar: a running call, or the notice of one that just ended. */ + fun concerns(threadId: String): Boolean = phase != LiveCallPhase.IDLE && this.threadId == threadId + + companion object { + const val CAPTION_CHARS = 240 + const val HEARD_CHARS = 160 + } +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallTransport.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallTransport.kt new file mode 100644 index 0000000000..19645a79ae --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCallTransport.kt @@ -0,0 +1,61 @@ +package com.openmausbot.companion.audio + +/** + * The phone's media for one Live call: microphone in, the voice out, and the + * `oai-events` data channel that carries captions. WebRTC in production + * ([WebRtcLiveCallTransport]); a fake in tests. Nothing else in the app may + * touch `org.webrtc`, so the JVM suite never loads its native library. + * + * One transport is one call: [offer] once, [accept] once, then [close]. + * `close()` may be called while `offer()`/`accept()` are suspended; both must + * then fail quietly (throw, and touch no released media). + */ +interface LiveCallTransport { + interface Listener { + /** A text frame from the data channel, as OpenAI sent it (JSON). Called on a WebRTC thread. */ + fun onMessage(text: String) + + /** The peer connection reached CONNECTED. */ + fun onConnected() + + /** + * The `oai-events` data channel opened: captions can arrive, and + * since the channel rides the call's own connection, its audio got + * through. With the computer's word that its side attached, the + * call is live. Called on a WebRTC thread. + */ + fun onChannelOpen() + + /** The peer connection failed or closed without [close] being called. */ + fun onDropped() + } + + /** + * Create the microphone track and the `oai-events` channel (before the + * offer, as OpenAI requires), create the offer, gather ICE without + * trickle, and return the local SDP. Waits at most [ICE_TIMEOUT_MS] for + * gathering; what has gathered by then is sent. + */ + suspend fun offer(listener: Listener): String + + /** Apply OpenAI's answer, unchanged. Throws when the SDP is rejected. */ + suspend fun accept(answerSdp: String) + + /** Local mute: the track stops sending; nothing is told to OpenAI. */ + fun setMuted(muted: Boolean) + + /** `{"type":"session.close"}` on the data channel — the one client event the harness allows. */ + fun sendClose() + + /** Release everything. Safe to call twice, and while [offer] or [accept] is suspended (they then throw). */ + fun close() + + companion object { + const val ICE_TIMEOUT_MS = 10_000L + const val DATA_CHANNEL = "oai-events" + } +} + +fun interface LiveCallTransportFactory { + fun create(): LiveCallTransport +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCaptions.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCaptions.kt new file mode 100644 index 0000000000..2c8b68f564 --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/LiveCaptions.kt @@ -0,0 +1,32 @@ +package com.openmausbot.companion.audio + +import com.openmausbot.companion.core.CompanionJson +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive + +/** One data-channel event, reduced to what the bar and the manager need. */ +data class LiveChannelEvent(val type: String, val delta: String?, val reason: String?) + +/** The captions rule, the same as the desktop's `applyCaption` (`src/lib/live-call-media.ts`). */ +object LiveCaptions { + /** Null for anything that is not a JSON object with a string `type`. */ + fun parse(text: String): LiveChannelEvent? = runCatching { + val value = CompanionJson.parseToJsonElement(text).jsonObject + val type = value["type"]?.jsonPrimitive?.contentOrNull ?: return@runCatching null + LiveChannelEvent( + type = type, + delta = value["delta"]?.jsonPrimitive?.contentOrNull, + reason = value["reason"]?.jsonPrimitive?.contentOrNull, + ) + }.getOrNull() + + /** The voice's words replace what the person was saying; the person's words accumulate while they speak. */ + fun apply(caption: String, heard: String, event: LiveChannelEvent): Pair = when (event.type) { + "session.output_transcript.delta" -> + (caption + event.delta.orEmpty()).takeLast(LiveCallSnapshot.CAPTION_CHARS) to "" + "session.input_transcript.delta" -> + caption to (heard + event.delta.orEmpty()).takeLast(LiveCallSnapshot.HEARD_CHARS) + else -> caption to heard + } +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/TransportCloseGuard.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/TransportCloseGuard.kt new file mode 100644 index 0000000000..8aaf9e3ec0 --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/TransportCloseGuard.kt @@ -0,0 +1,62 @@ +package com.openmausbot.companion.audio + +import java.util.concurrent.atomic.AtomicBoolean +import kotlinx.coroutines.CompletableDeferred + +/** + * What keeps [WebRtcLiveCallTransport] from touching a disposed peer, apart + * from `org.webrtc` so the JVM suite can pin it. + * + * A hang-up closes the transport on the main thread while `offer()` or + * `accept()` is suspended on a native callback (the offer, a description, ICE + * gathering). The peer is disposed then: the waiter must fail instead of + * calling into it, and a callback the disposed peer never delivers must not + * leave it waiting. + */ +internal class TransportCloseGuard { + private val closed = AtomicBoolean(false) + + /** The native callback a caller is waiting on; [failPending] fails it. */ + @Volatile + private var pending: CompletableDeferred<*>? = null + + val isClosed: Boolean + get() = closed.get() + + /** Throws once [close] has run: the native objects are gone. */ + fun ensureOpen() { + if (closed.get()) throw IllegalStateException(CLOSED) + } + + /** + * Runs [start], which hands [result] to a native observer, and waits for + * it. Throws when [close] ran before, during or just after the wait (a + * callback that landed first but resumed after the close still fails). + * + * A [CompletableDeferred] rather than a continuation: an observer that + * fires after [failPending] is then a no-op, not a second resume that + * would throw on a WebRTC thread. + */ + suspend fun await(result: CompletableDeferred, start: () -> Unit = {}): T { + pending = result + try { + ensureOpen() + start() + return result.await().also { ensureOpen() } + } finally { + pending = null + } + } + + /** True the first time only. The caller then releases its native objects and calls [failPending]. */ + fun close(): Boolean = closed.compareAndSet(false, true) + + /** Fails the wait in progress, if any. Last in `close()`, so the resumed caller finds everything released. */ + fun failPending() { + pending?.completeExceptionally(IllegalStateException(CLOSED)) + } + + companion object { + const val CLOSED = "transport closed" + } +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/VoiceNotePlayer.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/VoiceNotePlayer.kt index 0d8aee433c..0646475098 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/audio/VoiceNotePlayer.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/VoiceNotePlayer.kt @@ -7,6 +7,7 @@ import androidx.lifecycle.DefaultLifecycleObserver import androidx.lifecycle.Lifecycle import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.ProcessLifecycleOwner +import com.openmausbot.companion.ui.LiveCallRules import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharedFlow @@ -54,10 +55,12 @@ class VoiceNotePlayer internal constructor( constructor( context: Context, processLifecycle: Lifecycle = ProcessLifecycleOwner.get().lifecycle, + /** This phone's Live call holds the audio ([AudioFocusGate]). */ + liveCallHoldsAudio: () -> Boolean = { false }, ) : this( controller = VoiceNoteController( engineFactory = { MediaPlayerVoiceNoteEngine() }, - focus = AudioFocusGate(context.applicationContext), + focus = AudioFocusGate(context.applicationContext, liveCallHoldsAudio), ), processLifecycle = processLifecycle, ) @@ -78,7 +81,9 @@ class VoiceNotePlayer internal constructor( /** * Start [key] from its beginning. Any note already playing is released * first (the one-voice rule: the newcomer takes the window's voice). - * @return null on success, or failure copy for the bubble's retry row. + * @return null on success, [VoiceNoteController.DURING_LIVE_CALL] while + * this phone's Live call holds the audio (the clip waits), or failure + * copy for the bubble's retry row. */ fun play(key: String, data: ByteArray): String? = controller.play(key, data) @@ -127,6 +132,11 @@ class VoiceNoteController( val playbackErrors: SharedFlow = _playbackErrors.asSharedFlow() fun play(key: String, data: ByteArray): String? = synchronized(lock) { + // The one place a note asks for the audio: never while this phone's + // Live call holds it, however the request got here (a download that + // finished after the call started, say). Losing the focus would end + // the call. Nothing playing or paused is touched. + if (focus.heldByLiveCall) return DURING_LIVE_CALL releaseInternal(abandonFocus = true) if (!focus.request(onInterrupted = ::onFocusInterrupted)) { focus.abandon() @@ -185,6 +195,7 @@ class VoiceNoteController( val state = _playback.value ?: return null val current = engine ?: return null if (state.playing) return null + if (focus.heldByLiveCall) return DURING_LIVE_CALL if (!focus.request(onInterrupted = ::onFocusInterrupted)) { focus.abandon() return PLAYBACK_ERROR @@ -293,6 +304,9 @@ class VoiceNoteController( companion object { /** Failure copy for the bubble's retry row. */ const val PLAYBACK_ERROR: String = "The voice note could not be played." + + /** The refusal while this phone's Live call holds the audio: not a failure, the clip waits. */ + const val DURING_LIVE_CALL: String = LiveCallRules.VOICE_NOTE_DURING_CALL } } diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/VoicePreviewPlayer.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/VoicePreviewPlayer.kt index ee814565b0..91c30cff9c 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/audio/VoicePreviewPlayer.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/VoicePreviewPlayer.kt @@ -10,6 +10,7 @@ import androidx.lifecycle.DefaultLifecycleObserver import androidx.lifecycle.Lifecycle import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.ProcessLifecycleOwner +import com.openmausbot.companion.ui.LiveCallRules import java.util.concurrent.atomic.AtomicReference import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow @@ -48,10 +49,12 @@ class VoicePreviewPlayer internal constructor( constructor( context: Context, processLifecycle: Lifecycle = ProcessLifecycleOwner.get().lifecycle, + /** This phone's Live call holds the audio ([AudioFocusGate]). */ + liveCallHoldsAudio: () -> Boolean = { false }, ) : this( controller = VoicePreviewController( engineFactory = { MediaPlayerPreviewEngine() }, - focus = AudioFocusGate(context.applicationContext), + focus = AudioFocusGate(context.applicationContext, liveCallHoldsAudio), ), processLifecycle = processLifecycle, ) @@ -146,6 +149,9 @@ class VoicePreviewController( val playbackErrors: SharedFlow = _playbackErrors.asSharedFlow() fun play(data: ByteArray): String? = synchronized(lock) { + // Never while this phone's Live call holds the audio: a preview + // fetched before the call started would take its focus and end it. + if (focus.heldByLiveCall) return DURING_LIVE_CALL stopInternal(abandonFocus = true) if (!focus.request(onInterrupted = ::onFocusInterrupted)) { focus.abandon() @@ -227,6 +233,9 @@ class VoicePreviewController( companion object { /** `AgentProfileView.previewVoice` failure copy. */ const val PLAYBACK_ERROR: String = "The generated audio could not be played." + + /** The refusal while this phone's Live call holds the audio. */ + const val DURING_LIVE_CALL: String = LiveCallRules.PREVIEW_DURING_CALL } } @@ -247,14 +256,28 @@ interface PreviewAudioFocus { */ fun request(onInterrupted: () -> Unit): Boolean fun abandon() + + /** + * This phone's Live call holds the audio. Nothing may ask for the focus + * then: a voice note or a preview that took it would end the call. + */ + val heldByLiveCall: Boolean get() = false } +/** + * The one gate voice notes and voice previews ask for the audio through. It + * refuses while this phone's Live call holds the audio ([liveCallHoldsAudio], + * the call manager's `holdsMedia`), as the iPhone's `VoiceNoteCenter` does. + */ internal class AudioFocusGate( context: Context, + private val liveCallHoldsAudio: () -> Boolean = { false }, ) : PreviewAudioFocus { private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager private var focusRequest: AudioFocusRequest? = null + override val heldByLiveCall: Boolean get() = liveCallHoldsAudio() + private val attributes: AudioAttributes = AudioAttributes.Builder() .setUsage(AudioAttributes.USAGE_MEDIA) .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/audio/WebRtcLiveCallTransport.kt b/android/app/src/main/kotlin/com/openmausbot/companion/audio/WebRtcLiveCallTransport.kt new file mode 100644 index 0000000000..ba43200f3f --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/audio/WebRtcLiveCallTransport.kt @@ -0,0 +1,225 @@ +package com.openmausbot.companion.audio + +import android.content.Context +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicBoolean +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.withTimeoutOrNull +import org.webrtc.AudioSource +import org.webrtc.AudioTrack +import org.webrtc.DataChannel +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.RtpReceiver +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import org.webrtc.audio.JavaAudioDeviceModule + +/** + * libwebrtc (`io.github.webrtc-sdk:android`) behind [LiveCallTransport]. The + * only file in the app that imports `org.webrtc`. + * + * Verified 2026-09-25 on the arm64 API 37 emulator: the library loads, the + * offer carries an audio m-line and the `oai-events` channel, and ICE + * gathering completes in milliseconds with no ICE servers (OpenAI's own + * samples pass none). The voice's remote track plays through the audio device + * module on its own; nothing here renders it. + * + * [close] can run while [offer] or [accept] waits on a native callback (a + * hang-up during "Connecting…"); [guard] then fails the wait, so neither + * touches the disposed peer. + */ +internal class WebRtcLiveCallTransport(context: Context) : LiveCallTransport { + private val app = context.applicationContext + private var adm: JavaAudioDeviceModule? = null + private var factory: PeerConnectionFactory? = null + private var peer: PeerConnection? = null + private var channel: DataChannel? = null + private var source: AudioSource? = null + private var microphone: AudioTrack? = null + private val guard = TransportCloseGuard() + + override suspend fun offer(listener: LiveCallTransport.Listener): String { + // Closed before it began: build nothing that close() has already missed. + guard.ensureOpen() + initializeOnce() + val deviceModule = JavaAudioDeviceModule.builder(app) + // Hardware echo cancellation and noise suppression where the + // device has them; libwebrtc's software AEC otherwise (emulators). + .setUseHardwareAcousticEchoCanceler(JavaAudioDeviceModule.isBuiltInAcousticEchoCancelerSupported()) + .setUseHardwareNoiseSuppressor(JavaAudioDeviceModule.isBuiltInNoiseSuppressorSupported()) + .createAudioDeviceModule() + adm = deviceModule + val peerFactory = PeerConnectionFactory.builder() + .setAudioDeviceModule(deviceModule) + .createPeerConnectionFactory() + factory = peerFactory + + val gathered = CompletableDeferred() + val configuration = PeerConnection.RTCConfiguration(emptyList()).apply { + sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN + } + val connection = peerFactory.createPeerConnection( + configuration, + object : PeerConnection.Observer { + override fun onIceGatheringChange(state: PeerConnection.IceGatheringState) { + if (state == PeerConnection.IceGatheringState.COMPLETE) gathered.complete(Unit) + } + + override fun onConnectionChange(state: PeerConnection.PeerConnectionState) { + when (state) { + PeerConnection.PeerConnectionState.CONNECTED -> listener.onConnected() + PeerConnection.PeerConnectionState.FAILED, + PeerConnection.PeerConnectionState.CLOSED, + -> if (!guard.isClosed) listener.onDropped() + else -> Unit + } + } + + override fun onSignalingChange(state: PeerConnection.SignalingState) = Unit + override fun onIceConnectionChange(state: PeerConnection.IceConnectionState) = Unit + override fun onIceConnectionReceivingChange(receiving: Boolean) = Unit + override fun onIceCandidate(candidate: IceCandidate) = Unit + override fun onIceCandidatesRemoved(candidates: Array) = Unit + override fun onAddStream(stream: MediaStream) = Unit + override fun onRemoveStream(stream: MediaStream) = Unit + override fun onDataChannel(channel: DataChannel) = Unit + override fun onRenegotiationNeeded() = Unit + override fun onAddTrack(receiver: RtpReceiver, streams: Array) = Unit + }, + ) ?: error("createPeerConnection returned null") + peer = connection + + val audioSource = peerFactory.createAudioSource(MediaConstraints()) + source = audioSource + val track = peerFactory.createAudioTrack("mic0", audioSource) + microphone = track + connection.addTrack(track, listOf("omb")) + + // Before the offer, so the m=application section is negotiated. + val events = connection.createDataChannel(LiveCallTransport.DATA_CHANNEL, DataChannel.Init()) + channel = events + events.registerObserver(object : DataChannel.Observer { + override fun onBufferedAmountChange(previous: Long) = Unit + + // close() unregisters this observer before it closes the channel, + // so a channel this transport closed never reports here. + override fun onStateChange() { + if (events.state() == DataChannel.State.OPEN) listener.onChannelOpen() + } + + override fun onMessage(buffer: DataChannel.Buffer) { + if (buffer.binary) return + listener.onMessage(Charsets.UTF_8.decode(buffer.data).toString()) + } + }) + + val created = CompletableDeferred() + // Each wait below throws if close() ran meanwhile, before the peer is used again. + val offer = guard.await(created) { + connection.createOffer( + object : SdpObserver { + override fun onCreateSuccess(description: SessionDescription) { + created.complete(description) + } + + override fun onCreateFailure(reason: String?) { + created.completeExceptionally(IllegalStateException("createOffer failed: $reason")) + } + + override fun onSetSuccess() = Unit + override fun onSetFailure(reason: String?) = Unit + }, + MediaConstraints(), + ) + } + setDescription { observer -> connection.setLocalDescription(observer, offer) } + // Non-trickle ICE: OpenAI takes one complete offer. With no ICE servers + // this is quick; the timeout covers a phone whose network stalls. + withTimeoutOrNull(LiveCallTransport.ICE_TIMEOUT_MS) { guard.await(gathered) } + // A timeout returns without the guard's own check. + guard.ensureOpen() + return connection.localDescription?.description ?: offer.description + } + + override suspend fun accept(answerSdp: String) { + guard.ensureOpen() + val connection = checkNotNull(peer) { "accept before offer" } + setDescription { observer -> + connection.setRemoteDescription(observer, SessionDescription(SessionDescription.Type.ANSWER, answerSdp)) + } + } + + override fun setMuted(muted: Boolean) { + microphone?.setEnabled(!muted) + } + + override fun sendClose() { + val events = channel ?: return + if (events.state() != DataChannel.State.OPEN) return + events.send(DataChannel.Buffer(ByteBuffer.wrap(CLOSE_EVENT.toByteArray(Charsets.UTF_8)), false)) + } + + override fun close() { + if (!guard.close()) return + channel?.let { + it.unregisterObserver() + it.close() + it.dispose() + } + channel = null + microphone?.dispose() + microphone = null + peer?.let { + it.close() + it.dispose() + } + peer = null + // The track holds the source natively but never releases this handle + // on it, so without this every call would leak one audio source. + source?.dispose() + source = null + factory?.dispose() + factory = null + adm?.release() + adm = null + // Last: a waiter resumed by this finds everything already released. + guard.failPending() + } + + private suspend fun setDescription(apply: (SdpObserver) -> Unit) { + val done = CompletableDeferred() + guard.await(done) { + apply(object : SdpObserver { + override fun onSetSuccess() { + done.complete(Unit) + } + + override fun onSetFailure(reason: String?) { + done.completeExceptionally(IllegalStateException("setDescription failed: $reason")) + } + + override fun onCreateSuccess(description: SessionDescription?) = Unit + override fun onCreateFailure(reason: String?) = Unit + }) + } + } + + private fun initializeOnce() { + if (initialized.compareAndSet(false, true)) { + PeerConnectionFactory.initialize( + PeerConnectionFactory.InitializationOptions.builder(app).createInitializationOptions(), + ) + } + } + + private companion object { + const val CLOSE_EVENT = """{"type":"session.close"}""" + + /** `PeerConnectionFactory.initialize` is process-wide and must run once. */ + val initialized = AtomicBoolean(false) + } +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/permissions/CompanionPermissions.kt b/android/app/src/main/kotlin/com/openmausbot/companion/permissions/CompanionPermissions.kt index f546411da3..3bcd6cb17d 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/permissions/CompanionPermissions.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/permissions/CompanionPermissions.kt @@ -44,7 +44,7 @@ class CompanionPermissions( */ fun discoveryPermissions(): Array = refresh().missingDiscovery.toTypedArray() - /** Composer mic — checked at the button, never at cold start. */ + /** Composer mic and Live calls — checked at the button or the call, never at cold start. */ fun recordAudioGranted(): Boolean = granted(Manifest.permission.RECORD_AUDIO) private fun read(): Snapshot { diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/AgentProfileSheet.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/AgentProfileSheet.kt index 4fc804ea4b..a2ba67052a 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/AgentProfileSheet.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/AgentProfileSheet.kt @@ -101,6 +101,10 @@ internal fun AgentProfileSheet(bot: Bot, onDismiss: () -> Unit, onOpenOverview: val context = LocalContext.current val lifecycleOwner = LocalLifecycleOwner.current val player = environment.voicePreview + // A running Live call holds the audio. A preview asks for transient focus, + // which would end the call as "another app took the audio" — the reason + // ChatScreen keeps dictation off during a call, too. + val liveCall by environment.liveCalls.state.collectAsState() // The record the sheet was opened on, so the form has an origin even after // the fleet drops the agent; `current` is what every action is applied to. @@ -568,8 +572,11 @@ internal fun AgentProfileSheet(bot: Bot, onDismiss: () -> Unit, onOpenOverview: ActionRow( text = "Preview voice", painter = R.drawable.ic_volume_up, - enabled = ProfileRules.canPreview(busy, config, form.voice), + enabled = ProfileRules.canPreview(busy, config, form.voice) && !liveCall.holdsMedia, onClick = { + // Disabled is how it looks; this is what stops a tap that + // reaches the click action anyway. + if (environment.liveCalls.state.value.holdsMedia) return@ActionRow scope.launch { if (!ProfileRules.selectedVoiceCanSpeak(config, form.voice)) { session.actionError = ProfileRules.PREVIEW_REFUSED @@ -598,6 +605,9 @@ internal fun AgentProfileSheet(bot: Bot, onDismiss: () -> Unit, onOpenOverview: } }, ) + if (liveCall.holdsMedia) { + IconNote(text = LiveCallRules.PREVIEW_DURING_CALL, icon = Icons.Filled.Info) + } ProfileRules.pickAVoiceHint(config, form.voice)?.let { hint -> IconNote(text = hint, icon = Icons.Filled.Info) } diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatPolicy.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatPolicy.kt index 3110d0ecd2..9f0464b17e 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatPolicy.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatPolicy.kt @@ -130,6 +130,16 @@ object TranscriptLayout { message.role == Message.Role.USER -> BubbleTail.TRAILING else -> BubbleTail.LEADING } + + /** + * How far to scroll the transcript, in px, after it got [shrunkBy] px + * shorter from the bottom — the call bar under it grew — so that a list + * that showed its end shows it again. [hiddenBelow] is how much of the + * list's end is below it now. If more is hidden than the list lost, the + * end was out of view already: the reader had scrolled up, and stays put. + */ + fun keepEndInView(hiddenBelow: Int, shrunkBy: Int): Int = + if (shrunkBy > 0 && hiddenBelow in 1..shrunkBy) hiddenBelow else 0 } /** @@ -478,9 +488,15 @@ object MessageActions { // A tool chip is context, a screenshot is pixels, a digest is a log line. Message.Kind.ACTIVITY, Message.Kind.SCREEN, Message.Kind.DIGEST -> null Message.Kind.COMPACTION -> message.compaction?.summary ?: message.text?.takeIf { it.isNotBlank() } - // The run's report is the part worth keeping; the headline without one. + // The run's report and error are the parts worth keeping; the headline without either. Message.Kind.ROUTINE_RUN -> message.routineRun - ?.let { run -> listOfNotNull(run.headline, run.summary?.takeIf { it.isNotBlank() }).joinToString("\n\n") } + ?.let { run -> + listOfNotNull( + run.headline, + run.summary?.takeIf { it.isNotBlank() }, + run.error?.takeIf { it.isNotBlank() }, + ).joinToString("\n\n") + } ?: message.text?.takeIf { it.isNotBlank() } } diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatScreen.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatScreen.kt index 018938921f..1f152d2286 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatScreen.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatScreen.kt @@ -18,6 +18,7 @@ import androidx.compose.animation.slideOutVertically import androidx.compose.foundation.background import androidx.compose.foundation.clickable import androidx.compose.foundation.gestures.detectTapGestures +import androidx.compose.foundation.gestures.scrollBy import androidx.compose.foundation.interaction.MutableInteractionSource import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box @@ -33,8 +34,10 @@ import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.heightIn import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width import androidx.compose.foundation.layout.widthIn import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.LazyListLayoutInfo import androidx.compose.foundation.lazy.rememberLazyListState import androidx.compose.foundation.shape.CircleShape import androidx.compose.foundation.shape.RoundedCornerShape @@ -45,6 +48,7 @@ import androidx.compose.material.icons.automirrored.filled.List import androidx.compose.material.icons.automirrored.filled.Send import androidx.compose.material.icons.filled.Add import androidx.compose.material.icons.filled.ArrowDropDown +import androidx.compose.material.icons.filled.Call import androidx.compose.material.icons.filled.Close import androidx.compose.material.icons.filled.MoreVert import androidx.compose.material.icons.filled.Settings @@ -66,6 +70,7 @@ import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.rememberUpdatedState import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue +import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip @@ -97,6 +102,7 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.lifecycle.compose.LocalLifecycleOwner import com.openmausbot.companion.R +import com.openmausbot.companion.audio.MicrophoneAccess import com.openmausbot.companion.core.AttachmentPolicy import com.openmausbot.companion.core.Chat import com.openmausbot.companion.core.ChatTarget @@ -113,9 +119,12 @@ import com.openmausbot.companion.core.TranscriptRow import com.openmausbot.companion.core.target import com.openmausbot.companion.core.transcriptRows import java.util.Locale +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job +import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -216,6 +225,23 @@ private fun LoadedChat( val session = environment.session val dictation = environment.dictation val chatDrafts = environment.chatDrafts + val liveCalls = environment.liveCalls + val liveCall by liveCalls.state.collectAsState() + var showingLiveSettings by remember { mutableStateOf(false) } + // Sagax: the Live call button shows only when the computer turned Live + // calls on (SAGAX_LIVE_CALLS=1); off by default, our voice call engine is + // the call path. + var liveCallsEnabled by remember { mutableStateOf(false) } + LaunchedEffect(session) { liveCallsEnabled = session.liveSettings()?.enabled == true } + // The call that waits on this phone's first-call disclosure. + var pendingLiveCall by remember { mutableStateOf(null) } + fun startLiveCall(call: PendingLiveCall) = + liveCalls.start(call.botId, call.threadId, call.botName, MicrophoneAccess { environment.mic.ensure(it) }) + // A call holds the microphone in every chat, not only its own: the manager + // is app-scoped and the call runs on while the person reads another chat, + // where dictation's audio focus would end it as "another app took the audio". + val callHoldsMic = liveCall.holdsMedia + LaunchedEffect(callHoldsMic) { if (callHoldsMic) dictation.stop() } val haptics = rememberHaptics() val scope = rememberCoroutineScope() var threadOpenJob by remember { mutableStateOf(null) } @@ -586,6 +612,27 @@ private fun LoadedChat( if (liveCount == 0 || itemCount == 0) return@LaunchedEffect listState.scrollToItem(itemCount - 1) } + // The call bar sits under the transcript and changes height as a call + // goes on: a caption line once it is live, a second line on the remote + // bar. The list gets shorter from the bottom then, and a LazyColumn keeps + // its top where it was, so the newest message would slide out of sight + // under the bar. A list that showed its end keeps showing it, whatever + // made it shorter; one the reader has scrolled up stays where it is. + LaunchedEffect(listState) { + var height = -1 + snapshotFlow { listState.layoutInfo }.collect { info -> + val shrunkBy = if (height < 0) 0 else height - info.viewportSize.height + height = info.viewportSize.height + val by = TranscriptLayout.keepEndInView(info.endHiddenBelow(), shrunkBy) + if (by == 0 || listState.isScrollInProgress) return@collect + try { + listState.scrollBy(by.toFloat()) + } catch (taken: CancellationException) { + // A drag took the list first: the reader is in charge. + currentCoroutineContext().ensureActive() + } + } + } // A search hit lands on its message. LaunchedEffect(focusedMessageId, transcript.size) { val target = focusedMessageId ?: return@LaunchedEffect @@ -943,6 +990,18 @@ private fun LoadedChat( dictation.stop() if (bot != null) onOpenComputer(bot.id) }, + onCall = { + if (bot != null) { + // MicPermissionController holds one pending callback: + // dictation must be off before the call asks. + dictation.stop() + focusManager.clearFocus() + val call = PendingLiveCall(bot.id, threadId, bot.name) + // A phone's first Live call says first what a call sends to OpenAI. + if (liveCalls.disclosureDue) pendingLiveCall = call else startLiveCall(call) + } + }, + showCall = liveCallsEnabled && LiveCallRules.offersCall(liveCall, state.liveCall), // A bot's face and its name pill are both the door to its // profile; a room has no profile, so its pill opens the same // sheet the + does. @@ -961,6 +1020,16 @@ private fun LoadedChat( ) } + LiveCallBarHost( + chat = chat, + onSettings = { showingLiveSettings = true }, + modifier = Modifier + .align(Alignment.CenterHorizontally) + .widthIn(max = CHAT_CONTENT_MAX_WIDTH) + .fillMaxWidth() + .padding(horizontal = 12.dp), + ) + Composer( modifier = Modifier .align(Alignment.CenterHorizontally) @@ -1031,6 +1100,7 @@ private fun LoadedChat( publishFrom(composer) }, onToggleDictation = { + if (callHoldsMic) return@Composer focusManager.clearFocus() dictation.toggle(capturing = draft) }, @@ -1080,6 +1150,22 @@ private fun LoadedChat( ) } + if (showingLiveSettings) { + LiveCallSettingsSheet(onDismiss = { showingLiveSettings = false }) + } + + pendingLiveCall?.let { call -> + LiveCallDisclosureDialog( + onStart = { + pendingLiveCall = null + liveCalls.acceptDisclosure() + startLiveCall(call) + }, + // Records nothing: the next tap shows it again. + onCancel = { pendingLiveCall = null }, + ) + } + filePreview?.let { item -> FilePreviewSheet( item = item, @@ -1129,8 +1215,8 @@ private val HEADER_SCRIM_FADE = 24.dp private val HEADER_CLEARANCE = 128.dp /** - * Back on the left with the rest-of-app unread count, the bot's computer on the - * right, and the bot itself between them over its name. + * Back on the left with the rest-of-app unread count, a Live call and the bot's + * computer on the right, and the bot itself between them over its name. * * The strip behind the two buttons is opaque and then fades out, so the * transcript slides under the chrome and disappears rather than stopping at a @@ -1143,6 +1229,13 @@ private fun ChatHeader( unreadElsewhere: Int, onBack: () -> Unit, onWatchComputer: () -> Unit, + onCall: () -> Unit, + /** + * False while this phone is on a call (the bar has the controls) and while + * the computer reports one running from another device, which has to hang + * up first: [LiveCallRules.offersCall]. + */ + showCall: Boolean, onOpenProfile: () -> Unit, onOpenThreads: () -> Unit, modifier: Modifier = Modifier, @@ -1172,8 +1265,16 @@ private fun ChatHeader( ) { BackPill(unreadElsewhere = unreadElsewhere, onBack = onBack) Spacer(Modifier.weight(1f)) - // The computer is a bot idea; a room has none (§12). + // The computer and the phone are bot ideas; a room has neither (§12). if (chat is Chat.BotChat) { + if (showCall) { + ChromeButton( + icon = Icons.Filled.Call, + contentDescription = "Call ${chat.name}", + onClick = onCall, + ) + Spacer(Modifier.width(8.dp)) + } ChromeButton( painter = painterResource(R.drawable.ic_display), contentDescription = "Watch ${chat.name}'s computer", @@ -1765,3 +1866,17 @@ private fun Composer( } } } + +/** A Live call the phone button asked for, while the first-call disclosure is up. */ +private data class PendingLiveCall(val botId: String, val threadId: String, val botName: String) + +/** + * How much of the list's end — its last item and the padding after it — lies + * below the viewport, in px: 0 while the end shows, and [Int.MAX_VALUE] when + * the last item is not even laid out (the end is a screen or more away). + */ +private fun LazyListLayoutInfo.endHiddenBelow(): Int { + val last = visibleItemsInfo.lastOrNull() ?: return 0 + if (last.index < totalItemsCount - 1) return Int.MAX_VALUE + return (last.offset + last.size + afterContentPadding - viewportEndOffset).coerceAtLeast(0) +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/CompanionEnvironment.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/CompanionEnvironment.kt index 98db3a3a01..932bc9246a 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/CompanionEnvironment.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/CompanionEnvironment.kt @@ -2,6 +2,7 @@ package com.openmausbot.companion.ui import androidx.compose.runtime.Immutable import androidx.compose.runtime.staticCompositionLocalOf +import com.openmausbot.companion.audio.LiveCallManager import com.openmausbot.companion.audio.VoicePreviewPlayer import com.openmausbot.companion.audio.VoiceNotePlayer import com.openmausbot.companion.avatar.AvatarImageStore @@ -54,10 +55,11 @@ class CameraPermissionController( } /** - * Microphone permission for composer dictation. Asked only from the mic button, - * through [PermissionRequests] so the asked-flag has a single owner. Callers - * pass a result callback; a stop that races the system sheet is discarded by - * [SpeechDictation]'s generation guard, not by dropping this callback. + * Microphone permission for composer dictation and Live calls. Asked from the + * mic button or when a call starts, through [PermissionRequests] so the + * asked-flag has a single owner. Callers pass a result callback; a stop that + * races the system sheet is discarded by [SpeechDictation]'s generation guard, + * not by dropping this callback. */ class MicPermissionController( private val isGranted: () -> Boolean, @@ -116,6 +118,8 @@ class CompanionEnvironment( val voicePreview: VoicePreviewPlayer, /** One-at-a-time transcript voice notes; app-scoped, pauses in place. */ val voiceNotes: VoiceNotePlayer, + /** This phone's Live call: media, captions, controls. App-scoped; ends on process ON_STOP. */ + val liveCalls: LiveCallManager, /** Composer dictation; bind to the chat screen lifecycle. */ val dictation: SpeechDictation, /** diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallBanner.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallBanner.kt new file mode 100644 index 0000000000..b5e57b96f5 --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallBanner.kt @@ -0,0 +1,107 @@ +package com.openmausbot.companion.ui + +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.statusBarsPadding +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Call +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import com.openmausbot.companion.audio.LiveCallPhase +import com.openmausbot.companion.core.ChatTarget +import kotlinx.coroutines.delay + +object LiveCallBannerRules { + /** The call's own chat shows the bar instead; every other screen — Computer over it included — shows the banner. */ + fun onCallsChat(destination: Destination, threadId: String): Boolean = when (destination) { + is Destination.Chat -> destination.target.threadId == threadId + is Destination.Thread -> destination.threadId == threadId + else -> false + } +} + +/** A thin green strip: the call, its clock, and a way back to it. [onHangUp] is null while the hang-up is on its way. */ +@Composable +fun LiveCallBanner(title: String, onOpen: () -> Unit, onHangUp: (() -> Unit)?, modifier: Modifier = Modifier) { + Row( + modifier = modifier + .fillMaxWidth() + .background(LIVE_GREEN.copy(alpha = 0.16f)) + .statusBarsPadding() + .clickable(role = Role.Button, onClick = onOpen) + .padding(start = 16.dp, end = 4.dp, top = 2.dp, bottom = 2.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Icon(imageVector = Icons.Filled.Call, contentDescription = null, tint = LIVE_GREEN, modifier = Modifier.size(16.dp)) + Text( + text = title, + fontSize = 14.sp, + fontWeight = FontWeight.Medium, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + modifier = Modifier + .weight(1f) + .padding(start = 8.dp), + ) + if (onHangUp != null) { + TextButton(onClick = onHangUp) { Text("Hang up", color = MaterialTheme.colorScheme.error) } + } else { + // The button's height, so the strip keeps its size while it says "Hanging up…". + Spacer(Modifier.height(40.dp)) + } + } +} + +/** + * Over every screen but the call's own chat while this phone is on a call, + * or hanging one up (a call another device holds shows only in its chat — + * Ruling 10). Tapping it pushes the call's chat, task and all. + */ +@Composable +fun LiveCallBannerHost(navigator: CompanionNavigator) { + val liveCalls = LocalCompanion.current.liveCalls + val local by liveCalls.state.collectAsState() + if (!local.active) return + val botId = local.botId ?: return + val threadId = local.threadId ?: return + if (LiveCallBannerRules.onCallsChat(navigator.current, threadId)) return + var now by remember { mutableStateOf(System.currentTimeMillis()) } + LaunchedEffect(local.phase) { + while (local.phase == LiveCallPhase.LIVE) { + now = System.currentTimeMillis() + delay(1_000) + } + } + val title = when (local.phase) { + LiveCallPhase.LIVE -> LiveCallRules.title(local.botName, LiveCallRules.elapsed(local.liveSince ?: now, now)) + LiveCallPhase.ENDING -> LiveCallRules.HANGING_UP + else -> LiveCallRules.CONNECTING + } + LiveCallBanner( + title = title, + onOpen = { navigator.push(Destination.Chat(ChatTarget.Bot(botId, threadId))) }, + onHangUp = if (local.holdsMedia) liveCalls::hangUp else null, + ) +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallBar.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallBar.kt new file mode 100644 index 0000000000..3cb1528e5a --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallBar.kt @@ -0,0 +1,353 @@ +package com.openmausbot.companion.ui + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Call +import androidx.compose.material.icons.filled.Close +import androidx.compose.material.icons.filled.Settings +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.LocalTextStyle +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.painter.Painter +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.layout.Layout +import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.platform.testTag +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.Constraints +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import com.openmausbot.companion.R +import com.openmausbot.companion.audio.LiveCallPhase +import com.openmausbot.companion.audio.MicrophoneAccess +import com.openmausbot.companion.core.Chat +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch + +/** What the bar's buttons do. A remote bar uses only [onHangUp]. */ +data class LiveCallBarActions( + val onMute: (Boolean) -> Unit, + val onSettings: () -> Unit, + val onHangUp: () -> Unit, + val onRetry: () -> Unit, + val onDismiss: () -> Unit, +) + +/** The green a running call wears, in the bar and the banner. */ +internal val LIVE_GREEN = Color(0xFF34C759) + +/** + * The compact call bar above the composer: the bot and the clock, one caption + * line, gear, mute, hang up. After Hang up it says "Hanging up…", with nothing + * left to press, until the computer confirms. An ending or an error keeps the + * bar with the reason, Try again where trying again can help, and a cross; a + * reason too long for one line wraps (see [EndedRow]). The call never covers + * the chat — that is the whole point of the bar. + */ +@Composable +fun LiveCallBar(model: LiveCallBarModel, actions: LiveCallBarActions, modifier: Modifier = Modifier) { + when (model) { + LiveCallBarModel.Hidden -> Unit + is LiveCallBarModel.Local -> LocalBar(model, actions, modifier) + is LiveCallBarModel.Remote -> RemoteBar(model, actions.onHangUp, modifier) + } +} + +/** + * Lines a reason may take before it is cut. Three hold every message the bar + * knows on a 360 dp phone at 130 % font; a bigger font or a narrower phone cuts + * the longest ones short rather than grow the bar any further over the chat. + */ +private const val NOTICE_MAX_LINES = 3 + +/** One line of the bar's title or reason; the icon beside it is centred on this height. */ +private val TITLE_LINE_HEIGHT = 20.sp + +/** With the bar's 4 dp end padding, the 12 dp its start has. */ +private val NOTICE_END_INSET = 8.dp + +@Composable +private fun LocalBar(model: LiveCallBarModel.Local, actions: LiveCallBarActions, modifier: Modifier) { + Column( + modifier = modifier + .clip(RoundedCornerShape(18.dp)) + .background(MaterialTheme.colorScheme.onSurface.copy(alpha = 0.08f)) + .padding(start = 12.dp, end = 4.dp, top = 4.dp, bottom = 4.dp) + .semantics { contentDescription = "Live call" }, + verticalArrangement = Arrangement.spacedBy(2.dp), + ) { + if (model.phase == LiveCallPhase.ENDED) { + EndedRow(notice = model.title, canRetry = model.canRetry, actions = actions) + } else if (model.phase == LiveCallPhase.ENDING) { + // As tall as the row with its buttons, so the bar does not jump. + Row(modifier = Modifier.heightIn(min = 40.dp), verticalAlignment = Alignment.CenterVertically) { + BarTitle(text = model.title, tint = secondaryTint, maxLines = 1, modifier = Modifier.weight(1f)) + } + } else { + Row(verticalAlignment = Alignment.CenterVertically) { + BarTitle(text = model.title, tint = LIVE_GREEN, maxLines = 1, modifier = Modifier.weight(1f), clock = model.clock) + BarIcon(contentDescription = "Live call settings", onClick = actions.onSettings, icon = Icons.Filled.Settings) + BarIcon( + contentDescription = if (model.muted) "Unmute" else "Mute", + onClick = { actions.onMute(!model.muted) }, + painter = painterResource(if (model.muted) R.drawable.ic_live_mic_off else R.drawable.ic_mic), + tint = if (model.muted) MaterialTheme.colorScheme.error else Color.Unspecified, + ) + BarIcon( + contentDescription = "Hang up", + onClick = actions.onHangUp, + painter = painterResource(R.drawable.ic_live_hang_up), + tint = MaterialTheme.colorScheme.error, + ) + } + } + if (model.phase == LiveCallPhase.LIVE) { + // The person's own words, in grey, while they speak; the voice's words otherwise. + val speaking = model.heard.isNotEmpty() + Text( + // A line break would end the line early and hide the words after it. + text = (if (speaking) model.heard else model.caption).replace('\n', ' ').ifEmpty { " " }, + fontSize = 13.sp, + color = if (speaking) secondaryTint.copy(alpha = 0.6f) else secondaryTint, + maxLines = 1, + // One line that shows the newest words: a long caption drops its + // oldest words, with the ellipsis at the start — as on the desktop. + overflow = TextOverflow.StartEllipsis, + // A tag, not a description: TalkBack reads the caption itself. + modifier = Modifier + .padding(start = 24.dp, bottom = 2.dp) + .testTag("Captions"), + ) + } + } +} + +/** + * The phone icon and the title — or, once the call is over, the reason. The + * icon keeps to the first line however many lines follow it. With a [clock], + * the title is "Live with Ada" and the clock follows it (see [NameAndClock]). + */ +@Composable +private fun BarTitle(text: String, tint: Color, maxLines: Int, modifier: Modifier = Modifier, clock: String? = null) { + Row(modifier = modifier, verticalAlignment = Alignment.Top) { + Box( + modifier = Modifier.height(with(LocalDensity.current) { TITLE_LINE_HEIGHT.toDp() }), + contentAlignment = Alignment.Center, + ) { + Icon(imageVector = Icons.Filled.Call, contentDescription = null, tint = tint, modifier = Modifier.size(16.dp)) + } + if (clock != null) { + NameAndClock(title = text, clock = clock, modifier = Modifier.padding(start = 8.dp)) + } else { + Text( + text = text, + fontSize = 15.sp, + lineHeight = TITLE_LINE_HEIGHT, + fontWeight = FontWeight.SemiBold, + maxLines = maxLines, + overflow = TextOverflow.Ellipsis, + modifier = Modifier.padding(start = 8.dp), + ) + } + } +} + +/** + * "Live with Ada · 1:05" on one line, in two parts so that only the bot's + * name gives way: the clock is measured first, whole, and the name takes what + * is left, cut short with "…" when it needs more. The clock's digits are + * tabular, so a long name is not cut a letter shorter or longer every second. + * TalkBack reads the two as one, with the whole name. + */ +@Composable +private fun NameAndClock(title: String, clock: String, modifier: Modifier = Modifier) { + Row(modifier = modifier.semantics(mergeDescendants = true) {}) { + Text( + text = title, + fontSize = 15.sp, + lineHeight = TITLE_LINE_HEIGHT, + fontWeight = FontWeight.SemiBold, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + modifier = Modifier.weight(1f, fill = false), + ) + Text( + text = LiveCallRules.clockSuffix(clock), + fontSize = 15.sp, + lineHeight = TITLE_LINE_HEIGHT, + fontWeight = FontWeight.SemiBold, + maxLines = 1, + softWrap = false, + style = LocalTextStyle.current.copy(fontFeatureSettings = "tnum"), + ) + } +} + +/** + * Why the call ended, with Try again (only where trying again can help: not + * for a missing key, a busy line, a denied microphone or a lost pairing) and + * the cross. The reason sits beside the buttons while it fits there on one + * line. A longer one — "A Live call is already running from your computer. + * Hang up there first." — gets the bar's whole width, up to + * [NOTICE_MAX_LINES] lines, and the buttons move under it, at the end: cut to + * one line, it never said what to do. + * + * Beside or under is decided from the reason's one-line width in the same + * measure pass, so a long reason is never drawn cut first. Its first line stays + * where the one-line bar puts it, so the lines below it are all that is added. + */ +@Composable +private fun EndedRow(notice: String, canRetry: Boolean, actions: LiveCallBarActions) { + Layout( + content = { + BarTitle(text = notice, tint = MaterialTheme.colorScheme.error, maxLines = NOTICE_MAX_LINES) + Row(verticalAlignment = Alignment.CenterVertically) { + if (canRetry) TextButton(onClick = actions.onRetry) { Text("Try again") } + BarIcon(contentDescription = "Dismiss", onClick = actions.onDismiss, icon = Icons.Filled.Close) + } + }, + ) { (title, buttons), constraints -> + val buttonsPlaceable = buttons.measure(constraints.copy(minWidth = 0, minHeight = 0)) + val oneLine = title.maxIntrinsicWidth(Constraints.Infinity) + val width = if (constraints.hasBoundedWidth) constraints.maxWidth else oneLine + buttonsPlaceable.width + val besideWidth = (width - buttonsPlaceable.width).coerceAtLeast(0) + if (oneLine <= besideWidth) { + val titlePlaceable = title.measure(Constraints(maxWidth = besideWidth)) + val height = maxOf(titlePlaceable.height, buttonsPlaceable.height) + layout(width, height) { + titlePlaceable.placeRelative(0, (height - titlePlaceable.height) / 2) + buttonsPlaceable.placeRelative(besideWidth, (height - buttonsPlaceable.height) / 2) + } + } else { + // The bar's end padding is sized for icon buttons, which bring their + // own; text that runs to the end keeps the start's margin instead. + val titlePlaceable = title.measure(Constraints(maxWidth = (width - NOTICE_END_INSET.roundToPx()).coerceAtLeast(0))) + // Where the one-line bar centres its line against the buttons. + val top = ((buttonsPlaceable.height - TITLE_LINE_HEIGHT.roundToPx()) / 2).coerceAtLeast(0) + layout(width, top + titlePlaceable.height + buttonsPlaceable.height) { + titlePlaceable.placeRelative(0, top) + buttonsPlaceable.placeRelative(besideWidth, top + titlePlaceable.height) + } + } + } +} + +/** + * A call another device holds: "Live with Ada · 1:05" as on this phone's own + * call, with only Hang up, and where the call is ("From your computer") on a + * line of its own under it, where the live bar has its caption — so the clock + * never has to make room for it on a narrow phone. + */ +@Composable +private fun RemoteBar(model: LiveCallBarModel.Remote, onHangUp: () -> Unit, modifier: Modifier) { + Column( + modifier = modifier + .clip(RoundedCornerShape(18.dp)) + .background(MaterialTheme.colorScheme.onSurface.copy(alpha = 0.08f)) + .padding(start = 12.dp, end = 4.dp, top = 2.dp, bottom = 4.dp) + .semantics { contentDescription = "Live call on another device" }, + ) { + Row(verticalAlignment = Alignment.CenterVertically) { + BarTitle(text = model.title, tint = LIVE_GREEN, maxLines = 1, modifier = Modifier.weight(1f), clock = model.clock) + TextButton(onClick = onHangUp) { Text("Hang up", color = MaterialTheme.colorScheme.error) } + } + Text( + text = model.device, + fontSize = 13.sp, + color = secondaryTint, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + modifier = Modifier.padding(start = 24.dp, bottom = 2.dp), + ) + } +} + +/** A 40 dp icon button; Material keeps the touch target at 48 dp. */ +@Composable +private fun BarIcon( + contentDescription: String, + onClick: () -> Unit, + icon: ImageVector? = null, + painter: Painter? = null, + tint: Color = Color.Unspecified, +) { + val resolved = if (tint == Color.Unspecified) MaterialTheme.colorScheme.onSurface else tint + IconButton(onClick = onClick, modifier = Modifier.size(40.dp)) { + when { + painter != null -> Icon(painter = painter, contentDescription = contentDescription, tint = resolved, modifier = Modifier.size(20.dp)) + icon != null -> Icon(imageVector = icon, contentDescription = contentDescription, tint = resolved, modifier = Modifier.size(20.dp)) + } + } +} + +/** + * The bar for one chat: this phone's call, or a call another device holds on + * this chat. Ticks once a second while a call runs so the clock moves. Reads + * the manager and the session itself, so `ChatScreen` only has to place it. + */ +@Composable +fun LiveCallBarHost(chat: Chat, onSettings: () -> Unit, modifier: Modifier = Modifier) { + val environment = LocalCompanion.current + val liveCalls = environment.liveCalls + val session = environment.session + val scope = rememberCoroutineScope() + val local by liveCalls.state.collectAsState() + val state by session.state.collectAsState() + val server = state.liveCall + var now by remember { mutableStateOf(System.currentTimeMillis()) } + val ticking = local.phase == LiveCallPhase.LIVE || server?.isRunning == true + LaunchedEffect(ticking) { + while (ticking) { + now = System.currentTimeMillis() + delay(1_000) + } + } + val model = LiveCallRules.barModel(local, server, chat.threadId, chat.name, now) + if (model == LiveCallBarModel.Hidden) return + LiveCallBar( + model = model, + actions = LiveCallBarActions( + onMute = liveCalls::setMuted, + onSettings = onSettings, + onHangUp = { + when (model) { + // Someone else's microphone: only the computer can end it. + // A 404 there reads the line again (Session.endLiveCall). + is LiveCallBarModel.Remote -> scope.launch { session.endLiveCall(model.callId) } + else -> liveCalls.hangUp() + } + }, + onRetry = { liveCalls.retry(MicrophoneAccess { environment.mic.ensure(it) }) }, + onDismiss = liveCalls::dismiss, + ), + modifier = modifier, + ) +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallRules.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallRules.kt new file mode 100644 index 0000000000..41453175ee --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallRules.kt @@ -0,0 +1,229 @@ +package com.openmausbot.companion.ui + +import com.openmausbot.companion.audio.LiveCallPhase +import com.openmausbot.companion.audio.LiveCallSnapshot +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.LiveCallStatus +import java.util.Locale + +/** What the call bar shows for a chat: nothing, this phone's call, or a call another device holds. */ +sealed interface LiveCallBarModel { + data object Hidden : LiveCallBarModel + + data class Local( + /** "Live with Ada" while live, before the [clock]; otherwise what is happening, or why the call ended. */ + val title: String, + val caption: String, + val heard: String, + val muted: Boolean, + val speaker: Boolean, + val phase: LiveCallPhase, + /** An ended call offers Try again ([LiveCallSnapshot.canRetry]). */ + val canRetry: Boolean = true, + /** While live: the clock after [title]. A name too long for the line gives way; the clock never does. */ + val clock: String? = null, + ) : LiveCallBarModel + + /** + * A call on this chat that another device holds the microphone for. Only Hang up applies. + * [title] and [clock] make the first line, as on this phone's own call; [device], where the + * call is, has a line of its own under it. + */ + data class Remote(val title: String, val clock: String, val device: String, val callId: String) : LiveCallBarModel +} + +data class LiveVoiceOption(val id: String, val label: String) + +/** + * Wording and arithmetic for the Live call bar, banner and settings sheet. + * Pure, so the JVM suite pins it. The words for how a call ended are the + * desktop's (`call.live.*` in `src/locales/en.json`), so the three clients + * say the same thing for the same reason; the rest is the spec's and the + * iPhone's copy. The text says "computer", never "Mac": the harness runs on + * other systems too. + */ +object LiveCallRules { + const val NEEDS_KEY_MESSAGE = "Set up Live calls on your computer first." + const val MIC_DENIED_MESSAGE = "Live calls need Microphone access. Enable it in Settings → MausBot." + const val AUDIO_FAILED_MESSAGE = "Could not connect the call audio." + /** The answer went in, but the audio never connected in `LiveCallManager.MEDIA_CONNECT_TIMEOUT_MS` (the desktop's `call.live.droppedNoAudio`). */ + const val AUDIO_TIMEOUT_MESSAGE = "Call dropped: the audio could not connect." + const val START_FAILED_MESSAGE = "Could not start the call." + /** The 409 names this phone's own previous call: the computer is still winding it down (up to 5 s). */ + const val LAST_CALL_ENDING = "Your last call is still ending on your computer. Try again in a moment." + const val FOCUS_LOST_MESSAGE = "Call ended: another app took the audio." + /** + * The pairing went (unpaired, signed out, the token refused) and the call + * was hung up at once: the desktop's words for the `signed-out` end reason. + */ + const val SIGNED_OUT_MESSAGE = "Call ended: you were signed out." + const val CONNECTING = "Connecting…" + /** After Hang up, until the computer confirms the end. */ + const val HANGING_UP = "Hanging up…" + /** The desktop's `call.live.ended`: an end with no more particular reason. */ + const val CALL_ENDED = "Call ended." + /** The desktop's `call.live.dropped`. */ + const val CALL_DROPPED = "Call dropped." + const val VOICE_APPLIES_NEXT_CALL = "Takes effect on the next call." + /** Under the profile sheet's disabled "Preview voice": a preview would take the call's audio. */ + const val PREVIEW_DURING_CALL = "Voice preview is off during a Live call." + /** Under a voice note's disabled play button: the note would take the call's audio, which ends the call. */ + const val VOICE_NOTE_DURING_CALL = "Voice notes can't play during a Live call." + + /** Where Live is set up on a phone (its settings sheet): what a call sends to OpenAI, and where the key stays. */ + const val DISCLOSURE = + "A Live call sends your voice to OpenAI, along with the chat's recent messages, the bot's answers " + + "and the details of any approval it asks for. The OpenAI key stays on your computer." + + /** The first-call disclosure's button that starts the call (with Cancel beside it). */ + const val START_CALL = "Start call" + + /** Under "Read replies to typed messages": what turning it off keeps from OpenAI. */ + const val TYPED_REPLIES_FOOTER = "When this is off, messages you type during a call and the bot's answers to them are not sent to OpenAI." + + /** + * The desktop's `LIVE_VOICE_OPTIONS` (`src/lib/live-call.ts`), copied: the + * harness sends no list, and the two apps must offer the same voices. + */ + val VOICE_OPTIONS: List = listOf( + LiveVoiceOption("marin", "Marin (default)"), + LiveVoiceOption("cedar", "Cedar"), + LiveVoiceOption("alloy", "Alloy"), + LiveVoiceOption("ash", "Ash"), + LiveVoiceOption("ballad", "Ballad"), + LiveVoiceOption("coral", "Coral"), + LiveVoiceOption("echo", "Echo"), + LiveVoiceOption("sage", "Sage"), + LiveVoiceOption("shimmer", "Shimmer"), + LiveVoiceOption("verse", "Verse"), + LiveVoiceOption("gleam", "Gleam — North American, feminine"), + LiveVoiceOption("meridian", "Meridian — North American, masculine"), + LiveVoiceOption("quartz", "Quartz — Australian, feminine"), + LiveVoiceOption("ripple", "Ripple — Australian, masculine"), + LiveVoiceOption("vesper", "Vesper — British, masculine"), + LiveVoiceOption("willow", "Willow — Irish, feminine"), + LiveVoiceOption("stone", "Stone — Irish, masculine"), + LiveVoiceOption("delta", "Delta — Southern U.S., feminine"), + LiveVoiceOption("cinder", "Cinder — Southern U.S., masculine"), + LiveVoiceOption("beacon", "Beacon — Filipino, masculine"), + LiveVoiceOption("bossa", "Bossa — Brazilian Portuguese, feminine"), + LiveVoiceOption("tempo", "Tempo — Brazilian Portuguese, masculine"), + ) + + /** Idle hang-up choices, in minutes: the same list on every client. The harness allows 1–60 and defaults to 5. */ + val IDLE_PRESETS: List = listOf(1, 2, 3, 5, 10, 15, 30, 60) + + fun voiceLabel(id: String): String = + VOICE_OPTIONS.firstOrNull { it.id == id }?.label ?: id.ifBlank { VOICE_OPTIONS.first().label } + + /** The presets plus whatever another client set, so a value like 7 is shown and stays selected (as on the desktop). */ + fun idleChoices(current: Int): List = (IDLE_PRESETS + current).filter { it in 1..60 }.distinct().sorted() + + /** "m:ss", or "h:mm:ss" past an hour. A start ahead of the phone's clock shows 0:00, never a minus. */ + fun elapsed(startedAtMs: Long, nowMs: Long): String { + val total = ((nowMs - startedAtMs) / 1000).coerceAtLeast(0) + val hours = total / 3600 + val minutes = (total % 3600) / 60 + val seconds = total % 60 + // Locale.ROOT: the clock keeps Western digits on every device, as RoutineRules does. + return if (hours > 0) { + String.format(Locale.ROOT, "%d:%02d:%02d", hours, minutes, seconds) + } else { + String.format(Locale.ROOT, "%d:%02d", minutes, seconds) + } + } + + /** "Live with Ada · 1:05", the banner's line. The bars draw it in its two parts, [liveWith] and [clockSuffix]. */ + fun title(botName: String, elapsed: String): String = liveWith(botName) + clockSuffix(elapsed) + + /** "Live with Ada": the part of a bar's line that gives way when the bot's name is too long for it. */ + fun liveWith(botName: String): String = "Live with $botName" + + /** " · 1:05": the part of a bar's line after [liveWith] that always shows. */ + fun clockSuffix(elapsed: String): String = " · $elapsed" + + fun clientLabel(client: String): String = when (client) { + "desktop" -> "your computer" + "ios" -> "an iPhone" + "android" -> "another phone" + else -> "another device" + } + + /** + * The remote bar's second line: where the call is. The desktop's remote + * bar says it the same way ("Pepper is on a Live call from an iPhone"), + * and so does the iPhone's. + */ + fun fromDevice(client: String): String = "From ${clientLabel(client)}" + + /** The 409 `activeCall` wording: who is on the line. */ + fun busyMessage(active: LiveCallState): String = + "A Live call is already running from ${clientLabel(active.client)}. Hang up there first." + + /** [dropped]: a connection lost on the way, not an end someone chose (the desktop's and the iPhone's classification). */ + data class EndNotice(val text: String, val dropped: Boolean) + + /** + * The desktop's `endNotice` (`src/lib/live-call-media.ts`) for every + * harness `LiveEndReason`, and OpenAI's own `session.closed` reasons on + * this phone's data channel mapped the way the harness maps them + * (`CLOSE_REASONS` in `server/live-call-controller.ts`). + */ + fun endNotice(reason: String?): EndNotice = when (reason) { + "idle" -> EndNotice("Call ended after a long silence.", dropped = false) + "expired" -> EndNotice("Call ended: it reached OpenAI's time limit.", dropped = false) + "content" -> EndNotice("OpenAI ended the call under its content rules.", dropped = false) + "deleted" -> EndNotice("Call ended: the chat was deleted.", dropped = false) + "shutdown" -> EndNotice("Call ended: OpenMausBot restarted.", dropped = false) + "signed-out" -> EndNotice(SIGNED_OUT_MESSAGE, dropped = false) + "remote-hangup", "remote_hangup", "connection-lost", "connection_lost", "sideband-lost", "error" -> + EndNotice(CALL_DROPPED, dropped = true) + // hung-up, close_requested, null, and whatever the harness adds next + else -> EndNotice(CALL_ENDED, dropped = false) + } + + /** + * Whether a chat offers the phone button. Not while this phone is on a + * call, or hanging one up (the bar has the controls), and not while the + * computer reports a call running from any device: a start would only be + * refused as busy, and that device has to hang up first. + */ + fun offersCall(local: LiveCallSnapshot, server: LiveCallState?): Boolean = + !local.active && server?.isRunning != true + + /** + * Which bar a chat shows. This phone's call wins; otherwise a running call + * the computer reports on this chat is shown as remote, with Hang up only. + * + * A call with this phone's own [LiveCallSnapshot.callId] is never remote, + * and neither is a call that is `ending`: a hang-up is briefly still + * ending on the computer after its own bar has gone (the desktop's and + * the iPhone's rule), and without these checks that echo would come back + * as a call "on another phone". + */ + fun barModel(local: LiveCallSnapshot, server: LiveCallState?, threadId: String, botName: String, nowMs: Long): LiveCallBarModel { + if (local.concerns(threadId)) { + val name = local.botName.ifBlank { botName } + val title = when (local.phase) { + LiveCallPhase.STARTING -> CONNECTING + LiveCallPhase.LIVE -> liveWith(name) + LiveCallPhase.ENDING -> HANGING_UP + LiveCallPhase.ENDED -> local.notice ?: CALL_ENDED + LiveCallPhase.IDLE -> "" + } + val clock = if (local.phase == LiveCallPhase.LIVE) elapsed(local.liveSince ?: nowMs, nowMs) else null + return LiveCallBarModel.Local(title, local.caption, local.heard, local.muted, local.speaker, local.phase, local.canRetry, clock) + } + val remote = server + ?.takeIf { + it.isRunning && it.status != LiveCallStatus.ENDING && it.threadId == threadId && it.callId != local.callId + } + ?: return LiveCallBarModel.Hidden + return LiveCallBarModel.Remote( + title = liveWith(botName), + clock = elapsed(remote.startedAt.toLong(), nowMs), + device = fromDevice(remote.client), + callId = remote.callId, + ) + } +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallSettingsSheet.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallSettingsSheet.kt new file mode 100644 index 0000000000..c66da581b5 --- /dev/null +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/LiveCallSettingsSheet.kt @@ -0,0 +1,230 @@ +package com.openmausbot.companion.ui + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.selection.selectable +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.ModalBottomSheet +import androidx.compose.material3.RadioButton +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.material3.rememberModalBottomSheetState +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import com.openmausbot.companion.core.LiveSettings +import com.openmausbot.companion.core.LiveSettingsPatch +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.launch + +/** + * The gear's sheet: voice, sound output, read typed replies, idle hang-up. + * Every change is written to the computer as it is made — the settings are + * the computer's, shared with the desktop and the iPhone; the sound output is + * this phone's own. The OpenAI key is not here: phones can neither read nor + * write it. The sheet opens by saying what a call sends to OpenAI. + */ +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun LiveCallSettingsSheet(onDismiss: () -> Unit) { + val environment = LocalCompanion.current + val session = environment.session + val liveCalls = environment.liveCalls + val scope = rememberCoroutineScope() + val local by liveCalls.state.collectAsState() + var settings by remember { mutableStateOf(null) } + var error by remember { mutableStateOf(null) } + var saving by remember { mutableStateOf(false) } + + LaunchedEffect(Unit) { + val loaded = session.liveSettings() + settings = loaded ?: LiveSettings() + if (loaded == null) error = LOAD_FAILED + } + + fun save(patch: LiveSettingsPatch) { + scope.launch { + saving = true + try { + settings = session.updateLiveSettings(patch) + error = null + } catch (failure: CancellationException) { + throw failure + } catch (failure: Exception) { + error = failure.message?.takeIf { it.isNotBlank() } ?: SAVE_FAILED + } finally { + saving = false + } + } + } + + val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) + ModalBottomSheet(onDismissRequest = onDismiss, sheetState = sheetState) { + LiveCallSettingsForm( + settings = settings, + speaker = local.speaker, + saving = saving, + error = error, + onDone = onDismiss, + onVoice = { save(LiveSettingsPatch(voice = it)) }, + // Sound output is this phone's, not the computer's: no request. + onSpeaker = liveCalls::setSpeaker, + onReadTypedReplies = { save(LiveSettingsPatch(readTypedReplies = it)) }, + onIdleMinutes = { save(LiveSettingsPatch(idleMinutes = it)) }, + ) + } +} + +/** + * The sheet's content, kept apart from the sheet so a test can mount it without a bottom sheet. + * + * Done and the title stay put while the rows scroll: the voice list alone is + * twenty-two rows, and Done should not be a scroll back to the top away. + */ +@Composable +internal fun LiveCallSettingsForm( + settings: LiveSettings?, + speaker: Boolean, + saving: Boolean, + error: String?, + onDone: () -> Unit, + onVoice: (String) -> Unit, + onSpeaker: (Boolean) -> Unit, + onReadTypedReplies: (Boolean) -> Unit, + onIdleMinutes: (Int) -> Unit, +) { + Column(verticalArrangement = Arrangement.spacedBy(18.dp)) { + Box(modifier = Modifier.fillMaxWidth().padding(horizontal = 4.dp)) { + TextButton(onClick = onDone, modifier = Modifier.align(Alignment.CenterStart)) { Text("Done") } + Text( + text = "Live call settings", + fontSize = 17.sp, + fontWeight = FontWeight.SemiBold, + modifier = Modifier.align(Alignment.Center), + ) + } + Column( + modifier = Modifier + .weight(1f, fill = false) + .verticalScroll(rememberScrollState()) + .padding(bottom = 24.dp), + verticalArrangement = Arrangement.spacedBy(18.dp), + ) { + // First, before any choice: what a call sends to OpenAI, and that the key stays home. + Text( + text = LiveCallRules.DISCLOSURE, + fontSize = 13.sp, + color = secondaryTint, + modifier = Modifier.padding(horizontal = 20.dp), + ) + if (settings == null) { + Text( + text = "Loading…", + fontSize = 15.sp, + color = secondaryTint, + modifier = Modifier.padding(horizontal = 20.dp), + ) + return@Column + } + // The route gives a connected headset the call over this choice (`LiveCallAudioRouting`). + FormSection(header = "Sound output", footer = "A connected headset takes the call instead.") { + ChoiceRow(label = "Speaker", selected = speaker, onSelect = { onSpeaker(true) }) + ChoiceRow(label = "Earpiece", selected = !speaker, onSelect = { onSpeaker(false) }) + } + FormSection(header = "Voice", footer = LiveCallRules.VOICE_APPLIES_NEXT_CALL) { + val current = settings.voice.ifBlank { LiveCallRules.VOICE_OPTIONS.first().id } + LiveCallRules.VOICE_OPTIONS.forEach { option -> + ChoiceRow( + label = option.label, + selected = option.id == current, + enabled = !saving, + onSelect = { onVoice(option.id) }, + ) + } + } + FormSection(header = "During a call", footer = LiveCallRules.TYPED_REPLIES_FOOTER) { + SwitchRow( + label = "Read replies to typed messages", + checked = settings.readTypedReplies, + enabled = !saving, + onCheckedChange = onReadTypedReplies, + ) + } + FormSection( + header = "Hang up after silence", + footer = "Minutes without speech before the call ends on its own.", + ) { + LiveCallRules.idleChoices(settings.idleMinutes).forEach { minutes -> + ChoiceRow( + label = if (minutes == 1) "1 minute" else "$minutes minutes", + selected = minutes == settings.idleMinutes, + enabled = !saving, + onSelect = { onIdleMinutes(minutes) }, + ) + } + } + error?.let { + Text( + text = it, + fontSize = 13.sp, + color = MaterialTheme.colorScheme.error, + modifier = Modifier.padding(horizontal = 20.dp), + ) + } + } + } +} + +/** A radio row, as `SettingsScreen` draws the activity-detail picker: the whole line selects. */ +@Composable +private fun ChoiceRow(label: String, selected: Boolean, enabled: Boolean = true, onSelect: () -> Unit) { + Row( + modifier = Modifier + .fillMaxWidth() + .heightIn(min = MIN_TOUCH_TARGET) + .selectable(selected = selected, enabled = enabled, role = Role.RadioButton, onClick = onSelect), + horizontalArrangement = Arrangement.spacedBy(10.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + RadioButton(selected = selected, onClick = null, enabled = enabled) + Text(text = label, fontSize = 15.sp) + } +} + +private const val LOAD_FAILED = "Could not read the Live settings from your computer." +private const val SAVE_FAILED = "Could not save that setting." + +/** + * Before this phone's first Live call: what a call sends to OpenAI (the + * settings sheet's sentence), with Start call and Cancel. A phone has no Live + * switch, so its first call is where Live is turned on. + */ +@Composable +internal fun LiveCallDisclosureDialog(onStart: () -> Unit, onCancel: () -> Unit) { + AlertDialog( + onDismissRequest = onCancel, + text = { Text(LiveCallRules.DISCLOSURE) }, + confirmButton = { TextButton(onClick = onStart) { Text(LiveCallRules.START_CALL) } }, + dismissButton = { TextButton(onClick = onCancel) { Text("Cancel") } }, + ) +} diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/MessageRow.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/MessageRow.kt index 2afc2c5a98..73b7aa2e61 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/MessageRow.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/MessageRow.kt @@ -82,6 +82,7 @@ import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp +import com.openmausbot.companion.audio.VoiceNoteController import com.openmausbot.companion.core.Chat import com.openmausbot.companion.core.AttachedMessageContent import com.openmausbot.companion.core.generatedImages @@ -185,6 +186,11 @@ fun MessageRow( ) } + // A request the person spoke on a Live call; the harness labels it. + if (mine && message.via == "call") { + Text(text = "via call", fontSize = 12.sp, color = secondaryTint) + } + message.reactions?.takeIf { it.isNotEmpty() }?.let { reactions -> Row(horizontalArrangement = Arrangement.spacedBy(6.dp)) { Reactions.group(reactions).forEach { group -> @@ -743,6 +749,10 @@ private fun voiceNoteClock(ms: Long): String { * rather than talking over it. The clip's bytes are fetched through the same * authenticated file route as image thumbnails, but only on first play — a * note nobody opens costs no request, and a replay never refetches. + * + * While this phone is on a Live call the play button is off, with the reason + * under the bubble: a note asks for the audio focus the call holds, and the + * call ends when it loses it (as the profile sheet keeps its voice preview off). */ @Composable private fun VoiceNoteAttachmentView( @@ -753,6 +763,8 @@ private fun VoiceNoteAttachmentView( val foreground = if (message.role == Message.Role.USER) BubbleColor.mineText else MaterialTheme.colorScheme.onSurface val session = LocalCompanion.current.session val player = LocalCompanion.current.voiceNotes + val liveCall by LocalCompanion.current.liveCalls.state.collectAsState() + val callHoldsAudio = liveCall.holdsMedia val scope = rememberCoroutineScope() val key = remember(message.id, note.path) { message.id + ":" + note.path } var clip by remember(message.id, note.path) { mutableStateOf(VoiceNoteClipState.NotLoaded) } @@ -760,7 +772,10 @@ private fun VoiceNoteAttachmentView( var scrub by remember(key) { mutableStateOf(null) } fun startPlayback(data: ByteArray) { - if (player.play(key, data) != null) clip = VoiceNoteClipState.Failed + val failure = player.play(key, data) ?: return + // A Live call took the audio while the clip downloaded: the player + // refused it, and the clip waits, ready, for the call to end. + if (failure != VoiceNoteController.DURING_LIVE_CALL) clip = VoiceNoteClipState.Failed } fun loadAndPlay() { @@ -813,74 +828,89 @@ private fun VoiceNoteAttachmentView( val durationSeconds = durationMs?.let { it / 1000f } ?: 0f val positionMs = scrub?.toLong() ?: (active?.positionMs ?: 0L) - Row( - modifier = Modifier - .widthIn(max = 360.dp) - .clip(RoundedCornerShape(16.dp)) - .background(foreground.copy(alpha = 0.10f)) - .padding(horizontal = 12.dp, vertical = 8.dp), - horizontalArrangement = Arrangement.spacedBy(10.dp), - verticalAlignment = Alignment.CenterVertically, - ) { - Box( + // Pausing never takes the audio; starting or resuming would. + val playable = playing || !callHoldsAudio + Column(verticalArrangement = Arrangement.spacedBy(4.dp)) { + Row( modifier = Modifier - .size(28.dp) - .clip(CircleShape) - .background(MaterialTheme.colorScheme.primary) - .clickable(role = Role.Button) { - when { - playing -> player.pause() - clip is VoiceNoteClipState.Loading -> Unit - active != null && player.resumable(key) -> - if (player.resume() != null) clip = VoiceNoteClipState.Failed - clip is VoiceNoteClipState.Ready -> - startPlayback((clip as VoiceNoteClipState.Ready).data) - else -> loadAndPlay() - } - } - .semantics { - contentDescription = if (playing) "Pause voice note" else "Play voice note" - }, - contentAlignment = Alignment.Center, + .widthIn(max = 360.dp) + .clip(RoundedCornerShape(16.dp)) + .background(foreground.copy(alpha = 0.10f)) + .padding(horizontal = 12.dp, vertical = 8.dp), + horizontalArrangement = Arrangement.spacedBy(10.dp), + verticalAlignment = Alignment.CenterVertically, ) { - when { - clip is VoiceNoteClipState.Loading && active == null -> - CircularProgressIndicator( - modifier = Modifier.size(14.dp), - strokeWidth = 2.dp, - color = Color.White, + Box( + modifier = Modifier + .size(28.dp) + .clip(CircleShape) + .background(MaterialTheme.colorScheme.primary.copy(alpha = if (playable) 1f else 0.38f)) + .clickable(role = Role.Button, enabled = playable) { + when { + playing -> player.pause() + // Disabled is how it looks; this is what stops a tap + // that reaches the click action anyway. + callHoldsAudio -> Unit + clip is VoiceNoteClipState.Loading -> Unit + active != null && player.resumable(key) -> + player.resume()?.let { if (it != VoiceNoteController.DURING_LIVE_CALL) clip = VoiceNoteClipState.Failed } + clip is VoiceNoteClipState.Ready -> + startPlayback((clip as VoiceNoteClipState.Ready).data) + else -> loadAndPlay() + } + } + .semantics { + contentDescription = if (playing) "Pause voice note" else "Play voice note" + }, + contentAlignment = Alignment.Center, + ) { + when { + clip is VoiceNoteClipState.Loading && active == null -> + CircularProgressIndicator( + modifier = Modifier.size(14.dp), + strokeWidth = 2.dp, + color = Color.White, + ) + playing -> VoiceNotePauseGlyph(Color.White) + else -> Icon( + imageVector = Icons.Filled.PlayArrow, + contentDescription = null, + tint = Color.White, + modifier = Modifier.size(20.dp), ) - playing -> VoiceNotePauseGlyph(Color.White) - else -> Icon( - imageVector = Icons.Filled.PlayArrow, - contentDescription = null, - tint = Color.White, - modifier = Modifier.size(20.dp), - ) + } } + Slider( + // The slider works in seconds; without an explicit range Compose clamps + // it to 0f..1f and scrubs can only land inside the first second. + value = if (durationSeconds > 0f) (positionMs / 1000f).coerceIn(0f, durationSeconds) else 0f, + valueRange = if (durationSeconds > 0f) 0f..durationSeconds else 0f..1f, + onValueChange = { scrub = it * 1000f }, + onValueChangeFinished = { + val target = scrub + scrub = null + if (target != null && active != null) player.seek(key, target.toLong()) + }, + // Like the desktop range input: no scrubbing until the length is known. + enabled = active != null && durationMs != null, + modifier = Modifier + .weight(1f) + .semantics { contentDescription = "Seek voice note" }, + ) + Text( + voiceNoteClock(positionMs) + " / " + (durationMs?.let(::voiceNoteClock) ?: "--:--"), + fontSize = 11.sp, + color = foreground.copy(alpha = 0.80f), + ) + } + if (!playable) { + Text( + LiveCallRules.VOICE_NOTE_DURING_CALL, + fontSize = 11.sp, + color = foreground.copy(alpha = 0.80f), + modifier = Modifier.padding(horizontal = 12.dp), + ) } - Slider( - // The slider works in seconds; without an explicit range Compose clamps - // it to 0f..1f and scrubs can only land inside the first second. - value = if (durationSeconds > 0f) (positionMs / 1000f).coerceIn(0f, durationSeconds) else 0f, - valueRange = if (durationSeconds > 0f) 0f..durationSeconds else 0f..1f, - onValueChange = { scrub = it * 1000f }, - onValueChangeFinished = { - val target = scrub - scrub = null - if (target != null && active != null) player.seek(key, target.toLong()) - }, - // Like the desktop range input: no scrubbing until the length is known. - enabled = active != null && durationMs != null, - modifier = Modifier - .weight(1f) - .semantics { contentDescription = "Seek voice note" }, - ) - Text( - voiceNoteClock(positionMs) + " / " + (durationMs?.let(::voiceNoteClock) ?: "--:--"), - fontSize = 11.sp, - color = foreground.copy(alpha = 0.80f), - ) } } diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/RootScreen.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/RootScreen.kt index a23d933cce..1366cb5464 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/RootScreen.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/RootScreen.kt @@ -5,6 +5,7 @@ import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.safeDrawingPadding import androidx.compose.material3.AlertDialog @@ -361,43 +362,50 @@ private fun PairedScreen( BackHandler(enabled = navigator.canGoBack) { navigator.pop() } - when (val destination = navigator.current) { - Destination.Roster -> RosterScreen(navigator) - Destination.Settings -> SettingsScreen( - onBack = navigator::pop, - onOpenRoutines = { navigator.push(Destination.Routines) }, - onOpenConnectedApps = { navigator.push(Destination.ConnectedApps) }, - ) - Destination.Routines -> TasksRoutinesScreen( - onBack = navigator::pop, - // A receipt's "Open task" pushes the chat above this screen, the way - // iOS appends it to the same navigation path. - onOpenChat = navigator::open, - ) - Destination.ConnectedApps -> ConnectedAppsScreen(onBack = navigator::pop) - // One branch for both shapes of chat address, so a notification's thread - // becoming an addressed chat re-reads the same screen instead of - // rebuilding it. - is Destination.Conversation -> ChatScreen( - destination = destination, - onResolved = { target -> - navigator.selectTask(destination, target) - }, - onBack = navigator::pop, - onOpenComputer = { navigator.push(Destination.Computer(it)) }, - onOpenOverview = { navigator.push(Destination.Overview(it)) }, - // Push Computer keeps the chat under the top; pop to roster does not. - retainsDraft = navigator::retainsChatDraft, - onOpenChat = navigator::open, - ) - is Destination.Computer -> ComputerScreen( - botId = destination.botId, - onBack = navigator::pop, - ) - is Destination.Overview -> BotOverviewScreen( - botId = destination.botId, - onBack = navigator::pop, - ) + // The banner follows a call to every screen but its own chat, and pushes + // whatever is on screen down by its height rather than covering it. + Column(modifier = Modifier.fillMaxSize()) { + LiveCallBannerHost(navigator) + Box(modifier = Modifier.weight(1f).fillMaxWidth()) { + when (val destination = navigator.current) { + Destination.Roster -> RosterScreen(navigator) + Destination.Settings -> SettingsScreen( + onBack = navigator::pop, + onOpenRoutines = { navigator.push(Destination.Routines) }, + onOpenConnectedApps = { navigator.push(Destination.ConnectedApps) }, + ) + Destination.Routines -> TasksRoutinesScreen( + onBack = navigator::pop, + // A receipt's "Open task" pushes the chat above this screen, the way + // iOS appends it to the same navigation path. + onOpenChat = navigator::open, + ) + Destination.ConnectedApps -> ConnectedAppsScreen(onBack = navigator::pop) + // One branch for both shapes of chat address, so a notification's thread + // becoming an addressed chat re-reads the same screen instead of + // rebuilding it. + is Destination.Conversation -> ChatScreen( + destination = destination, + onResolved = { target -> + navigator.selectTask(destination, target) + }, + onBack = navigator::pop, + onOpenComputer = { navigator.push(Destination.Computer(it)) }, + onOpenOverview = { navigator.push(Destination.Overview(it)) }, + // Push Computer keeps the chat under the top; pop to roster does not. + retainsDraft = navigator::retainsChatDraft, + onOpenChat = navigator::open, + ) + is Destination.Computer -> ComputerScreen( + botId = destination.botId, + onBack = navigator::pop, + ) + is Destination.Overview -> BotOverviewScreen( + botId = destination.botId, + onBack = navigator::pop, + ) + } + } } } diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/RoutineRunCardView.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/RoutineRunCardView.kt index 966bf621a5..f6e72c918e 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/RoutineRunCardView.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/RoutineRunCardView.kt @@ -161,7 +161,9 @@ internal fun RoutineRunCardView(message: Message, openRun: (() -> Unit)?) { } run.error?.takeIf { it.isNotBlank() }?.let { error -> - Text(error.trim(), fontSize = 13.sp, color = MaterialTheme.colorScheme.error) + SelectionContainer { + Text(error.trim(), fontSize = 13.sp, color = MaterialTheme.colorScheme.error) + } } } } diff --git a/android/app/src/main/res/drawable/ic_live_hang_up.xml b/android/app/src/main/res/drawable/ic_live_hang_up.xml new file mode 100644 index 0000000000..e6814b9072 --- /dev/null +++ b/android/app/src/main/res/drawable/ic_live_hang_up.xml @@ -0,0 +1,16 @@ + + + + + diff --git a/android/app/src/main/res/drawable/ic_live_mic_off.xml b/android/app/src/main/res/drawable/ic_live_mic_off.xml new file mode 100644 index 0000000000..fa03c035b6 --- /dev/null +++ b/android/app/src/main/res/drawable/ic_live_mic_off.xml @@ -0,0 +1,14 @@ + + + + + diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/LiveCallManifestTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/LiveCallManifestTest.kt new file mode 100644 index 0000000000..b68d22e672 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/LiveCallManifestTest.kt @@ -0,0 +1,45 @@ +package com.openmausbot.companion + +import java.io.File +import javax.xml.parsers.DocumentBuilderFactory +import kotlin.test.Test +import kotlin.test.assertTrue +import org.w3c.dom.Element + +/** + * Live calls need two permissions no compiler checks: RECORD_AUDIO (already + * there for dictation) and MODIFY_AUDIO_SETTINGS, without which + * `AudioManager.mode` and the speaker/earpiece route silently do nothing. + * Pinned from the manifest the way `PairingLinkManifestTest` pins its attributes. + */ +class LiveCallManifestTest { + private val android = "http://schemas.android.com/apk/res/android" + + @Test + fun declaresTheAudioPermissionsACallNeeds() { + val names = permissions() + assertTrue("android.permission.RECORD_AUDIO" in names) + assertTrue("android.permission.MODIFY_AUDIO_SETTINGS" in names) + } + + private fun permissions(): Set { + val document = DocumentBuilderFactory.newInstance() + .apply { isNamespaceAware = true } + .newDocumentBuilder() + .parse(locateManifest()) + val nodes = document.documentElement.getElementsByTagName("uses-permission") + return (0 until nodes.length).map { (nodes.item(it) as Element).getAttributeNS(android, "name") }.toSet() + } + + private fun locateManifest(): File { + var directory: File? = File(".").absoluteFile + while (directory != null) { + for (candidate in listOf("src/main/AndroidManifest.xml", "app/src/main/AndroidManifest.xml")) { + val file = File(directory, candidate) + if (file.isFile) return file + } + directory = directory.parentFile + } + error("could not find AndroidManifest.xml from ${File(".").absolutePath}") + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallAudioGateTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallAudioGateTest.kt new file mode 100644 index 0000000000..1af7efdfb0 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallAudioGateTest.kt @@ -0,0 +1,66 @@ +package com.openmausbot.companion.audio + +import android.app.Application +import android.content.Context +import android.media.AudioManager +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.LifecycleRegistry +import kotlin.test.assertEquals +import kotlin.test.assertNull +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config + +/** + * The players as the app builds them ([com.openmausbot.companion.OpenMausApp]): + * the shared focus gate itself refuses a voice note or a voice preview while + * this phone's Live call holds the audio, before anything asks Android for + * the focus. Whatever reaches the player then (a note whose download + * finishes after the call started, a preview fetched before it) cannot take + * the call's audio and end it. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class LiveCallAudioGateTest { + private val context = RuntimeEnvironment.getApplication() as Application + private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + @Test + fun theGateRefusesVoiceNotesAndPreviewsWhileACallHoldsTheAudio() { + val notes = VoiceNotePlayer(context, idleLifecycle(), liveCallHoldsAudio = { true }) + assertEquals(VoiceNoteController.DURING_LIVE_CALL, notes.play("m1:/attachments/note.mp3", ByteArray(64))) + assertNull(notes.playback.value) + assertNull(shadowOf(audioManager).lastAudioFocusRequest, "nothing asked Android for the focus") + + val previews = VoicePreviewPlayer(context, idleLifecycle(), liveCallHoldsAudio = { true }) + assertEquals(VoicePreviewController.DURING_LIVE_CALL, previews.play(ByteArray(64))) + assertEquals(false, previews.playing.value) + assertNull(shadowOf(audioManager).lastAudioFocusRequest, "nothing asked Android for the focus") + } + + @Test + fun endingWithoutAudioOwnershipDoesNotChangeAnotherCallsRoute() { + audioManager.mode = AudioManager.MODE_IN_CALL + @Suppress("DEPRECATION") + audioManager.isSpeakerphoneOn = true + AndroidLiveCallAudioRoute(context).end() + assertEquals(AudioManager.MODE_IN_CALL, audioManager.mode) + @Suppress("DEPRECATION") + assertEquals(true, audioManager.isSpeakerphoneOn) + } + + private fun idleLifecycle(): Lifecycle { + val owner = object : LifecycleOwner { + lateinit var registry: LifecycleRegistry + override val lifecycle: Lifecycle get() = registry + } + val registry = LifecycleRegistry.createUnsafe(owner) + owner.registry = registry + registry.currentState = Lifecycle.State.STARTED + return registry + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallAudioRoutingTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallAudioRoutingTest.kt new file mode 100644 index 0000000000..0a81e24d6c --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallAudioRoutingTest.kt @@ -0,0 +1,96 @@ +package com.openmausbot.companion.audio + +import android.media.AudioDeviceInfo.TYPE_BLE_HEADSET +import android.media.AudioDeviceInfo.TYPE_BLE_SPEAKER +import android.media.AudioDeviceInfo.TYPE_BLUETOOTH_A2DP +import android.media.AudioDeviceInfo.TYPE_BLUETOOTH_SCO +import android.media.AudioDeviceInfo.TYPE_BUILTIN_EARPIECE +import android.media.AudioDeviceInfo.TYPE_BUILTIN_SPEAKER +import android.media.AudioDeviceInfo.TYPE_HDMI +import android.media.AudioDeviceInfo.TYPE_HEARING_AID +import android.media.AudioDeviceInfo.TYPE_LINE_ANALOG +import android.media.AudioDeviceInfo.TYPE_USB_DEVICE +import android.media.AudioDeviceInfo.TYPE_USB_HEADSET +import android.media.AudioDeviceInfo.TYPE_WIRED_HEADPHONES +import android.media.AudioDeviceInfo.TYPE_WIRED_HEADSET +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Where a Live call's voice goes. The speaker is on by default (plan Ruling 8), + * so without the headset rule a person wearing earbuds would hear the bot's + * replies, which may be private, out loud on the loudspeaker. + */ +class LiveCallAudioRoutingTest { + private val phone = listOf(TYPE_BUILTIN_EARPIECE, TYPE_BUILTIN_SPEAKER) + + @Test + fun `with no headset the speaker setting picks the loudspeaker or the earpiece`() { + assertEquals(TYPE_BUILTIN_SPEAKER, LiveCallAudioRouting.communicationDevice(phone, speaker = true)) + assertEquals(TYPE_BUILTIN_EARPIECE, LiveCallAudioRouting.communicationDevice(phone, speaker = false)) + } + + @Test + fun `a connected headset takes the call whatever the speaker setting`() { + for (headset in listOf( + TYPE_WIRED_HEADSET, + TYPE_WIRED_HEADPHONES, + TYPE_USB_HEADSET, + TYPE_BLE_HEADSET, + TYPE_BLUETOOTH_SCO, + TYPE_HEARING_AID, + )) { + for (speaker in listOf(true, false)) { + assertEquals( + headset, + LiveCallAudioRouting.communicationDevice(phone + headset, speaker), + "headset type $headset, speaker=$speaker", + ) + } + } + } + + @Test + fun `a hearing aid comes first, then a plug, then Bluetooth`() { + val bluetooth = listOf(TYPE_BLUETOOTH_SCO, TYPE_BLE_HEADSET) + val everything = phone + bluetooth + listOf(TYPE_USB_HEADSET, TYPE_WIRED_HEADSET, TYPE_HEARING_AID) + assertEquals(TYPE_HEARING_AID, LiveCallAudioRouting.communicationDevice(everything, speaker = true)) + assertEquals( + TYPE_WIRED_HEADSET, + LiveCallAudioRouting.communicationDevice(everything - TYPE_HEARING_AID, speaker = true), + ) + assertEquals(TYPE_BLE_HEADSET, LiveCallAudioRouting.communicationDevice(phone + bluetooth, speaker = true)) + } + + @Test + fun `speakers, docks and screens are not headsets`() { + val shared = phone + listOf(TYPE_BLE_SPEAKER, TYPE_USB_DEVICE, TYPE_HDMI, TYPE_LINE_ANALOG) + assertEquals(TYPE_BUILTIN_SPEAKER, LiveCallAudioRouting.communicationDevice(shared, speaker = true)) + assertEquals(TYPE_BUILTIN_EARPIECE, LiveCallAudioRouting.communicationDevice(shared, speaker = false)) + } + + @Test + fun `a missing built-in device leaves the choice to the system`() { + // A tablet has no earpiece. + assertNull(LiveCallAudioRouting.communicationDevice(listOf(TYPE_BUILTIN_SPEAKER), speaker = false)) + assertNull(LiveCallAudioRouting.communicationDevice(emptyList(), speaker = true)) + } + + @Test + fun `below API 31 connected headphones and hearing aids keep private replies off the loudspeaker`() { + for (headset in listOf( + TYPE_WIRED_HEADSET, TYPE_WIRED_HEADPHONES, TYPE_USB_HEADSET, + TYPE_HEARING_AID, TYPE_BLUETOOTH_SCO, TYPE_BLUETOOTH_A2DP, + )) { + assertFalse(LiveCallAudioRouting.speakerphone(phone + headset, speaker = true), "headset type $headset") + } + assertTrue(LiveCallAudioRouting.speakerphone(phone, speaker = true)) + assertFalse(LiveCallAudioRouting.speakerphone(phone, speaker = false)) + // Without SCO routing on these APIs, use the earpiece, never the loudspeaker. + val bluetooth = listOf(TYPE_BLUETOOTH_SCO, TYPE_BLUETOOTH_A2DP) + assertFalse(LiveCallAudioRouting.speakerphone(phone + bluetooth, speaker = true)) + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallManagerTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallManagerTest.kt new file mode 100644 index 0000000000..4cf4dc419e --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallManagerTest.kt @@ -0,0 +1,1143 @@ +package com.openmausbot.companion.audio + +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.LifecycleRegistry +import com.openmausbot.companion.core.APIError +import com.openmausbot.companion.core.LiveCallStart +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.LiveCallStatus +import com.openmausbot.companion.ui.LiveCallRules +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotSame +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.currentTime +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest + +/** + * The state machine behind the call bar, with the WebRTC transport, the audio + * route, the phone's remembered choices and the computer all faked. The rules + * it pins come from the spec's error table, its lifecycle decisions and the + * shared contract rulings; nothing here loads `org.webrtc`. + * + * Driven on a TestScope: the manager launches everything on its scope, so + * `runCurrent()` after each action is the "let the main thread run" of a test, + * and the manager's clock is the test's virtual clock. + */ +@OptIn(ExperimentalCoroutinesApi::class) +class LiveCallManagerTest { + private val call = LiveCallState("c1", "b1", "t1", "android", "marin", 5_000.0, LiveCallStatus.CONNECTING) + private val grant = MicrophoneAccess { it(true) } + private val deny = MicrophoneAccess { it(false) } + + // ------------------------------------------------------------ going live + + @Test + fun `start asks for the microphone, posts the offer and says Connecting until the computer and the channel are both ready`() = runTest { + val f = Fixture(this) + f.manager.start("b1", "t1", "Ada", grant) + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase) + runCurrent() + + assertEquals(listOf(Triple("b1", "t1", FakeTransport.OFFER)), f.api.starts) + assertEquals(FakeApi.ANSWER, f.transport.accepted) + var state = f.manager.state.value + assertEquals(LiveCallPhase.STARTING, state.phase, "answered, but the computer has not attached and the channel is closed") + assertEquals("c1", state.callId) + assertNull(state.liveSince) + assertEquals(listOf(true), f.audio.begins) + assertTrue(state.active) + assertEquals(false, f.transport.muted, "the track starts unmuted") + + // The computer attaches its side first: still connecting, the channel is not open. + advanceTimeBy(1_500) + f.api.serverCall.value = call.copy(status = LiveCallStatus.LIVE) + runCurrent() + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase) + + // The channel opens: live, and the clock counts from now, on this phone's clock. + advanceTimeBy(500) + requireNotNull(f.transport.listener).onChannelOpen() + runCurrent() + state = f.manager.state.value + assertEquals(LiveCallPhase.LIVE, state.phase) + assertEquals(currentTime, state.liveSince, "not the computer's startedAt (5000), not the 201") + assertEquals("Ada", state.botName) + } + + @Test + fun `a channel that opens first still waits for the computer to attach`() = runTest { + val f = Fixture(this) + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + requireNotNull(f.transport.listener).onChannelOpen() + runCurrent() + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase) + + f.api.serverCall.value = call.copy(status = LiveCallStatus.CONNECTING) + runCurrent() + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase, "connecting is not attached") + + f.api.serverCall.value = call.copy(status = LiveCallStatus.LIVE) + runCurrent() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + assertEquals(currentTime, f.manager.state.value.liveSince) + } + + @Test + fun `a status this build does not know counts as attached`() = runTest { + val f = Fixture(this) + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + requireNotNull(f.transport.listener).onChannelOpen() + f.api.serverCall.value = call.copy(status = LiveCallStatus.UNKNOWN) + runCurrent() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase, "an unknown status is a call still running") + } + + @Test + fun `a 201 that says live counts as the computer's word`() = runTest { + val f = Fixture(this) + f.api.answer = { LiveCallStart.Started(call.copy(status = LiveCallStatus.LIVE), FakeApi.ANSWER) } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase) + requireNotNull(f.transport.listener).onChannelOpen() + runCurrent() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + } + + // ------------------------------------------------------------ the first call's disclosure + + @Test + fun `a phone that never called owes the disclosure until Start call, then never again`() = runTest { + val preferences = FakePreferences() + val f = Fixture(this, preferences) + assertTrue(f.manager.disclosureDue, "before this phone's first Live call") + f.manager.acceptDisclosure() + assertFalse(f.manager.disclosureDue) + assertTrue(preferences.disclosureShown, "kept on this phone") + assertFalse(Fixture(this, preferences).manager.disclosureDue, "a later launch does not ask again") + assertFalse(Fixture(this, FakePreferences(disclosureShown = true)).manager.disclosureDue) + } + + // ------------------------------------------------------------ failures and Try again + + @Test + fun `a denied microphone ends the attempt with the settings hint, no Try again, and touches no media`() = runTest { + val f = Fixture(this) + f.manager.start("b1", "t1", "Ada", deny) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals(LiveCallRules.MIC_DENIED_MESSAGE, state.notice) + assertFalse(state.canRetry, "a retry cannot help until the person changes the setting") + assertEquals(0, f.transport.offers) + assertTrue(f.audio.begins.isEmpty()) + assertTrue(f.api.starts.isEmpty()) + + f.manager.retry(grant) + runCurrent() + assertTrue(f.api.starts.isEmpty(), "retry does nothing where Try again is not offered") + } + + @Test + fun `needsKey and a busy line end the attempt with the phone wording and no Try again`() = runTest { + val f = Fixture(this) + f.api.answer = { LiveCallStart.NeedsKey("Add an OpenAI API key to use Live calls.") } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertEquals(LiveCallRules.NEEDS_KEY_MESSAGE, f.manager.state.value.notice) + assertFalse(f.manager.state.value.canRetry) + assertEquals(1, f.transport.closes) + assertEquals(1, f.audio.ends) + assertTrue(f.api.ends.isEmpty(), "nothing was started on the computer") + + f.manager.dismiss() + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + + f.api.answer = { LiveCallStart.Busy(call.copy(client = "desktop"), "busy") } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals("A Live call is already running from your computer. Hang up there first.", f.manager.state.value.notice) + assertFalse(f.manager.state.value.canRetry, "the other device has to hang up first") + } + + @Test + fun `a refusal from the computer shows its own words and offers Try again`() = runTest { + val f = Fixture(this) + f.api.answer = { throw APIError.Status(502, "OpenAI refused the call (HTTP 401).") } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertEquals("OpenAI refused the call (HTTP 401).", f.manager.state.value.notice) + assertTrue(f.manager.state.value.canRetry) + + f.manager.dismiss() + f.api.answer = { throw java.io.IOException() } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(LiveCallRules.START_FAILED_MESSAGE, f.manager.state.value.notice, "no words of its own: the generic line") + assertTrue(f.manager.state.value.canRetry) + } + + @Test + fun `a transport that cannot build the offer shows the audio message, not its own words`() = runTest { + val f = Fixture(this) + f.transport.failOffer = IllegalStateException("createPeerConnection returned null") + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals(LiveCallRules.AUDIO_FAILED_MESSAGE, state.notice) + assertTrue(state.canRetry) + assertTrue(f.api.starts.isEmpty(), "no offer, nothing asked of the computer") + assertEquals(1, f.transport.closes) + assertEquals(1, f.audio.ends) + } + + @Test + fun `a rejected answer releases the call on the computer and offers Try again`() = runTest { + val f = Fixture(this) + f.transport.rejectAnswer = true + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals(LiveCallRules.AUDIO_FAILED_MESSAGE, state.notice) + assertTrue(state.canRetry) + assertEquals(listOf("c1"), f.api.ends) + assertEquals(1, f.transport.closes) + assertEquals(1, f.audio.ends) + } + + @Test + fun `start is ignored while a call is starting or running`() = runTest { + val f = Fixture(this) + f.manager.start("b1", "t1", "Ada", grant) + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(1, f.api.starts.size) + f.manager.start("b2", "t2", "Bo", grant) + runCurrent() + assertEquals(1, f.api.starts.size) + assertEquals("t1", f.manager.state.value.threadId) + } + + @Test + fun `retry starts again on the same chat`() = runTest { + val f = Fixture(this) + f.api.answer = { throw APIError.Status(502, "OpenAI refused the call (HTTP 500).") } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + f.api.answer = { LiveCallStart.Started(call, FakeApi.ANSWER) } + f.manager.retry(grant) + runCurrent() + assertEquals(2, f.api.starts.size) + assertEquals(Triple("b1", "t1", FakeTransport.OFFER), f.api.starts[1]) + assertEquals("c1", f.manager.state.value.callId) + f.goLive() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + f.manager.retry(grant) + runCurrent() + assertEquals(2, f.api.starts.size, "retry while live is a no-op") + } + + // ------------------------------------------------------------ during the call + + @Test + fun `captions come from the data channel`() = runTest { + val f = Fixture(this).live() + val listener = requireNotNull(f.transport.listener) + listener.onMessage("""{"type":"session.input_transcript.delta","delta":"what time ","start_ms":1,"end_ms":2}""") + listener.onMessage("""{"type":"session.input_transcript.delta","delta":"is it","start_ms":2,"end_ms":3}""") + runCurrent() + assertEquals("what time is it", f.manager.state.value.heard) + assertEquals("", f.manager.state.value.caption) + listener.onMessage("""{"type":"session.output_transcript.delta","delta":"It is noon.","start_ms":4,"end_ms":5}""") + listener.onMessage("not json") + runCurrent() + assertEquals("It is noon.", f.manager.state.value.caption) + assertEquals("", f.manager.state.value.heard) + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + } + + @Test + fun `speaker routes through the audio route and is remembered on this phone`() = runTest { + val f = Fixture(this) + f.manager.setSpeaker(false) + assertTrue(f.audio.speakers.isEmpty(), "no call, no route to change") + assertEquals(false, f.preferences.speaker, "kept on this phone, not only in memory") + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(listOf(false), f.audio.begins) + f.goLive() + f.manager.hangUp() + runCurrent() + assertEquals(false, f.manager.state.value.speaker, "the settings sheet still shows the earpiece after a hang-up") + + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(listOf(false, false), f.audio.begins) + f.manager.setSpeaker(true) + assertEquals(listOf(true), f.audio.speakers) + assertTrue(f.manager.state.value.speaker) + assertEquals(true, f.preferences.speaker) + + f.manager.setSpeaker(false) + requireNotNull(f.transport.listener).onDropped() + runCurrent() + f.manager.dismiss() + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + assertEquals(false, f.manager.state.value.speaker, "and after a dismissed notice") + } + + @Test + fun `the next launch starts from the speaker choice this phone remembered`() = runTest { + val f = Fixture(this, FakePreferences(speaker = false)) + assertEquals(false, f.manager.state.value.speaker, "the settings sheet shows the remembered choice before any call") + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(listOf(false), f.audio.begins, "and the first call uses it") + } + + @Test + fun `a call that cannot take the audio from another call ends at once`() = runTest { + val f = Fixture(this) + f.audio.refuseFocus = true + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(LiveCallRules.FOCUS_LOST_MESSAGE, f.manager.state.value.notice) + assertEquals(false, f.manager.state.value.holdsMedia) + } + + @Test + fun `losing the audio to another app ends the call`() = runTest { + val f = Fixture(this).live() + requireNotNull(f.audio.onFocusLost).invoke() + runCurrent() + assertEquals(LiveCallRules.FOCUS_LOST_MESSAGE, f.manager.state.value.notice) + assertEquals(listOf("c1"), f.api.ends) + } + + // ------------------------------------------------------------ hanging up + + @Test + fun `mute flips the track, and hang up says Hanging up until the computer confirms the end`() = runTest { + val f = Fixture(this).live() + f.manager.setMuted(true) + assertEquals(true, f.transport.muted) + assertTrue(f.manager.state.value.muted) + + val answer = CompletableDeferred() + f.api.endGate = answer + f.manager.hangUp() + var state = f.manager.state.value + assertEquals(LiveCallPhase.ENDING, state.phase, "the bar says Hanging up… until the computer answers") + assertEquals("c1", state.callId) + assertEquals(1, f.transport.closeSent, "OpenAI is told at once") + assertEquals(1, f.transport.closes) + assertEquals(1, f.audio.ends, "the microphone and the audio route are free at once") + runCurrent() + assertEquals(listOf("c1"), f.api.ends) + assertEquals(LiveCallPhase.ENDING, f.manager.state.value.phase) + + f.manager.hangUp() + runCurrent() + assertEquals(listOf("c1"), f.api.ends, "a second hang up does nothing") + + answer.complete(Unit) + runCurrent() + state = f.manager.state.value + assertEquals(LiveCallPhase.IDLE, state.phase, "the computer answered: the bar goes") + assertEquals("c1", state.callId, "the computer's ending echo of c1 still reads as this phone's") + assertNull(state.notice) + } + + @Test + fun `the computer's ended frame confirms a hang-up too`() = runTest { + val f = Fixture(this).live() + f.api.endGate = CompletableDeferred() + f.manager.hangUp() + runCurrent() + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDING) + runCurrent() + assertEquals(LiveCallPhase.ENDING, f.manager.state.value.phase, "ending is not ended") + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "hung-up") + runCurrent() + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + assertNull(f.manager.state.value.notice, "a hang-up the person asked for needs no notice") + } + + @Test + fun `a hang-up the computer never answers stops saying Hanging up after 8 s`() = runTest { + assertEquals(8_000L, LiveCallManager.END_TIMEOUT_MS) + val f = Fixture(this).live() + f.api.endGate = CompletableDeferred() + f.manager.hangUp() + runCurrent() + advanceTimeBy(LiveCallManager.END_TIMEOUT_MS - 1) + runCurrent() + assertEquals(LiveCallPhase.ENDING, f.manager.state.value.phase) + advanceTimeBy(1) + runCurrent() + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + assertEquals("c1", f.manager.state.value.callId) + } + + @Test + fun `hanging up before the computer named the call leaves at once`() = runTest { + val f = Fixture(this) + val gate = CompletableDeferred() + f.api.gate = gate + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(1, f.api.starts.size, "the offer is on its way to the computer") + + f.manager.hangUp() + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase, "nothing on the computer to wait for") + assertEquals(1, f.transport.closes) + + gate.complete(Unit) + runCurrent() + assertEquals(listOf("c1"), f.api.ends, "the 201 landed on a phone that had given up") + assertNull(f.transport.accepted) + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + assertEquals("c1", f.manager.state.value.callId) + } + + @Test + fun `hanging up while Connecting after the 201 says Hanging up too`() = runTest { + val f = Fixture(this) + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals("c1", f.manager.state.value.callId) + f.api.endGate = CompletableDeferred() + f.manager.hangUp() + assertEquals(LiveCallPhase.ENDING, f.manager.state.value.phase) + runCurrent() + assertEquals(listOf("c1"), f.api.ends) + } + + @Test + fun `hanging up while the offer is being built ends quietly`() = runTest { + val f = Fixture(this) + val gate = CompletableDeferred() + f.transport.offerGate = gate + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(1, f.transport.offers, "the offer is being built") + + f.manager.hangUp() + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + assertEquals(1, f.transport.closes) + + // The closed transport's offer then fails, as the real one does. + gate.complete(Unit) + runCurrent() + assertTrue(f.api.starts.isEmpty(), "nothing reached the computer") + assertNull(f.transport.accepted) + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + assertNull(f.manager.state.value.notice, "the transport's own error is not shown") + } + + @Test + fun `a 409 naming this phone's own ending call says so, not another phone`() = runTest { + val f = Fixture(this).live() + f.manager.hangUp() + runCurrent() + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + // Tapped again while the computer still winds c1 down. + f.api.answer = { LiveCallStart.Busy(call.copy(status = LiveCallStatus.ENDING), "busy") } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertEquals(LiveCallRules.LAST_CALL_ENDING, f.manager.state.value.notice) + assertTrue(f.manager.state.value.canRetry, "in a moment it will work") + assertEquals("c1", f.manager.state.value.callId, "its echo still reads as this phone's call") + + f.manager.retry(grant) + runCurrent() + assertEquals(LiveCallRules.LAST_CALL_ENDING, f.manager.state.value.notice, "and again on Try again") + + f.api.answer = { LiveCallStart.Busy(call.copy(callId = "c2", client = "ios", status = LiveCallStatus.LIVE), "busy") } + f.manager.retry(grant) + runCurrent() + assertEquals("A Live call is already running from an iPhone. Hang up there first.", f.manager.state.value.notice) + assertFalse(f.manager.state.value.canRetry) + } + + // ------------------------------------------------------------ the computer ends it + + @Test + fun `a server frame ending our call drops the media and says why`() = runTest { + val f = Fixture(this).live() + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "sideband-lost") + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals("Call dropped.", state.notice) + assertTrue(state.canRetry) + assertEquals(1, f.transport.closeSent, "the harness cannot close OpenAI without its sideband; the phone does") + assertEquals(1, f.transport.closes) + assertTrue(f.api.ends.isEmpty(), "the computer already knows") + } + + @Test + fun `a call the computer ended because this phone was signed out offers no Try again`() = runTest { + val f = Fixture(this).live() + val unpaired = "The call has ended because the phone that started it was unpaired from this computer." + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "signed-out", error = unpaired) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals(unpaired, state.notice, "the computer's own words first") + assertFalse(state.canRetry, "a new call would be refused the same way") + } + + @Test + fun `the computer's own error text wins over the generic reason`() = runTest { + val f = Fixture(this).live() + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "error", error = "OpenAI closed the session.") + runCurrent() + assertEquals("OpenAI closed the session.", f.manager.state.value.notice) + } + + @Test + fun `frames about other calls are ignored`() = runTest { + val f = Fixture(this).live() + f.api.serverCall.value = LiveCallState("other", "b9", "t9", "desktop", "marin", 1.0, LiveCallStatus.ENDED, endReason = "hung-up") + runCurrent() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + assertEquals(0, f.transport.closes) + } + + @Test + fun `a line that empties after the computer reported our call ends it, but a null before the first report does not`() = runTest { + val f = Fixture(this) + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + requireNotNull(f.transport.listener).onChannelOpen() + runCurrent() + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase, "the empty line from before the call is not news about it") + + f.api.serverCall.value = call.copy(status = LiveCallStatus.LIVE) + runCurrent() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + f.api.serverCall.value = null + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertEquals(LiveCallRules.CALL_ENDED, f.manager.state.value.notice, "the desktop's and the iPhone's words for a call that is gone") + assertEquals(1, f.transport.closes) + assertTrue(f.api.ends.isEmpty()) + } + + @Test + fun `session closed on the data channel ends the call with OpenAI's reason`() = runTest { + val f = Fixture(this).live() + requireNotNull(f.transport.listener).onMessage("""{"type":"session.closed","reason":"expired","session":{"id":"s"}}""") + runCurrent() + assertEquals("Call ended: it reached OpenAI's time limit.", f.manager.state.value.notice) + assertEquals(listOf("c1"), f.api.ends) + } + + @Test + fun `the computer's reason replaces a plain Call ended that the data channel brought first`() = runTest { + val f = Fixture(this).live() + // The idle hang-up: OpenAI closes on the computer's request, and the + // phone's channel says so before the computer's frame gets there. + requireNotNull(f.transport.listener).onMessage("""{"type":"session.closed","reason":"close_requested"}""") + runCurrent() + assertEquals(LiveCallRules.CALL_ENDED, f.manager.state.value.notice) + + f.api.serverCall.value = LiveCallState("other", "b9", "t9", "desktop", "marin", 1.0, LiveCallStatus.ENDED, endReason = "idle") + runCurrent() + assertEquals(LiveCallRules.CALL_ENDED, f.manager.state.value.notice, "another call's reason is not ours") + + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "idle") + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals("Call ended after a long silence.", state.notice) + assertFalse(state.canRetry, "an end that is not a drop: the phone button starts the next call") + + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "idle", error = "The call connection to OpenAI dropped.") + runCurrent() + assertEquals("Call ended after a long silence.", f.manager.state.value.notice, "only a plain Call ended is replaced") + } + + @Test + fun `the computer's error text replaces a plain Call ended`() = runTest { + val f = Fixture(this).live() + requireNotNull(f.transport.listener).onMessage("""{"type":"session.closed"}""") + runCurrent() + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "sideband-lost", error = "The call connection to OpenAI dropped.") + runCurrent() + assertEquals("The call connection to OpenAI dropped.", f.manager.state.value.notice) + } + + @Test + fun `a drop the phone saw itself stays a drop`() = runTest { + val f = Fixture(this).live() + requireNotNull(f.transport.listener).onDropped() + runCurrent() + assertEquals(LiveCallRules.CALL_DROPPED, f.manager.state.value.notice) + // The computer records the phone's own end request. + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "hung-up") + runCurrent() + assertEquals(LiveCallRules.CALL_DROPPED, f.manager.state.value.notice) + } + + @Test + fun `the transport dropping ends the call with Try again and tells the computer`() = runTest { + val f = Fixture(this).live() + requireNotNull(f.transport.listener).onDropped() + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertEquals("Call dropped.", f.manager.state.value.notice) + assertTrue(f.manager.state.value.canRetry) + assertEquals(1, f.transport.closeSent) + assertEquals(listOf("c1"), f.api.ends) + } + + @Test + fun `a frame about our call that beats the 201 still counts`() = runTest { + val f = Fixture(this) + val gate = CompletableDeferred() + f.api.gate = gate + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + // A fast computer: the live frame lands before the 201 that names the call. + f.api.serverCall.value = call.copy(status = LiveCallStatus.LIVE) + runCurrent() + gate.complete(Unit) + runCurrent() + requireNotNull(f.transport.listener).onChannelOpen() + runCurrent() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase, "the frame that came first still said attached") + + f.api.serverCall.value = null + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase, "the computer had reported the call, so no call now means it is gone") + assertEquals(LiveCallRules.CALL_ENDED, f.manager.state.value.notice) + assertTrue(f.api.ends.isEmpty()) + } + + @Test + fun `an ended frame that beats the 201 ends the call before any audio`() = runTest { + val f = Fixture(this) + val gate = CompletableDeferred() + f.api.gate = gate + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = "idle") + runCurrent() + gate.complete(Unit) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals("Call ended after a long silence.", state.notice) + assertNull(f.transport.accepted, "no answer is applied to a call that is over") + assertEquals(1, f.transport.closes) + assertTrue(f.api.ends.isEmpty(), "the computer already knows") + } + + @Test + fun `a 201 that already reports the call ended never goes live`() = runTest { + val f = Fixture(this) + f.api.answer = { + LiveCallStart.Started( + call.copy(status = LiveCallStatus.ENDED, endReason = "sideband-lost", error = "The call could not connect to OpenAI."), + FakeApi.ANSWER, + ) + } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals("The call could not connect to OpenAI.", state.notice) + assertNull(f.transport.accepted) + assertEquals(1, f.transport.closes) + assertEquals(1, f.audio.ends) + assertTrue(f.api.ends.isEmpty(), "the computer already ended it") + } + + /** + * Try again as on the desktop (`canRetry: notice.dropped`) and the spec: + * only a dropped call and a failed start offer it. An end that is not a + * drop offers only the cross, whatever words it carries. + */ + @Test + fun `Try again only after a drop or a failed start, as on the desktop`() = runTest { + val words = "The computer's own words." + for (reason in listOf("hung-up", "idle", "expired", "content", "deleted", "shutdown", "signed-out", "a-reason-from-a-newer-computer", null)) { + for (error in listOf(null, words)) { + val f = Fixture(this).live() + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = reason, error = error) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertFalse(f.manager.state.value.canRetry, "$reason, words: $error") + } + } + for (reason in listOf("remote-hangup", "connection-lost", "sideband-lost", "error")) { + for (error in listOf(null, words)) { + val f = Fixture(this).live() + f.api.serverCall.value = call.copy(status = LiveCallStatus.ENDED, endReason = reason, error = error) + runCurrent() + assertTrue(f.manager.state.value.canRetry, "$reason, words: $error") + } + } + } + + @Test + fun `the phone's own ends offer Try again only when they are drops`() = runTest { + // OpenAI closing on the data channel: its own reason decides. + for ((reason, dropped) in listOf("expired" to false, "close_requested" to false, "remote_hangup" to true, "connection_lost" to true)) { + val f = Fixture(this).live() + requireNotNull(f.transport.listener).onMessage("""{"type":"session.closed","reason":"$reason"}""") + runCurrent() + assertEquals(dropped, f.manager.state.value.canRetry, reason) + } + // The computer no longer reports the call: "Call ended.", not a drop. + val gone = Fixture(this).live() + gone.api.serverCall.value = null + runCurrent() + assertEquals(LiveCallRules.CALL_ENDED, gone.manager.state.value.notice) + assertFalse(gone.manager.state.value.canRetry) + // Another app took the audio: the call ended, it did not drop. + val focus = Fixture(this).live() + requireNotNull(focus.audio.onFocusLost).invoke() + runCurrent() + assertEquals(LiveCallRules.FOCUS_LOST_MESSAGE, focus.manager.state.value.notice) + assertFalse(focus.manager.state.value.canRetry) + // The transport dropping and audio that never connects are drops. + val dropped = Fixture(this).live() + requireNotNull(dropped.transport.listener).onDropped() + runCurrent() + assertTrue(dropped.manager.state.value.canRetry) + } + + @Test + fun `a 201 that already reports the call ended offers Try again only for a drop`() = runTest { + for ((reason, retry) in listOf("sideband-lost" to true, "deleted" to false)) { + val f = Fixture(this) + f.api.answer = { LiveCallStart.Started(call.copy(status = LiveCallStatus.ENDED, endReason = reason), FakeApi.ANSWER) } + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertEquals(retry, f.manager.state.value.canRetry, reason) + } + } + + // ------------------------------------------------------------ the audio deadline + + @Test + fun `a call whose audio never connects drops after 20 s and tells the computer once`() = runTest { + assertEquals(20_000L, LiveCallManager.MEDIA_CONNECT_TIMEOUT_MS) + val f = Fixture(this).answered() + advanceTimeBy(LiveCallManager.MEDIA_CONNECT_TIMEOUT_MS - 1) + runCurrent() + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase, "not a moment early") + assertTrue(f.api.ends.isEmpty()) + + advanceTimeBy(1) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase, "the bar stays up with the reason and Try again") + assertEquals("Call dropped: the audio could not connect.", state.notice) + assertTrue(state.canRetry) + assertEquals(1, f.transport.closeSent, "session.close on the data channel, as a drop sends it") + assertEquals(1, f.transport.closes) + assertEquals(1, f.audio.ends) + assertEquals(listOf("c1"), f.api.ends) + + advanceTimeBy(10 * 60_000L) + runCurrent() + assertEquals(listOf("c1"), f.api.ends, "one end request, however long the phone waits after") + f.manager.retry(grant) + runCurrent() + assertEquals(2, f.api.starts.size, "Try again starts a fresh call") + f.goLive() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + } + + @Test + fun `a call whose audio connects in time is not dropped`() = runTest { + val f = Fixture(this).answered() + advanceTimeBy(LiveCallManager.MEDIA_CONNECT_TIMEOUT_MS - 1) + requireNotNull(f.transport.listener).onConnected() + runCurrent() + advanceTimeBy(10 * 60_000L) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.STARTING, state.phase) + assertNull(state.notice) + assertEquals(0, f.transport.closes) + assertTrue(f.api.ends.isEmpty()) + } + + @Test + fun `a channel that opens in time counts as audio that connected`() = runTest { + val f = Fixture(this).answered() + advanceTimeBy(LiveCallManager.MEDIA_CONNECT_TIMEOUT_MS - 1) + requireNotNull(f.transport.listener).onChannelOpen() + runCurrent() + advanceTimeBy(10 * 60_000L) + runCurrent() + assertNull(f.manager.state.value.notice) + assertTrue(f.api.ends.isEmpty()) + } + + @Test + fun `hanging up before the deadline leaves no late drop`() = runTest { + val f = Fixture(this).answered() + advanceTimeBy(5_000) + f.manager.hangUp() + runCurrent() + advanceTimeBy(10 * 60_000L) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.IDLE, state.phase, "no \"Call dropped\" over a call the person hung up") + assertNull(state.notice) + assertEquals(listOf("c1"), f.api.ends, "the hang-up's end request, and no second one") + assertEquals(1, f.transport.closeSent) + assertEquals(1, f.transport.closes) + assertEquals(1, f.audio.ends) + } + + @Test + fun `an earlier call's deadline and peer never reach a later call`() = runTest { + val f = Fixture(this).answered() + val first = f.transport + advanceTimeBy(5_000) + f.manager.hangUp() + runCurrent() + f.api.answer = { LiveCallStart.Started(call.copy(callId = "c2"), FakeApi.ANSWER) } + f.answered() + assertNotSame(first, f.transport, "a fresh transport for the second call") + + // The first call's peer, late: a connect, an open channel, a drop and a close, all about that call. + val stale = requireNotNull(first.listener) + stale.onConnected() + stale.onChannelOpen() + stale.onDropped() + stale.onMessage("""{"type":"session.closed","reason":"expired","session":{"id":"s"}}""") + runCurrent() + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase) + assertEquals("c2", f.manager.state.value.callId) + + // 20 s: the first call's deadline, had it outlived the hang-up. + advanceTimeBy(LiveCallManager.MEDIA_CONNECT_TIMEOUT_MS - 5_000) + runCurrent() + assertEquals(LiveCallPhase.STARTING, f.manager.state.value.phase) + assertEquals(listOf("c1"), f.api.ends) + + // 25 s: the second call's own, 20 s after its answer; the stale connect did not clear it. + advanceTimeBy(5_000) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals(LiveCallRules.AUDIO_TIMEOUT_MESSAGE, state.notice) + assertEquals(listOf("c1", "c2"), f.api.ends) + } + + // ------------------------------------------------------------ lifecycle, sign-out, another computer + + @Test + fun `only the process lifecycle ends the call`() = runTest { + val f = Fixture(this) + val owner = TestOwner() + owner.registry.addObserver(f.manager) + owner.registry.handleLifecycleEvent(Lifecycle.Event.ON_CREATE) + owner.registry.handleLifecycleEvent(Lifecycle.Event.ON_START) + owner.registry.handleLifecycleEvent(Lifecycle.Event.ON_RESUME) + f.live() + + // An Activity pause (a dialog, a rotation in progress) is not leaving the app. + owner.registry.handleLifecycleEvent(Lifecycle.Event.ON_PAUSE) + runCurrent() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + + owner.registry.handleLifecycleEvent(Lifecycle.Event.ON_STOP) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertEquals(LiveCallRules.CALL_ENDED, f.manager.state.value.notice) + assertFalse(f.manager.state.value.canRetry, "leaving the app is not a drop") + assertEquals(1, f.transport.closeSent) + assertEquals(listOf("c1"), f.api.ends) + + owner.registry.handleLifecycleEvent(Lifecycle.Event.ON_START) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase, "coming back does not restart a call on its own") + } + + @Test + fun `a microphone answer for an attempt that already ended starts nothing`() = runTest { + val f = Fixture(this) + val answers = mutableListOf<(Boolean) -> Unit>() + val sheet = MicrophoneAccess { answers += it } + f.manager.start("b1", "t1", "Ada", sheet) + // The system sheet is still up when the app leaves the foreground. + f.manager.onStop(TestOwner()) + answers[0](true) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase, "coming back does not restart a call on its own") + assertEquals(LiveCallRules.CALL_ENDED, f.manager.state.value.notice) + assertTrue(f.api.starts.isEmpty()) + + // "Call ended." offers no Try again: the phone button starts anew. + f.manager.dismiss() + f.manager.start("b1", "t1", "Ada", sheet) + answers[0](true) + runCurrent() + assertTrue(f.api.starts.isEmpty(), "only the attempt that asked may connect") + answers[1](true) + runCurrent() + assertEquals(1, f.api.starts.size) + assertEquals("c1", f.manager.state.value.callId) + } + + @Test + fun `losing the pairing hangs up at once and says why, without Try again`() = runTest { + val f = Fixture(this).live() + f.api.link.value = LiveCallLink(computerId = "computer-1", signedIn = false) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.ENDED, state.phase) + assertEquals(LiveCallRules.SIGNED_OUT_MESSAGE, state.notice) + assertFalse(state.canRetry, "a new call would be refused the same way") + assertEquals(1, f.transport.closeSent, "OpenAI is told at once: the computer may no longer hear this phone") + assertEquals(1, f.transport.closes) + assertEquals(1, f.audio.ends) + assertEquals(listOf("c1"), f.api.ends, "and the computer, in case it still does") + } + + @Test + fun `losing the pairing while the call starts hangs up too, and ends what the computer made`() = runTest { + val f = Fixture(this) + val gate = CompletableDeferred() + f.api.gate = gate + f.manager.start("b1", "t1", "Ada", grant) + runCurrent() + f.api.link.value = LiveCallLink(computerId = null, signedIn = false) + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + assertEquals(LiveCallRules.SIGNED_OUT_MESSAGE, f.manager.state.value.notice) + gate.complete(Unit) + runCurrent() + assertEquals(listOf("c1"), f.api.ends) + assertNull(f.transport.accepted) + } + + @Test + fun `switching to another computer drops this phone's call without a notice`() = runTest { + val f = Fixture(this).live() + f.api.link.value = LiveCallLink(computerId = "computer-2", signedIn = true) + runCurrent() + val state = f.manager.state.value + assertEquals(LiveCallPhase.IDLE, state.phase, "a deliberate switch leaves no bar") + assertNull(state.notice) + assertNull(state.callId, "that id belongs to the other computer") + assertEquals(1, f.transport.closeSent) + assertEquals(1, f.audio.ends) + assertTrue(f.api.ends.isEmpty(), "the new computer is not asked to end the old one's call") + } + + @Test + fun `a switch clears the notice the emptied line left first`() = runTest { + val f = Fixture(this).live() + // The switch resets the line before it names the new computer. + f.api.serverCall.value = null + runCurrent() + assertEquals(LiveCallPhase.ENDED, f.manager.state.value.phase) + f.api.link.value = LiveCallLink(computerId = "computer-2", signedIn = true) + runCurrent() + assertEquals(LiveCallPhase.IDLE, f.manager.state.value.phase) + assertNull(f.manager.state.value.notice) + } + + @Test + fun `a reconnect to the same computer is not a switch`() = runTest { + val f = Fixture(this).live() + f.api.link.value = LiveCallLink(computerId = "computer-1", signedIn = true) + runCurrent() + assertEquals(LiveCallPhase.LIVE, f.manager.state.value.phase) + assertEquals(0, f.transport.closes) + } + + // ------------------------------------------------------------ fixtures + + /** + * The manager runs on [TestScope.backgroundScope]: its collectors of the + * computer's frames and of the pairing never complete, and `runTest` + * would otherwise wait for them. The background scope shares the test + * scheduler, so `runCurrent()` still drives the manager, and its clock is + * the scheduler's. + * + * One transport is one call, as in production: [transport] is the one + * the next call gets (so a test can set it up first), and a call after + * that one gets a fresh transport, which [transport] then names. + */ + private class Fixture(scope: TestScope, val preferences: FakePreferences = FakePreferences()) { + val api = FakeApi() + var transport = FakeTransport() + private set + val audio = FakeAudio() + val manager = LiveCallManager( + api = api, + scope = scope.backgroundScope, + transports = { + if (transport.offers > 0) transport = FakeTransport() + transport + }, + audio = audio, + preferences = preferences, + clock = { scope.testScheduler.currentTime }, + ) + private val testScope = scope + + init { + testScope.runCurrent() + } + + /** Started and answered: the computer named the call, which is still connecting. */ + fun answered(): Fixture { + manager.start("b1", "t1", "Ada", MicrophoneAccess { it(true) }) + testScope.runCurrent() + check(manager.state.value.phase == LiveCallPhase.STARTING && manager.state.value.callId != null) { + "fixture was not answered: ${manager.state.value}" + } + return this + } + + /** The computer reports the call attached and the data channel opens. */ + fun goLive(): Fixture { + val callId = checkNotNull(manager.state.value.callId) { "no call to go live: ${manager.state.value}" } + api.serverCall.value = LiveCallState(callId, "b1", "t1", "android", "marin", 5_000.0, LiveCallStatus.LIVE) + requireNotNull(transport.listener).onChannelOpen() + testScope.runCurrent() + check(manager.state.value.phase == LiveCallPhase.LIVE) { "fixture did not go live: ${manager.state.value}" } + return this + } + + fun live(): Fixture = answered().goLive() + } + + private class FakePreferences( + override var speaker: Boolean = true, + override var disclosureShown: Boolean = false, + ) : LiveCallPreferences + + private class FakeTransport : LiveCallTransport { + var listener: LiveCallTransport.Listener? = null + var accepted: String? = null + var rejectAnswer = false + var muted: Boolean? = null + var offers = 0 + var closeSent = 0 + var closes = 0 + var offerGate: CompletableDeferred? = null + var failOffer: Exception? = null + + override suspend fun offer(listener: LiveCallTransport.Listener): String { + offers += 1 + this.listener = listener + offerGate?.await() + // The transport's contract: closed while the offer was pending, it throws. + if (closes > 0) throw IllegalStateException("transport closed") + failOffer?.let { throw it } + return OFFER + } + + override suspend fun accept(answerSdp: String) { + if (closes > 0) throw IllegalStateException("closed") + if (rejectAnswer) throw IllegalStateException("setDescription failed: Failed to set remote answer sdp") + accepted = answerSdp + } + + override fun setMuted(muted: Boolean) { + this.muted = muted + } + + override fun sendClose() { + closeSent += 1 + } + + override fun close() { + closes += 1 + } + + companion object { + const val OFFER = "v=0\r\noffer\r\n" + } + } + + private class FakeApi : LiveCallApi { + val starts = mutableListOf>() + val ends = mutableListOf() + var gate: CompletableDeferred? = null + /** Holds the computer's answer to a hang-up until the test lets it go. */ + var endGate: CompletableDeferred? = null + var answer: () -> LiveCallStart = { + LiveCallStart.Started( + LiveCallState("c1", "b1", "t1", "android", "marin", 5_000.0, LiveCallStatus.CONNECTING), + ANSWER, + ) + } + override val serverCall = MutableStateFlow(null) + override val link = MutableStateFlow(LiveCallLink(computerId = "computer-1", signedIn = true)) + + override suspend fun start(botId: String, threadId: String, sdp: String): LiveCallStart { + starts += Triple(botId, threadId, sdp) + gate?.await() + return answer() + } + + override suspend fun end(callId: String) { + ends += callId + endGate?.await() + } + + companion object { + const val ANSWER = "v=0\r\nanswer\r\n" + } + } + + private class FakeAudio : LiveCallAudioRoute { + val begins = mutableListOf() + val speakers = mutableListOf() + var ends = 0 + var onFocusLost: (() -> Unit)? = null + /** Another call holds the audio: focus is refused as the call begins. */ + var refuseFocus = false + + override fun begin(speaker: Boolean, onFocusLost: () -> Unit) { + begins += speaker + this.onFocusLost = onFocusLost + if (refuseFocus) onFocusLost() + } + + override fun setSpeaker(speaker: Boolean) { + speakers += speaker + } + + override fun end() { + ends += 1 + } + } + + private class TestOwner : LifecycleOwner { + val registry = LifecycleRegistry.createUnsafe(this) + override val lifecycle: Lifecycle get() = registry + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallNativeIsolationTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallNativeIsolationTest.kt new file mode 100644 index 0000000000..8bb38bbfc6 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallNativeIsolationTest.kt @@ -0,0 +1,35 @@ +package com.openmausbot.companion.audio + +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * The JVM suite must never load libjingle_peerconnection_so: Robolectric + * cannot, and a test that reached `PeerConnectionFactory.initialize` would die + * with an UnsatisfiedLinkError. So exactly one production file may import + * `org.webrtc`, and everything else goes through [LiveCallTransport]. + */ +class LiveCallNativeIsolationTest { + @Test + fun onlyTheWebRtcTransportImportsOrgWebrtc() { + val importers = locate("src/main/kotlin").walkTopDown() + .filter { it.isFile && it.extension == "kt" } + .filter { file -> file.readLines().any { it.startsWith("import org.webrtc") } } + .map { it.name } + .toSet() + assertEquals(setOf("WebRtcLiveCallTransport.kt"), importers) + } + + private fun locate(relative: String): File { + var directory: File? = File(".").absoluteFile + while (directory != null) { + for (prefix in listOf("", "app/")) { + val file = File(directory, prefix + relative) + if (file.isDirectory) return file + } + directory = directory.parentFile + } + error("could not find $relative from ${File(".").absolutePath}") + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallPreferencesTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallPreferencesTest.kt new file mode 100644 index 0000000000..801ae0b6cb --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallPreferencesTest.kt @@ -0,0 +1,39 @@ +package com.openmausbot.companion.audio + +import android.app.Application +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config + +/** + * Speaker or earpiece is this phone's own choice, not the computer's: it is + * kept on the phone across calls and launches, as the iPhone keeps it. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class LiveCallPreferencesTest { + @Test + fun theSpeakerChoiceIsKeptOnThePhone() { + val context = RuntimeEnvironment.getApplication() as Application + val first = SharedLiveCallPreferences(context) + assertTrue(first.speaker, "the loudspeaker until the person picks otherwise") + first.speaker = false + assertFalse(SharedLiveCallPreferences(context).speaker, "a later launch reads the earpiece back") + first.speaker = true + assertTrue(SharedLiveCallPreferences(context).speaker) + } + + /** The first-call disclosure, once Start call was chosen, is this phone's to remember. */ + @Test + fun theFirstCallDisclosureIsRememberedOnThePhone() { + val context = RuntimeEnvironment.getApplication() as Application + val first = SharedLiveCallPreferences(context) + assertFalse(first.disclosureShown, "a phone that never called shows it first") + first.disclosureShown = true + assertTrue(SharedLiveCallPreferences(context).disclosureShown, "a later launch remembers it") + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallWiringTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallWiringTest.kt new file mode 100644 index 0000000000..e8e8f55980 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCallWiringTest.kt @@ -0,0 +1,42 @@ +package com.openmausbot.companion.audio + +import java.io.File +import kotlin.test.Test +import kotlin.test.assertTrue + +/** + * A call must outlive the chat screen and end only when the process leaves + * the foreground. That is two wiring facts no runtime test in this module can + * prove without instantiating the real Application (DataStore, the Keystore, + * MediaPlayer), so they are pinned from the source, the way + * `SessionLingerWiringTest` pins the linger install. + */ +class LiveCallWiringTest { + @Test + fun `the Application owns the manager and puts it on the process lifecycle`() { + val source = sourceFile("OpenMausApp.kt").readText() + assertTrue(source.contains("liveCalls = LiveCallManager(this, session, appScope)"), "app-scoped, on the app scope") + assertTrue( + source.contains("ProcessLifecycleOwner.get().lifecycle.addObserver(liveCalls)"), + "the process lifecycle, not an Activity's, ends the call", + ) + } + + @Test + fun `the Activity hands the same manager to every screen`() { + val source = sourceFile("MainActivity.kt").readText() + assertTrue(source.contains("liveCalls = app.liveCalls"), "screens must reach the one app-scoped manager") + } + + private fun sourceFile(name: String): File { + var directory: File? = File(".").absoluteFile + while (directory != null) { + for (prefix in listOf("", "app/")) { + val file = File(directory, prefix + "src/main/kotlin/com/openmausbot/companion/$name") + if (file.isFile) return file + } + directory = directory.parentFile + } + error("could not find $name from ${File(".").absolutePath}") + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCaptionsTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCaptionsTest.kt new file mode 100644 index 0000000000..4178e0e662 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/LiveCaptionsTest.kt @@ -0,0 +1,47 @@ +package com.openmausbot.companion.audio + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** The captions rule, the same as the desktop's `applyCaption` in `src/lib/live-call-media.ts`. */ +class LiveCaptionsTest { + private fun event(json: String): LiveChannelEvent = requireNotNull(LiveCaptions.parse(json)) + + @Test + fun buildsCaptionsFromTheDataChannel() { + var (caption, heard) = "" to "" + LiveCaptions.apply(caption, heard, event("""{"type":"session.input_transcript.delta","delta":"what time ","start_ms":10,"end_ms":20}""")) + .let { (c, h) -> caption = c; heard = h } + LiveCaptions.apply(caption, heard, event("""{"type":"session.input_transcript.delta","delta":"is it","start_ms":20,"end_ms":40}""")) + .let { (c, h) -> caption = c; heard = h } + assertEquals("" to "what time is it", caption to heard) + LiveCaptions.apply(caption, heard, event("""{"type":"session.output_transcript.delta","delta":"It is noon.","start_ms":50,"end_ms":90}""")) + .let { (c, h) -> caption = c; heard = h } + assertEquals("It is noon." to "", caption to heard) + } + + @Test + fun keepsOnlyTheTailOfALongLine() { + val (caption, _) = LiveCaptions.apply("x".repeat(300), "", event("""{"type":"session.output_transcript.delta","delta":"y"}""")) + assertEquals(LiveCallSnapshot.CAPTION_CHARS, caption.length) + assertEquals("y", caption.takeLast(1)) + val (_, heard) = LiveCaptions.apply("", "h".repeat(200), event("""{"type":"session.input_transcript.delta","delta":"!"}""")) + assertEquals(LiveCallSnapshot.HEARD_CHARS, heard.length) + } + + @Test + fun unreadableAndUnrelatedFramesChangeNothing() { + assertNull(LiveCaptions.parse("not json")) + assertNull(LiveCaptions.parse("""{"delta":"no type"}""")) + assertEquals("a" to "b", LiveCaptions.apply("a", "b", event("""{"type":"session.usage.updated","usage":{"seconds":4}}"""))) + } + + @Test + fun sessionClosedCarriesItsReason() { + assertEquals( + LiveChannelEvent("session.closed", null, "connection_lost"), + event("""{"type":"session.closed","reason":"connection_lost","session":{"id":"s"}}"""), + ) + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/TransportCloseGuardTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/TransportCloseGuardTest.kt new file mode 100644 index 0000000000..bbb7a449fc --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/TransportCloseGuardTest.kt @@ -0,0 +1,74 @@ +package com.openmausbot.companion.audio + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest + +/** + * Why a hang-up during "Connecting…" cannot reach a disposed peer: the WebRTC + * transport waits on its native callbacks through this guard, and a close + * fails the wait. The transport itself needs the native library, so the rules + * are pinned here. + */ +@OptIn(ExperimentalCoroutinesApi::class) +class TransportCloseGuardTest { + @Test + fun `a wait in progress fails when the transport closes`() = runTest { + val guard = TransportCloseGuard() + val callback = CompletableDeferred() + val waiter = async { runCatching { guard.await(callback) } } + runCurrent() + + assertTrue(guard.close()) + guard.failPending() + runCurrent() + assertEquals(TransportCloseGuard.CLOSED, waiter.await().exceptionOrNull()?.message) + // The disposed peer's observer firing afterwards is a no-op, not a second resume. + assertFalse(callback.complete("late")) + } + + @Test + fun `a callback that landed just before the close still fails the wait`() = runTest { + val guard = TransportCloseGuard() + val callback = CompletableDeferred() + val waiter = async { runCatching { guard.await(callback) } } + runCurrent() + + // The WebRTC thread delivers; the main thread closes before the waiter resumes. + callback.complete("v=0") + guard.close() + guard.failPending() + runCurrent() + assertEquals(TransportCloseGuard.CLOSED, waiter.await().exceptionOrNull()?.message) + } + + @Test + fun `nothing native starts once the transport is closed`() = runTest { + val guard = TransportCloseGuard() + guard.close() + var started = false + val failure = runCatching { guard.await(CompletableDeferred()) { started = true } }.exceptionOrNull() + assertEquals(TransportCloseGuard.CLOSED, failure?.message) + assertFalse(started) + assertFailsWith { guard.ensureOpen() } + assertFalse(guard.close(), "a second close is a no-op") + } + + @Test + fun `an open transport's wait returns the callback's value`() = runTest { + val guard = TransportCloseGuard() + val callback = CompletableDeferred() + assertEquals("v=0", guard.await(callback) { callback.complete("v=0") }) + assertFalse(guard.isClosed) + guard.close() + guard.failPending() // nothing is waiting: nothing happens + assertTrue(guard.isClosed) + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/VoiceNotePlayerTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/VoiceNotePlayerTest.kt index 0847bcb073..8ac91a11c0 100644 --- a/android/app/src/test/kotlin/com/openmausbot/companion/audio/VoiceNotePlayerTest.kt +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/VoiceNotePlayerTest.kt @@ -253,10 +253,44 @@ class VoiceNotePlayerTest { assertEquals(1, focus.abandons) } + /** + * The one place a note asks for the audio refuses it while this phone's + * Live call holds it, however the request got there (a download that + * finished after the call started, a resume): the call would lose the + * focus and end. Nothing playing or paused is touched. + */ + @Test + fun `a Live call holding the audio refuses play and resume without asking for the focus`() { + val focus = FakeFocus(grant = true) + val engines = ArrayDeque() + val controller = VoiceNoteController( + engineFactory = { FakeEngine(ok = true).also(engines::add) }, + focus = focus, + ) + assertNull(controller.play("a", byteArrayOf(1))) + controller.pause() + val paused = controller.playback.value + assertEquals(1, focus.requests) + + focus.callHolds = true + assertEquals(VoiceNoteController.DURING_LIVE_CALL, controller.play("b", byteArrayOf(2))) + assertEquals(VoiceNoteController.DURING_LIVE_CALL, controller.resume()) + assertEquals(1, focus.requests, "no focus asked for while the call holds it") + assertEquals(1, engines.size, "no engine made") + assertEquals(0, engines.single().releases, "the paused note is kept") + assertEquals(paused, controller.playback.value) + + focus.callHolds = false + assertNull(controller.resume()) + assertTrue(controller.playback.value?.playing == true) + } + private class FakeFocus(var grant: Boolean) : PreviewAudioFocus { var requests = 0 var abandons = 0 var lastOnInterrupted: (() -> Unit)? = null + var callHolds = false + override val heldByLiveCall: Boolean get() = callHolds override fun request(onInterrupted: () -> Unit): Boolean { requests += 1 diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/audio/VoicePreviewControllerTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/audio/VoicePreviewControllerTest.kt index d07dcf5435..d83a53c78b 100644 --- a/android/app/src/test/kotlin/com/openmausbot/companion/audio/VoicePreviewControllerTest.kt +++ b/android/app/src/test/kotlin/com/openmausbot/companion/audio/VoicePreviewControllerTest.kt @@ -240,10 +240,36 @@ class VoicePreviewControllerTest { assertTrue(controller.playing.value) } + /** A preview asks for the focus a Live call holds: refused, not failed, and nothing asked of the audio. */ + @Test + fun `a Live call holding the audio refuses a preview without asking for the focus`() { + val focus = FakeFocus(grant = true) + var engines = 0 + val controller = VoicePreviewController( + engineFactory = { + engines += 1 + FakeEngine(ok = true) + }, + focus = focus, + ) + focus.callHolds = true + assertEquals(VoicePreviewController.DURING_LIVE_CALL, controller.play(byteArrayOf(1))) + assertEquals("Voice preview is off during a Live call.", VoicePreviewController.DURING_LIVE_CALL) + assertFalse(controller.playing.value) + assertEquals(0, focus.requests) + assertEquals(0, engines) + + focus.callHolds = false + assertNull(controller.play(byteArrayOf(1))) + assertTrue(controller.playing.value) + } + private class FakeFocus(private val grant: Boolean) : PreviewAudioFocus { var requests = 0 var abandons = 0 var lastOnInterrupted: (() -> Unit)? = null + var callHolds = false + override val heldByLiveCall: Boolean get() = callHolds override fun request(onInterrupted: () -> Unit): Boolean { requests += 1 diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/onboarding/OnboardingScene.kt b/android/app/src/test/kotlin/com/openmausbot/companion/onboarding/OnboardingScene.kt index d967a98351..4601293bb1 100644 --- a/android/app/src/test/kotlin/com/openmausbot/companion/onboarding/OnboardingScene.kt +++ b/android/app/src/test/kotlin/com/openmausbot/companion/onboarding/OnboardingScene.kt @@ -28,6 +28,7 @@ import com.openmausbot.companion.ui.CompanionEnvironment import com.openmausbot.companion.ui.MicPermissionController import com.openmausbot.companion.ui.NotificationPermissionController import com.openmausbot.companion.ui.PermissionPreferences +import com.openmausbot.companion.audio.LiveCallManager import com.openmausbot.companion.audio.VoicePreviewPlayer import com.openmausbot.companion.audio.VoiceNotePlayer import java.util.concurrent.atomic.AtomicInteger @@ -149,6 +150,7 @@ class OnboardingScene( avatars = AvatarImageStore(fetch = { null }), voicePreview = VoicePreviewPlayer(context), voiceNotes = VoiceNotePlayer(context), + liveCalls = LiveCallManager(context, session, scope), dictation = SpeechDictation( context = context, hasRecordAudio = { false }, diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/permissions/CompanionPermissionsTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/permissions/CompanionPermissionsTest.kt index de583236ca..4350334687 100644 --- a/android/app/src/test/kotlin/com/openmausbot/companion/permissions/CompanionPermissionsTest.kt +++ b/android/app/src/test/kotlin/com/openmausbot/companion/permissions/CompanionPermissionsTest.kt @@ -60,7 +60,7 @@ class CompanionPermissionsTest { assertFalse(permissions.recordAudioGranted()) assertFalse( permissions.discoveryPermissions().contains(Manifest.permission.RECORD_AUDIO), - "RECORD_AUDIO must only be asked from the mic button", + "RECORD_AUDIO is asked from the mic button or when a Live call starts, never at startup", ) recordAudioGranted = true assertTrue(permissions.recordAudioGranted()) diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/ChatPolicyTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/ChatPolicyTest.kt index f7131144ab..92c0a93d33 100644 --- a/android/app/src/test/kotlin/com/openmausbot/companion/ui/ChatPolicyTest.kt +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/ChatPolicyTest.kt @@ -8,6 +8,7 @@ import com.openmausbot.companion.core.ChatTarget import com.openmausbot.companion.core.CompanionState import com.openmausbot.companion.core.GroupResponder import com.openmausbot.companion.core.Message +import com.openmausbot.companion.core.RoutineRunCard import com.openmausbot.companion.core.ModelSelection import com.openmausbot.companion.core.OptionCard import com.openmausbot.companion.core.Reaction @@ -282,6 +283,23 @@ class TranscriptLayoutTest { fun `an index past the end is not a stretch`() { assertFalse(TranscriptLayout.startsNewStretch(listOf(message("a", 0.0)), 4)) } + + /** + * The call bar under the transcript grows (its caption line, the remote + * bar's second line), so the list gets shorter from the bottom. A list + * that showed its end scrolls back by what the end lost; a reader who had + * scrolled up stays where they are. + */ + @Test + fun `a list that showed its end keeps it in view when it gets shorter`() { + assertEquals(40, TranscriptLayout.keepEndInView(hiddenBelow = 40, shrunkBy = 40), "it was at the end") + assertEquals(25, TranscriptLayout.keepEndInView(hiddenBelow = 25, shrunkBy = 40), "part of what it lost was empty space") + assertEquals(0, TranscriptLayout.keepEndInView(hiddenBelow = 0, shrunkBy = 40), "the end still shows") + assertEquals(0, TranscriptLayout.keepEndInView(hiddenBelow = 300, shrunkBy = 40), "the reader had scrolled up") + assertEquals(0, TranscriptLayout.keepEndInView(hiddenBelow = 41, shrunkBy = 40), "the end was already out of view") + assertEquals(0, TranscriptLayout.keepEndInView(hiddenBelow = 40, shrunkBy = 0), "the list kept its height") + assertEquals(0, TranscriptLayout.keepEndInView(hiddenBelow = 40, shrunkBy = -20), "a taller list needs no help") + } } class SearchPolicyTest { @@ -606,6 +624,13 @@ class MessageActionsTest { assertEquals("hello", MessageActions.copyableText(message(Message.Kind.UNKNOWN, "hello"))) } + @Test + fun `a failed routine run copies the error the card shows`() { + val run = RoutineRunCard(routineName = "Brief", status = "failed", summary = "old report", error = "boom") + val copied = MessageActions.copyableText(message(Message.Kind.ROUTINE_RUN, null).copy(routineRun = run)) + assertEquals(true, copied?.contains("boom")) + } + @Test fun `empty or absent text offers nothing`() { assertNull(MessageActions.copyableText(message(Message.Kind.TEXT, null))) diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallBannerTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallBannerTest.kt new file mode 100644 index 0000000000..b8b16c6590 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallBannerTest.kt @@ -0,0 +1,154 @@ +package com.openmausbot.companion.ui + +import androidx.activity.ComponentActivity +import androidx.compose.runtime.CompositionLocalProvider +import androidx.compose.ui.test.assertCountEquals +import androidx.compose.ui.test.assertIsDisplayed +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onAllNodesWithText +import androidx.compose.ui.test.onNodeWithText +import androidx.compose.ui.test.performClick +import com.openmausbot.companion.audio.LiveCallApi +import com.openmausbot.companion.audio.LiveCallLink +import com.openmausbot.companion.audio.LiveCallPhase +import com.openmausbot.companion.audio.LiveCallTransport +import com.openmausbot.companion.audio.MicrophoneAccess +import com.openmausbot.companion.core.ChatTarget +import com.openmausbot.companion.core.LiveCallStart +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.LiveCallStatus +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.flow.MutableStateFlow +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.GraphicsMode + +/** + * The thin strip that follows a call to every other screen: tapping it goes + * back to the call's chat, Hang up hangs up without going anywhere, and the + * call's own chat — which has the bar — never shows it. The host reads the + * real manager, so a call that outlives its chat (Computer pushed over it) + * keeps its clock in the banner. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +@GraphicsMode(GraphicsMode.Mode.NATIVE) +class LiveCallBannerTest { + @get:Rule val compose = createAndroidComposeRule() + + @Test + fun tappingTheBannerOpensTheChatAndHangUpOnlyHangsUp() { + val taps = mutableListOf() + compose.setContent { + CompanionTheme(darkTheme = false) { + LiveCallBanner(title = "Live with Ada · 0:05", onOpen = { taps += "open" }, onHangUp = { taps += "hangup" }) + } + } + compose.onNodeWithText("Live with Ada · 0:05").assertIsDisplayed().performClick() + compose.onNodeWithText("Hang up").performClick() + assertEquals(listOf("open", "hangup"), taps) + } + + @Test + fun theCallsOwnChatHidesTheBanner() { + assertTrue(LiveCallBannerRules.onCallsChat(Destination.Chat(ChatTarget.Bot("b1", "t1")), "t1")) + assertTrue(LiveCallBannerRules.onCallsChat(Destination.Thread("t1"), "t1")) + assertFalse(LiveCallBannerRules.onCallsChat(Destination.Chat(ChatTarget.Bot("b1", "t2")), "t1"), "another task of the same bot") + assertFalse(LiveCallBannerRules.onCallsChat(Destination.Roster, "t1")) + assertFalse(LiveCallBannerRules.onCallsChat(Destination.Computer("b1"), "t1"), "Computer over the chat still shows the banner") + } + + @Test + fun `the banner shows on other screens while the call runs`() { + val api = FakeApi() + // The call went live 90 s ago on this phone's clock: the clock reads + // "1:" for a full 30 s of wall time around the assertion. + val scene = WiringScene( + liveTransports = { FakeTransport() }, + liveApi = api, + liveClock = { System.currentTimeMillis() - 90_000 }, + ) + val liveCalls = scene.environment.liveCalls + val chat = Destination.Chat(ChatTarget.Bot("b1", "t1")) + // The person pushed Computer over the call's chat, which left composition. + val navigator = CompanionNavigator(listOf(Destination.Roster, chat, Destination.Computer("b1"))) + compose.setContent { + CompositionLocalProvider(LocalCompanion provides scene.environment) { + CompanionTheme(darkTheme = false) { LiveCallBannerHost(navigator) } + } + } + compose.onAllNodesWithText("Hang up").assertCountEquals(0) + + compose.runOnIdle { liveCalls.start("b1", "t1", "Ada", MicrophoneAccess { it(true) }) } + compose.onNodeWithText("Connecting…").assertIsDisplayed() + // The computer attaches its side; the transport opened its channel with the answer. + compose.runOnIdle { api.serverCall.value = api.call(LiveCallStatus.LIVE) } + compose.waitUntil(5_000) { compose.runOnIdle { liveCalls.state.value.phase } == LiveCallPhase.LIVE } + // The clock counts from the moment the call went live, not from when this screen appeared. + compose.onNodeWithText("Live with Ada · 1:", substring = true).assertIsDisplayed().performClick() + assertEquals(chat, navigator.current, "tapping it opens the call's chat") + compose.onAllNodesWithText("Hang up").assertCountEquals(0) + assertEquals(LiveCallPhase.LIVE, liveCalls.state.value.phase, "opening the chat leaves the call up") + + compose.runOnIdle { navigator.pop() } + val answer = CompletableDeferred() + api.endGate = answer + compose.onNodeWithText("Hang up").performClick() + // Until the computer answers, the banner says so, and has nothing left to tap but itself. + compose.onNodeWithText("Hanging up…").assertIsDisplayed() + compose.onAllNodesWithText("Hang up").assertCountEquals(0) + assertEquals(LiveCallPhase.ENDING, liveCalls.state.value.phase) + assertEquals(listOf("c1"), api.ends, "Hang up tells the computer") + + compose.runOnIdle { answer.complete(Unit) } + compose.waitForIdle() + assertEquals(LiveCallPhase.IDLE, liveCalls.state.value.phase) + compose.onAllNodesWithText("Hanging up…").assertCountEquals(0) + compose.onAllNodesWithText("Live with", substring = true).assertCountEquals(0) + assertEquals(Destination.Computer("b1"), navigator.current, "Hang up goes nowhere") + } + + /** The computer's side, answering at once unless [endGate] holds the hang-up; the manager test's fake, trimmed. */ + private class FakeApi : LiveCallApi { + val ends = mutableListOf() + var endGate: CompletableDeferred? = null + override val serverCall = MutableStateFlow(null) + override val link = MutableStateFlow(LiveCallLink(computerId = "computer-1", signedIn = true)) + + fun call(status: LiveCallStatus) = LiveCallState("c1", "b1", "t1", "android", "marin", 1.0, status) + + override suspend fun start(botId: String, threadId: String, sdp: String): LiveCallStart = + LiveCallStart.Started(call(LiveCallStatus.CONNECTING), answerSdp = "v=0\r\nanswer\r\n") + + override suspend fun end(callId: String) { + ends += callId + endGate?.await() + } + } + + /** Opens its data channel as the answer goes in, as a working connection does. */ + private class FakeTransport : LiveCallTransport { + private var listener: LiveCallTransport.Listener? = null + + override suspend fun offer(listener: LiveCallTransport.Listener): String { + this.listener = listener + return "v=0\r\noffer\r\n" + } + + override suspend fun accept(answerSdp: String) { + listener?.onChannelOpen() + } + + override fun setMuted(muted: Boolean) = Unit + + override fun sendClose() = Unit + + override fun close() = Unit + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallBarTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallBarTest.kt new file mode 100644 index 0000000000..a4c768a0b7 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallBarTest.kt @@ -0,0 +1,354 @@ +package com.openmausbot.companion.ui + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.view.View +import androidx.activity.ComponentActivity +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.CompositionLocalProvider +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.semantics.SemanticsActions +import androidx.compose.ui.test.SemanticsNodeInteraction +import androidx.compose.ui.test.assertCountEquals +import androidx.compose.ui.test.assertIsDisplayed +import androidx.compose.ui.test.assertTextEquals +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onAllNodesWithContentDescription +import androidx.compose.ui.test.onAllNodesWithTag +import androidx.compose.ui.test.onAllNodesWithText +import androidx.compose.ui.test.onNodeWithContentDescription +import androidx.compose.ui.test.onNodeWithTag +import androidx.compose.ui.test.onNodeWithText +import androidx.compose.ui.test.performClick +import androidx.compose.ui.text.TextLayoutResult +import androidx.compose.ui.unit.Density +import androidx.compose.ui.unit.dp +import com.openmausbot.companion.audio.LiveCallPhase +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.LiveCallStatus +import kotlin.math.roundToInt +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.GraphicsMode + +/** + * The bar as a person meets it: which line and which buttons each phase shows, + * what the buttons call, and that on a narrow phone a long reason still says + * what to do and a long caption still shows its newest words. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +@GraphicsMode(GraphicsMode.Mode.NATIVE) +class LiveCallBarTest { + @get:Rule val compose = createAndroidComposeRule() + private val taps = mutableListOf() + private val actions = LiveCallBarActions( + onMute = { taps += "mute:$it" }, + onSettings = { taps += "settings" }, + onHangUp = { taps += "hangup" }, + onRetry = { taps += "retry" }, + onDismiss = { taps += "dismiss" }, + ) + + private fun mount(model: LiveCallBarModel) { + compose.setContent { CompanionTheme(darkTheme = false) { LiveCallBar(model, actions) } } + } + + @Test + fun aLiveBarShowsTheClockTheCaptionAndTheThreeControls() { + mount(LiveCallBarModel.Local("Live with Ada", "Hello there", "", muted = false, speaker = true, phase = LiveCallPhase.LIVE, clock = "1:05")) + compose.onNodeWithContentDescription("Live call").assertExists() + compose.onNodeWithText("Live with Ada", useUnmergedTree = true).assertIsDisplayed() + compose.onNodeWithText(" · 1:05", useUnmergedTree = true).assertIsDisplayed() + compose.onNodeWithTag("Captions").assertTextEquals("Hello there") + // TalkBack reads the caption, not a label standing in for it. + compose.onAllNodesWithContentDescription("Captions").assertCountEquals(0) + compose.onNodeWithContentDescription("Mute").performClick() + compose.onNodeWithContentDescription("Live call settings").performClick() + compose.onNodeWithContentDescription("Hang up").performClick() + assertEquals(listOf("mute:true", "settings", "hangup"), taps) + compose.onAllNodesWithText("Try again").assertCountEquals(0) + } + + @Test + fun thePersonsOwnWordsReplaceTheCaptionWhileTheySpeakAndMuteReadsUnmute() { + mount(LiveCallBarModel.Local("Live with Ada · 0:09", "It is noon.", "what time", muted = true, speaker = true, phase = LiveCallPhase.LIVE)) + compose.onNodeWithTag("Captions").assertTextEquals("what time") + compose.onNodeWithContentDescription("Unmute").performClick() + assertEquals(listOf("mute:false"), taps) + } + + @Test + fun connectingHasNoCaptionLineYet() { + mount(LiveCallBarModel.Local("Connecting…", "", "", muted = false, speaker = true, phase = LiveCallPhase.STARTING)) + compose.onNodeWithText("Connecting…").assertIsDisplayed() + compose.onAllNodesWithTag("Captions").assertCountEquals(0) + compose.onNodeWithContentDescription("Hang up").assertIsDisplayed() + } + + @Test + fun anEndedBarOffersTryAgainAndDismissOnly() { + mount(LiveCallBarModel.Local("Call dropped", "", "", muted = false, speaker = true, phase = LiveCallPhase.ENDED)) + compose.onNodeWithText("Call dropped").assertIsDisplayed() + compose.onAllNodesWithContentDescription("Hang up").assertCountEquals(0) + compose.onAllNodesWithContentDescription("Mute").assertCountEquals(0) + compose.onAllNodesWithTag("Captions").assertCountEquals(0) + compose.onNodeWithText("Try again").performClick() + compose.onNodeWithContentDescription("Dismiss").performClick() + assertEquals(listOf("retry", "dismiss"), taps) + } + + @Test + fun hangingUpShowsWhatIsHappeningAndNoControls() { + mount(LiveCallBarModel.Local("Hanging up…", "", "", muted = false, speaker = true, phase = LiveCallPhase.ENDING)) + compose.onNodeWithContentDescription("Live call").assertExists() + compose.onNodeWithText("Hanging up…").assertIsDisplayed() + listOf("Hang up", "Mute", "Live call settings", "Dismiss").forEach { + compose.onAllNodesWithContentDescription(it).assertCountEquals(0) + } + compose.onAllNodesWithText("Try again").assertCountEquals(0) + compose.onAllNodesWithTag("Captions").assertCountEquals(0) + } + + @Test + fun anEndThatTryingAgainCannotFixOffersOnlyTheCross() { + mount(LiveCallBarModel.Local(LiveCallRules.NEEDS_KEY_MESSAGE, "", "", muted = false, speaker = true, phase = LiveCallPhase.ENDED, canRetry = false)) + compose.onNodeWithText(LiveCallRules.NEEDS_KEY_MESSAGE).assertIsDisplayed() + compose.onAllNodesWithText("Try again").assertCountEquals(0) + compose.onNodeWithContentDescription("Dismiss").performClick() + assertEquals(listOf("dismiss"), taps) + } + + @Test + @Config(qualifiers = "w320dp-h640dp") + fun aLongReasonWithoutTryAgainStillWrapsWholeWithTheCrossUnderIt() { + val busy = LiveCallRules.busyMessage(LiveCallState("c1", "b1", "t1", "desktop", startedAt = 0.0, status = LiveCallStatus.LIVE)) + mountInChat { LiveCallBarModel.Local(busy, "", "", muted = false, speaker = true, phase = LiveCallPhase.ENDED, canRetry = false) } + assertFalse(layoutOf(busy).multiParagraph.didExceedMaxLines, "cut off: $busy") + val reason = compose.onNodeWithText(busy).fetchSemanticsNode().boundsInRoot + val dismiss = compose.onNodeWithContentDescription("Dismiss").assertIsDisplayed().fetchSemanticsNode().boundsInRoot + assertFalse(dismiss.overlaps(reason), "the cross covers the reason") + compose.onAllNodesWithText("Try again").assertCountEquals(0) + } + + @Test + fun aRemoteBarOnlyHangsUp() { + mount(LiveCallBarModel.Remote("Live with Ada", "0:09", "From your computer", "c1")) + compose.onNodeWithContentDescription("Live call on another device").assertExists() + compose.onNodeWithText("Live with Ada", useUnmergedTree = true).assertIsDisplayed() + compose.onNodeWithText(" · 0:09", useUnmergedTree = true).assertIsDisplayed() + compose.onNodeWithText("From your computer").assertIsDisplayed() + compose.onAllNodesWithContentDescription("Mute").assertCountEquals(0) + compose.onAllNodesWithContentDescription("Live call settings").assertCountEquals(0) + compose.onNodeWithText("Hang up").performClick() + assertEquals(listOf("hangup"), taps) + } + + /** + * A name too long for the line gives way, and only the name: on the + * narrowest phone in common use the clock, where the call is and every + * button stay whole, on the remote bar and on this phone's own call. + */ + @Test + @Config(qualifiers = "w360dp-h640dp") + fun aLongNameGivesWayToTheClockOnBothBars() { + val name = "Live with $LONG_NAME" + var model by mutableStateOf(LiveCallBarModel.Remote(name, "12:34", "From your computer", "c1")) + mountInChat { model } + assertTheNameGivesWay(name, " · 12:34", before = compose.onNodeWithText("Hang up")) + val device = compose.onNodeWithText("From your computer", useUnmergedTree = true).assertIsDisplayed() + val where = layoutOf("From your computer") + assertFalse(where.multiParagraph.didExceedMaxLines || where.isLineEllipsized(0), "where the call is was cut off") + val bar = compose.onNodeWithContentDescription("Live call on another device").fetchSemanticsNode().boundsInRoot + val line = device.fetchSemanticsNode().boundsInRoot + assertTrue(line.left >= bar.left && line.top >= bar.top && line.right <= bar.right && line.bottom <= bar.bottom, "where the call is sits outside the bar: $line $bar") + + compose.runOnIdle { + model = LiveCallBarModel.Local(name, "It is noon.", "", muted = false, speaker = true, phase = LiveCallPhase.LIVE, clock = "12:34") + } + assertTheNameGivesWay(name, " · 12:34", before = compose.onNodeWithContentDescription("Live call settings")) + listOf("Live call settings", "Mute", "Hang up").forEach { compose.onNodeWithContentDescription(it).assertIsDisplayed() } + } + + @Test + fun hiddenDrawsNothing() { + mount(LiveCallBarModel.Hidden) + compose.onAllNodesWithContentDescription("Live call").assertCountEquals(0) + compose.onAllNodesWithContentDescription("Live call on another device").assertCountEquals(0) + } + + @Test + @Config(qualifiers = "w320dp-h640dp") + fun aLongReasonWrapsWholeWithTryAgainAndTheCrossUnderIt() { + val busy = "A Live call is already running from your computer. Hang up there first." + mountInChat { ended(busy) } + val layout = layoutOf(busy) + // On one line it read "A Live call is already running …" and never said what to do. + assertTrue(layout.lineCount in 2..3, "the reason wraps: ${layout.lineCount} line(s)") + assertFalse(layout.multiParagraph.didExceedMaxLines, "\"Hang up there first.\" was cut off") + val reason = compose.onNodeWithText(busy).fetchSemanticsNode().boundsInRoot + val retry = compose.onNodeWithText("Try again").assertIsDisplayed().fetchSemanticsNode().boundsInRoot + val dismiss = compose.onNodeWithContentDescription("Dismiss").assertIsDisplayed().fetchSemanticsNode().boundsInRoot + assertTrue(retry.top >= reason.bottom && dismiss.top >= reason.bottom, "the buttons sit under the reason: $reason $retry $dismiss") + compose.onNodeWithText("Try again").performClick() + compose.onNodeWithContentDescription("Dismiss").performClick() + assertEquals(listOf("retry", "dismiss"), taps) + } + + @Test + @Config(qualifiers = "w360dp-h640dp") + fun everyReasonTheBarKnowsReadsWholeAt130PercentFont() { + val busy = listOf("desktop", "ios", "android", "watch").map { client -> + LiveCallRules.busyMessage(LiveCallState("c1", "b1", "t1", client, startedAt = 0.0, status = LiveCallStatus.LIVE)) + } + val endings = listOf("idle", "expired", "content", "deleted", "shutdown", "error", null).map { LiveCallRules.endNotice(it).text } + val reasons = listOf( + LiveCallRules.NEEDS_KEY_MESSAGE, + LiveCallRules.MIC_DENIED_MESSAGE, + LiveCallRules.AUDIO_FAILED_MESSAGE, + LiveCallRules.AUDIO_TIMEOUT_MESSAGE, + LiveCallRules.START_FAILED_MESSAGE, + LiveCallRules.LAST_CALL_ENDING, + LiveCallRules.FOCUS_LOST_MESSAGE, + LiveCallRules.SIGNED_OUT_MESSAGE, + ) + busy + endings + var reason by mutableStateOf(reasons.first()) + mountInChat(fontScale = 1.3f) { ended(reason) } + reasons.forEach { text -> + compose.runOnIdle { reason = text } + assertFalse(layoutOf(text).multiParagraph.didExceedMaxLines, "cut off at 130 %: $text") + val shown = compose.onNodeWithText(text).fetchSemanticsNode().boundsInRoot + listOf(compose.onNodeWithText("Try again"), compose.onNodeWithContentDescription("Dismiss")).forEach { button -> + val bounds = button.assertIsDisplayed().fetchSemanticsNode().boundsInRoot + assertFalse(bounds.overlaps(shown), "a button covers \"$text\"") + } + } + } + + @Test + @Config(qualifiers = "w320dp-h640dp") + fun atTheLargestFontTheReasonStopsAtThreeLinesAndTheButtonsStayInReach() { + val reason = LiveCallRules.MIC_DENIED_MESSAGE + mountInChat(fontScale = 2f) { ended(reason) } + assertTrue(layoutOf(reason).lineCount <= 3, "the bar grows no further over the chat") + val shown = compose.onNodeWithText(reason).fetchSemanticsNode().boundsInRoot + val retry = compose.onNodeWithText("Try again").assertIsDisplayed().fetchSemanticsNode().boundsInRoot + val dismiss = compose.onNodeWithContentDescription("Dismiss").assertIsDisplayed().fetchSemanticsNode().boundsInRoot + assertFalse(retry.overlaps(shown) || dismiss.overlaps(shown), "a button covers the reason") + compose.onNodeWithText("Try again").performClick() + compose.onNodeWithContentDescription("Dismiss").performClick() + assertEquals(listOf("retry", "dismiss"), taps) + } + + @Test + @Config(qualifiers = "w320dp-h640dp") + fun aShortReasonKeepsTryAgainAndTheCrossOnItsLine() { + mountInChat { ended(LiveCallRules.CALL_ENDED) } + assertEquals(1, layoutOf(LiveCallRules.CALL_ENDED).lineCount) + val reason = compose.onNodeWithText(LiveCallRules.CALL_ENDED).fetchSemanticsNode().boundsInRoot + val retry = compose.onNodeWithText("Try again").fetchSemanticsNode().boundsInRoot + val dismiss = compose.onNodeWithContentDescription("Dismiss").fetchSemanticsNode().boundsInRoot + assertTrue( + listOf(retry, dismiss).all { it.left >= reason.right && it.top < reason.bottom && reason.top < it.bottom }, + "the buttons share the reason's line: $reason $retry $dismiss", + ) + } + + @Test + fun aLongCaptionShowsItsNewestWordsOnOneLine() { + val tail = " then I open the report and read every failure to you, one by one" + var caption by mutableStateOf("Here is the plan.\nFirst I run the tests,$tail") + compose.setContent { + CompanionTheme(darkTheme = false) { + LiveCallBar(LiveCallBarModel.Local("Live with Ada · 1:05", caption, "", muted = false, speaker = true, phase = LiveCallPhase.LIVE), actions) + } + } + // A line break reads as a space: it would end the line and hide the words after it. + compose.onNodeWithTag("Captions").assertTextEquals("Here is the plan. First I run the tests,$tail") + val drawn = captionPixels() + compose.runOnIdle { caption = "So this is what we will do.\nI run the tests,$tail" } + assertTrue(drawn.contentEquals(captionPixels()), "only the oldest words changed, and they are the ones off the line") + compose.runOnIdle { caption = "Here is the plan.\nFirst I run the tests,$tail, twice" } + assertFalse(drawn.contentEquals(captionPixels()), "the newest words changed, and they are the ones on the line") + } + + private fun ended(reason: String) = + LiveCallBarModel.Local(reason, "", "", muted = false, speaker = true, phase = LiveCallPhase.ENDED) + + /** The line's name part is cut short; its clock is whole, and ends before [before], the line's first button. */ + private fun assertTheNameGivesWay(name: String, clock: String, before: SemanticsNodeInteraction) { + assertTrue(layoutOf(name).isLineEllipsized(0), "the name should give way") + compose.onNodeWithText(name, useUnmergedTree = true).assertIsDisplayed() + val shown = compose.onNodeWithText(clock, useUnmergedTree = true).assertIsDisplayed().fetchSemanticsNode().boundsInRoot + val clockLayout = layoutOf(clock) + // It never wraps, so its layout is as wide as its words: drawn whole, the node is too. + val whole = clockLayout.multiParagraph.intrinsics.maxIntrinsicWidth + assertTrue(shown.width >= whole - 0.5f, "the clock was cut off: ${shown.width} of $whole px") + val button = before.assertIsDisplayed().fetchSemanticsNode().boundsInRoot + assertTrue(shown.right <= button.left, "the clock runs into the button: $shown $button") + } + + /** + * The bar as `ChatScreen` places it, full width less 12 dp a side, on the + * phone the test's qualifiers set up: w320dp is the narrowest Android draws + * a normal screen at, w360dp the narrowest phone in common use. + * [fontScale] scales sp linearly, never smaller than a phone's own curve at + * the same setting. + */ + private fun mountInChat(fontScale: Float = 1f, model: () -> LiveCallBarModel) { + compose.setContent { + val phone = LocalDensity.current + CompositionLocalProvider(LocalDensity provides Density(phone.density, fontScale)) { + CompanionTheme(darkTheme = false) { + LiveCallBar(model(), actions, Modifier.fillMaxWidth().padding(horizontal = 12.dp)) + } + } + } + } + + /** + * The layout the reason's text node published. A `Text(String)` rebuilds it + * on request (see ParagraphDirectionTest); at the same width and with an end + * ellipsis it breaks the lines where the drawn one does, which is all these + * tests read. It ignores a start ellipsis, so the caption is read as pixels. + */ + private fun layoutOf(text: String): TextLayoutResult { + val node = compose.onNodeWithText(text, useUnmergedTree = true).fetchSemanticsNode() + val results = mutableListOf() + val action = node.config[SemanticsActions.GetTextLayoutResult].action + assertTrue(action != null && action(results), "the node produced no text layout") + return results.first() + } + + /** The caption line as drawn: the window drawn into a bitmap, cut to the line's bounds. */ + private fun captionPixels(): IntArray { + val line = compose.onNodeWithTag("Captions").fetchSemanticsNode().boundsInRoot + return compose.runOnIdle { + val window = compose.activity.window.decorView + val bitmap = Bitmap.createBitmap(window.width, window.height, Bitmap.Config.ARGB_8888) + window.draw(Canvas(bitmap)) + val root = IntArray(2).also { compose.activity.findViewById(android.R.id.content).getLocationInWindow(it) } + val width = line.width.roundToInt() + val height = line.height.roundToInt() + IntArray(width * height).also { pixels -> + bitmap.getPixels(pixels, 0, width, root[0] + line.left.roundToInt(), root[1] + line.top.roundToInt(), width, height) + } + } + } + + private companion object { + /** Forty characters: more than the bar's line holds on a phone. */ + const val LONG_NAME = "Scout, the Quarterly Release Coordinator" + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallChatWiringTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallChatWiringTest.kt new file mode 100644 index 0000000000..b6ccf850de --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallChatWiringTest.kt @@ -0,0 +1,411 @@ +package com.openmausbot.companion.ui + +import androidx.activity.ComponentActivity +import androidx.compose.runtime.Composable +import androidx.compose.runtime.CompositionLocalProvider +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.ui.MotionDurationScale +import androidx.compose.ui.test.ExperimentalTestApi +import androidx.compose.ui.test.assertCountEquals +import androidx.compose.ui.test.assertIsDisplayed +import androidx.compose.ui.test.assertIsEnabled +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onAllNodesWithContentDescription +import androidx.compose.ui.test.onAllNodesWithText +import androidx.compose.ui.test.onNodeWithContentDescription +import androidx.compose.ui.test.onNodeWithText +import androidx.compose.ui.test.performClick +import com.openmausbot.companion.audio.LiveCallPhase +import com.openmausbot.companion.audio.LiveCallTransport +import com.openmausbot.companion.audio.MicrophoneAccess +import com.openmausbot.companion.core.Bot +import com.openmausbot.companion.core.ChatTarget +import com.openmausbot.companion.core.CompanionJson +import com.openmausbot.companion.core.Connection +import com.openmausbot.companion.core.Fleet +import com.openmausbot.companion.core.Frame +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.LiveCallStatus +import com.openmausbot.companion.core.Message +import com.openmausbot.companion.core.StreamFrame +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.emitAll +import kotlinx.coroutines.flow.flow +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.GraphicsMode + +/** + * The chat screen end to end, with only the WebRTC transport faked: the phone + * button starts a call through the real manager, the real Session and a + * loopback server; the bar says "Connecting…" until the computer reports the + * call attached and the data channel is open, then shows the bot's name; Hang + * up ends it on the server. The phone button is hidden while this phone is on + * a call and while the computer reports one running from another device. A + * call the computer reports on this chat shows as the remote bar. A call + * keeps the microphone wherever the person goes: another chat's dictation + * stays off, since its audio focus would end the call. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +@GraphicsMode(GraphicsMode.Mode.NATIVE) +@OptIn(ExperimentalTestApi::class) +class LiveCallChatWiringTest { + @get:Rule + val compose = createAndroidComposeRule( + effectContext = object : MotionDurationScale { override val scaleFactor = 0f }, + ) + + private lateinit var server: MockWebServer + private lateinit var scene: WiringScene + private val requests = ConcurrentLinkedQueue() + private val frames = MutableSharedFlow(extraBufferCapacity = 16) + private val transport = FakeTransport() + private val fixture = bot().copy( + threadId = "thread-bot-1", + messages = listOf(Message("m1", Message.Role.USER, Message.Kind.TEXT, 1.0, text = "what time is it", via = "call")), + ) + private val nova = bot(id = "bot-2", name = "Nova") + /** Scout's chat, longer than the screen, ending on the answer to a spoken request. */ + private val longChat = fixture.copy( + messages = (1..14).map { index -> + if (index % 2 == 1) { + Message("q$index", Message.Role.USER, Message.Kind.TEXT, index.toDouble(), text = "Where are we with release step $index?") + } else { + Message("a$index", Message.Role.BOT, Message.Kind.TEXT, index.toDouble(), text = "Step $index is done: the branch is cut, the notes are drafted and the nightly build passed.") + } + } + listOf( + Message("q15", Message.Role.USER, Message.Kind.TEXT, 15.0, text = "What is left on the release checklist?", via = "call"), + Message("a16", Message.Role.BOT, Message.Kind.TEXT, 16.0, text = NEWEST), + ), + ) + + /** Set to hold the computer's answer to a hang-up, so the bar stays on "Hanging up…". */ + @Volatile private var endGate: CountDownLatch? = null + + @Before + fun startServer() { + server = MockWebServer() + server.dispatcher = object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + requests.add(request) + return when { + request.method == "POST" && request.path == "/api/live/session" -> + json(201, """{"call":${call("connecting")},"transport":{"type":"webrtc","sdp":"v=0\r\nanswer\r\n"}}""") + request.method == "POST" && request.path == "/api/live/call/end" -> { + endGate?.await(5, TimeUnit.SECONDS) + json(200, """{"call":${call("ended")}}""") + } + request.path == "/api/live/call" -> json(200, """{"call":null}""") + request.path == "/api/instances" -> json(200, """{"instances":[]}""") + request.path?.startsWith("/api/threads/") == true -> json(200, """{"messages":[],"hasMore":false}""") + else -> MockResponse().setResponseCode(404) + } + } + } + server.start() + } + + @After + fun stopServer() { + if (::scene.isInitialized) scene.session.disconnect() + server.shutdown() + } + + @Test + fun `the phone button starts a call, the bar shows it and Hang up ends it on the Mac`() { + mount { chatScreen() } + compose.onNodeWithText("via call").assertIsDisplayed() + + compose.onNodeWithContentDescription("Call Scout").performClick() + compose.waitUntil(10_000) { requests.any { it.path == "/api/live/session" } } + compose.onNodeWithText("Connecting…").assertIsDisplayed() + attach() + compose.waitUntil(10_000) { compose.onAllNodesWithText("Live with Scout", substring = true).fetchSemanticsNodes().isNotEmpty() } + + val start = requests.first { it.path == "/api/live/session" } + val body = CompanionJson.parseToJsonElement(start.body.readUtf8()).jsonObject + assertEquals("android", body.getValue("client").jsonPrimitive.content) + assertEquals("thread-bot-1", body.getValue("threadId").jsonPrimitive.content) + assertEquals(FakeTransport.OFFER, body.getValue("sdp").jsonPrimitive.content) + assertEquals("v=0\r\nanswer\r\n", transport.accepted) + compose.onNodeWithContentDescription("Call Scout").assertDoesNotExist() + + compose.onNodeWithContentDescription("Hang up").performClick() + compose.waitUntil(5_000) { requests.any { it.path == "/api/live/call/end" } } + compose.waitUntil(5_000) { compose.onAllNodesWithContentDescription("Live call").fetchSemanticsNodes().isEmpty() } + compose.onNodeWithContentDescription("Call Scout").assertIsEnabled() + assertEquals(1, transport.closeSent) + } + + /** + * A phone has no Live switch: its first call is where Live is turned on, + * so it says first what a call sends to OpenAI, with Start call and + * Cancel. Cancel starts nothing and asks again on the next tap; Start call + * starts the call, and the phone does not ask again. + */ + @Test + fun `a phone's first call says what it sends to OpenAI first, once`() { + mount(disclosureShown = false) { chatScreen() } + compose.onNodeWithContentDescription("Call Scout").performClick() + compose.onNodeWithText(LiveCallRules.DISCLOSURE).assertIsDisplayed() + compose.onNodeWithText("Cancel").performClick() + compose.waitForIdle() + compose.onAllNodesWithText(LiveCallRules.DISCLOSURE).assertCountEquals(0) + assertTrue(requests.none { it.path == "/api/live/session" }, "Cancel starts nothing") + assertEquals(0, transport.offers) + + compose.onNodeWithContentDescription("Call Scout").performClick() + compose.onNodeWithText(LiveCallRules.DISCLOSURE).assertIsDisplayed() + compose.onNodeWithText(LiveCallRules.START_CALL).performClick() + compose.waitUntil(10_000) { requests.any { it.path == "/api/live/session" } } + compose.onAllNodesWithText(LiveCallRules.DISCLOSURE).assertCountEquals(0) + + compose.onNodeWithContentDescription("Hang up").performClick() + compose.waitUntil(5_000) { requests.any { it.path == "/api/live/call/end" } } + compose.waitUntil(5_000) { compose.onAllNodesWithContentDescription("Call Scout").fetchSemanticsNodes().isNotEmpty() } + compose.onNodeWithContentDescription("Call Scout").performClick() + compose.waitUntil(10_000) { requests.count { it.path == "/api/live/session" } == 2 } + compose.onAllNodesWithText(LiveCallRules.DISCLOSURE).assertCountEquals(0) + } + + @Test + fun `a call from the Mac shows as a remote bar with Hang up`() { + mount { chatScreen() } + // A replay-0 flow drops what is emitted before the stream subscribes. + compose.waitUntil(5_000) { frames.subscriptionCount.value > 0 } + compose.runOnIdle { + frames.tryEmit( + StreamFrame( + Frame.LiveCall( + "bot-1", "thread-bot-1", + LiveCallState("c7", "bot-1", "thread-bot-1", "desktop", "marin", System.currentTimeMillis().toDouble(), LiveCallStatus.LIVE), + ), + seq = 2, + ), + ) + } + compose.waitUntil(5_000) { compose.onAllNodesWithText("From your computer").fetchSemanticsNodes().isNotEmpty() } + compose.onAllNodesWithContentDescription("Mute").assertCountEquals(0) + // The line is busy: a call from here would only be refused. + compose.onNodeWithContentDescription("Call Scout").assertDoesNotExist() + + compose.onNodeWithText("Hang up").performClick() + compose.waitUntil(5_000) { requests.any { it.path == "/api/live/call/end" } } + val end = requests.first { it.path == "/api/live/call/end" } + assertEquals("c7", CompanionJson.parseToJsonElement(end.body.readUtf8()).jsonObject.getValue("callId").jsonPrimitive.content) + assertEquals(0, transport.offers, "another device's call never touches this phone's media") + } + + /** + * The bar sits under the transcript and changes height: one line while it + * connects, a caption line more once the call is live, one line again + * while it hangs up; the remote bar comes with a second line of its own. + * Each time, the chat's newest message stays whole above the bar. + */ + @Test + @Config(qualifiers = "w411dp-h891dp") + fun `the newest message stays above the bar as the bar grows and shrinks`() { + mount(bots = listOf(longChat)) { chatScreen(longChat) } + compose.waitUntil(5_000) { compose.onAllNodesWithText(NEWEST, useUnmergedTree = true).fetchSemanticsNodes().isNotEmpty() } + + compose.onNodeWithContentDescription("Call Scout").performClick() + compose.waitUntil(10_000) { requests.any { it.path == "/api/live/session" } } + compose.onNodeWithText("Connecting…").assertIsDisplayed() + assertNewestAbove("Live call") + + attach() + compose.waitUntil(10_000) { compose.onAllNodesWithText("Live with Scout", substring = true).fetchSemanticsNodes().isNotEmpty() } + compose.runOnIdle { transport.say("""{"type":"session.output_transcript.delta","delta":"Three things are left."}""") } + compose.waitUntil(5_000) { compose.onAllNodesWithText("Three things are left.").fetchSemanticsNodes().isNotEmpty() } + assertNewestAbove("Live call") + + endGate = CountDownLatch(1) + compose.onNodeWithContentDescription("Hang up").performClick() + compose.waitUntil(5_000) { compose.onAllNodesWithText("Hanging up…").fetchSemanticsNodes().isNotEmpty() } + assertNewestAbove("Live call") + endGate?.countDown() + compose.waitUntil(5_000) { compose.onAllNodesWithContentDescription("Live call").fetchSemanticsNodes().isEmpty() } + + compose.runOnIdle { + frames.tryEmit( + StreamFrame( + Frame.LiveCall( + "bot-1", "thread-bot-1", + LiveCallState("c7", "bot-1", "thread-bot-1", "desktop", "marin", System.currentTimeMillis().toDouble(), LiveCallStatus.LIVE), + ), + seq = 4, + ), + ) + } + compose.waitUntil(5_000) { compose.onAllNodesWithText("From your computer").fetchSemanticsNodes().isNotEmpty() } + assertNewestAbove("Live call on another device") + } + + @Test + fun `a call in one chat keeps dictation off in every other chat`() { + mount(bots = listOf(fixture, nova)) { chatScreen(nova) } + val liveCalls = scene.environment.liveCalls + val dictation = scene.environment.dictation + // The call runs on Scout's chat; the person has since opened Nova's. + compose.runOnIdle { liveCalls.start(fixture.id, fixture.threadId, fixture.name, MicrophoneAccess { it(true) }) } + compose.waitUntil(10_000) { requests.any { it.path == "/api/live/session" } } + attach() + // Reads through runOnIdle: the 201 lands on the main looper, which only + // an idle sync drains here (no node query in the condition would). + compose.waitUntil(10_000) { compose.runOnIdle { liveCalls.state.value.phase } == LiveCallPhase.LIVE } + compose.onNodeWithContentDescription("Call Nova").assertDoesNotExist() + + compose.onNodeWithContentDescription("Start dictation").performClick() + compose.waitForIdle() + assertFalse(dictation.locksComposer(), "dictation must not start while a call holds the microphone") + assertNull(dictation.error.value, "the tap must not reach dictation") + assertEquals(LiveCallPhase.LIVE, liveCalls.state.value.phase) + assertTrue(requests.none { it.path == "/api/live/call/end" }, "the call is still up") + + // Once the call is over, the same tap reaches dictation (which the + // scene denies the microphone, so it settles on the denied notice). + compose.runOnIdle { liveCalls.hangUp() } + compose.waitUntil(5_000) { compose.runOnIdle { liveCalls.state.value.phase } == LiveCallPhase.IDLE } + compose.onNodeWithContentDescription("Start dictation").performClick() + compose.waitUntil(5_000) { compose.runOnIdle { dictation.error.value } != null } + } + + /** The computer's `live` frame for this phone's call, once its sideband has attached. */ + private fun attach() { + compose.waitUntil(5_000) { frames.subscriptionCount.value > 0 } + compose.runOnIdle { + frames.tryEmit( + StreamFrame( + Frame.LiveCall( + "bot-1", "thread-bot-1", + LiveCallState("c1", "bot-1", "thread-bot-1", "android", "marin", System.currentTimeMillis().toDouble(), LiveCallStatus.LIVE), + ), + seq = 3, + ), + ) + } + } + + /** + * The newest message is whole above the bar [description] names: its last + * line ends at or above the bar's top. Its own size and place, not what + * the list's clip leaves of it. + */ + private fun assertNewestAbove(description: String) { + compose.waitForIdle() + val newest = compose.onNodeWithText(NEWEST, useUnmergedTree = true).fetchSemanticsNode() + val bottom = newest.positionInRoot.y + newest.size.height + val bar = compose.onNodeWithContentDescription(description).fetchSemanticsNode().boundsInRoot + assertTrue(bottom <= bar.top, "the newest message runs under the bar: it ends at $bottom, the bar starts at ${bar.top}") + } + + @Composable + private fun chatScreen(target: Bot = fixture) { + ChatScreen( + destination = Destination.Chat(ChatTarget.Bot(target.id, target.threadId)), + onResolved = {}, + onBack = {}, + onOpenComputer = {}, + onOpenOverview = {}, + ) + } + + /** [disclosureShown]: this phone made a Live call before (the first-call disclosure is behind it). */ + private fun mount(bots: List = listOf(fixture), disclosureShown: Boolean = true, content: @Composable () -> Unit) { + scene = WiringScene( + connection = Connection(id = "live-fixture", name = "Fixture", host = "127.0.0.1", port = server.port), + fleet = Fleet(bots, emptyList()), + liveTransports = { transport }, + liveDisclosureShown = disclosureShown, + events = { + flow { + emit(StreamFrame(Frame.Hello(cursor = "fixture:1", resumed = false), seq = 1)) + emitAll(frames) + } + }, + ) + compose.setContent { + CompositionLocalProvider(LocalCompanion provides scene.environment) { + CompanionTheme(darkTheme = false) { + val state by scene.session.state.collectAsState() + if (state.bot(fixture.id) != null) content() + } + } + } + compose.runOnIdle { scene.session.connect() } + compose.waitUntil(5_000) { scene.session.state.value.bot(fixture.id) != null } + compose.waitForIdle() + } + + private fun call(status: String): String = + """{"callId":"c1","botId":"bot-1","threadId":"thread-bot-1","client":"android","voice":"marin","startedAt":${System.currentTimeMillis()},"status":"$status"}""" + + private fun json(code: Int, body: String): MockResponse = MockResponse() + .setResponseCode(code) + .setHeader("Content-Type", "application/json") + .setBody(body) + + /** + * The manager test's fake, repeated: fakes stay private to the test that + * owns them. Its data channel opens as the answer goes in, as a working + * connection's does. + */ + private class FakeTransport : LiveCallTransport { + var accepted: String? = null + var offers = 0 + var closeSent = 0 + private var listener: LiveCallTransport.Listener? = null + + override suspend fun offer(listener: LiveCallTransport.Listener): String { + offers += 1 + this.listener = listener + return OFFER + } + + override suspend fun accept(answerSdp: String) { + accepted = answerSdp + listener?.onChannelOpen() + } + + /** A frame on the data channel, as OpenAI would send it. */ + fun say(json: String) { + listener?.onMessage(json) + } + + override fun setMuted(muted: Boolean) = Unit + + override fun sendClose() { + closeSent += 1 + } + + override fun close() = Unit + + companion object { + const val OFFER = "v=0\r\noffer\r\n" + } + } + + private companion object { + const val NEWEST = "Three things: update the changelog, tag the release, and post the notes." + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallRulesTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallRulesTest.kt new file mode 100644 index 0000000000..6e7b4773fd --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallRulesTest.kt @@ -0,0 +1,256 @@ +package com.openmausbot.companion.ui + +import com.openmausbot.companion.audio.LiveCallPhase +import com.openmausbot.companion.audio.LiveCallSnapshot +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.LiveCallStatus +import java.util.Locale +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * The bar's wording and clock, pinned to the spec's copy, the desktop's + * `liveCallBarView` and its words (`src/locales/en.json`), and the shared + * contract rulings. + */ +class LiveCallRulesTest { + private val call = LiveCallState("c1", "b1", "t1", "desktop", "marin", 60_000.0, LiveCallStatus.LIVE) + + @Test + fun elapsedCountsFromTheStart() { + assertEquals("0:00", LiveCallRules.elapsed(60_000, 60_000)) + assertEquals("1:05", LiveCallRules.elapsed(1_000, 66_000)) + assertEquals("0:00", LiveCallRules.elapsed(90_000, 60_000), "a phone clock behind the computer's never shows a minus") + assertEquals("1:00:07", LiveCallRules.elapsed(0, 3_607_000)) + } + + @Test + fun theClockKeepsWesternDigitsWhateverTheDevicesLanguage() { + val saved = Locale.getDefault() + try { + // Arabic formats %d with Arabic-Indic digits by default. + Locale.setDefault(Locale.forLanguageTag("ar")) + assertEquals("1:05", LiveCallRules.elapsed(1_000, 66_000)) + assertEquals("1:00:07", LiveCallRules.elapsed(0, 3_607_000)) + } finally { + Locale.setDefault(saved) + } + } + + @Test + fun theTitleReadsLiveWithTheBotAndTheClock() { + assertEquals("Live with Ada · 1:05", LiveCallRules.title("Ada", "1:05")) + // The bars draw it in two parts, so that a long name gives way and the clock does not. + assertEquals("Live with Ada", LiveCallRules.liveWith("Ada")) + assertEquals(" · 1:05", LiveCallRules.clockSuffix("1:05")) + } + + /** + * The remote bar's second line: where the call is, in the desktop's words + * for the same bar ("Pepper is on a Live call from an iPhone", + * `call.live.onPhone`) and the iPhone's. + */ + @Test + fun aRemoteBarSaysWhereTheCallIsFrom() { + assertEquals("From your computer", LiveCallRules.fromDevice("desktop")) + assertEquals("From an iPhone", LiveCallRules.fromDevice("ios")) + assertEquals("From another phone", LiveCallRules.fromDevice("android")) + assertEquals("From another device", LiveCallRules.fromDevice("fridge")) + } + + @Test + fun busyNamesTheDeviceOnTheLine() { + assertEquals( + "A Live call is already running from your computer. Hang up there first.", + LiveCallRules.busyMessage(call), + ) + assertEquals( + "A Live call is already running from an iPhone. Hang up there first.", + LiveCallRules.busyMessage(call.copy(client = "ios")), + ) + assertEquals("another phone", LiveCallRules.clientLabel("android")) + assertEquals("another device", LiveCallRules.clientLabel("fridge")) + } + + /** + * The desktop's `endNotice` (`src/lib/live-call-media.ts`) with its English + * words (`src/locales/en.json`, `call.live.*`), for every `LiveEndReason`, + * and OpenAI's own `session.closed` spellings mapped the way the harness + * maps them (`CLOSE_REASONS`). + */ + @Test + fun endReasonsReadAsTheDesktopWordsThem() { + val table = mapOf( + "idle" to LiveCallRules.EndNotice("Call ended after a long silence.", dropped = false), + "expired" to LiveCallRules.EndNotice("Call ended: it reached OpenAI's time limit.", dropped = false), + "content" to LiveCallRules.EndNotice("OpenAI ended the call under its content rules.", dropped = false), + "deleted" to LiveCallRules.EndNotice("Call ended: the chat was deleted.", dropped = false), + "shutdown" to LiveCallRules.EndNotice("Call ended: OpenMausBot restarted.", dropped = false), + "signed-out" to LiveCallRules.EndNotice("Call ended: you were signed out.", dropped = false), + "remote-hangup" to LiveCallRules.EndNotice("Call dropped.", dropped = true), + "connection-lost" to LiveCallRules.EndNotice("Call dropped.", dropped = true), + "sideband-lost" to LiveCallRules.EndNotice("Call dropped.", dropped = true), + "error" to LiveCallRules.EndNotice("Call dropped.", dropped = true), + "hung-up" to LiveCallRules.EndNotice("Call ended.", dropped = false), + // OpenAI's spellings, on the phone's own data channel. + "remote_hangup" to LiveCallRules.EndNotice("Call dropped.", dropped = true), + "connection_lost" to LiveCallRules.EndNotice("Call dropped.", dropped = true), + "close_requested" to LiveCallRules.EndNotice("Call ended.", dropped = false), + "something-new" to LiveCallRules.EndNotice("Call ended.", dropped = false), + ) + table.forEach { (reason, notice) -> assertEquals(notice, LiveCallRules.endNotice(reason), reason) } + assertEquals(LiveCallRules.EndNotice("Call ended.", dropped = false), LiveCallRules.endNotice(null)) + assertEquals("Call ended.", LiveCallRules.CALL_ENDED) + assertEquals("Call dropped.", LiveCallRules.CALL_DROPPED) + assertEquals("Call dropped: the audio could not connect.", LiveCallRules.AUDIO_TIMEOUT_MESSAGE) + } + + @Test + fun liveCallWordsSayComputerNotMac() { + val words = listOf( + LiveCallRules.NEEDS_KEY_MESSAGE, + LiveCallRules.LAST_CALL_ENDING, + LiveCallRules.SIGNED_OUT_MESSAGE, + LiveCallRules.DISCLOSURE, + LiveCallRules.TYPED_REPLIES_FOOTER, + LiveCallRules.clientLabel("desktop"), + LiveCallRules.busyMessage(call), + ) + listOf("idle", "expired", "content", "deleted", "shutdown", "error", null).map { LiveCallRules.endNotice(it).text } + words.forEach { assertFalse(Regex("\\bMac\\b").containsMatchIn(it), "\"Mac\" in: $it") } + assertEquals("Set up Live calls on your computer first.", LiveCallRules.NEEDS_KEY_MESSAGE) + assertEquals("Your last call is still ending on your computer. Try again in a moment.", LiveCallRules.LAST_CALL_ENDING) + } + + @Test + fun losingThePairingReadsAsTheSignedOutEndReason() { + assertEquals(LiveCallRules.endNotice("signed-out").text, LiveCallRules.SIGNED_OUT_MESSAGE) + } + + @Test + fun theTypedRepliesLineSaysWhatTurningItOffKeepsFromOpenAI() { + assertEquals( + "When this is off, messages you type during a call and the bot's answers to them are not sent to OpenAI.", + LiveCallRules.TYPED_REPLIES_FOOTER, + ) + } + + @Test + fun idleChoicesAreTheSharedListAndKeepTheDesktopsValueVisible() { + assertEquals(listOf(1, 2, 3, 5, 10, 15, 30, 60), LiveCallRules.IDLE_PRESETS, "the same minutes on every client") + assertEquals(listOf(1, 2, 3, 5, 7, 10, 15, 30, 60), LiveCallRules.idleChoices(7)) + assertEquals(LiveCallRules.IDLE_PRESETS, LiveCallRules.idleChoices(5)) + assertEquals(LiveCallRules.IDLE_PRESETS, LiveCallRules.idleChoices(0), "out of range is not a choice") + } + + @Test + fun voicesMatchTheDesktopList() { + assertEquals(22, LiveCallRules.VOICE_OPTIONS.size) + assertEquals("marin", LiveCallRules.VOICE_OPTIONS.first().id) + assertEquals("Marin (default)", LiveCallRules.voiceLabel("")) + assertEquals("Cedar", LiveCallRules.voiceLabel("cedar")) + assertEquals("nova", LiveCallRules.voiceLabel("nova"), "a voice the list lacks is shown by id, not hidden") + } + + @Test + fun thisPhonesCallWinsTheBarAndItsClockCountsFromGoingLive() { + val local = LiveCallSnapshot( + phase = LiveCallPhase.LIVE, botId = "b1", threadId = "t1", botName = "Ada", callId = "c9", + liveSince = 1_000, caption = "Hello there", heard = "hi", muted = true, + ) + assertEquals( + LiveCallBarModel.Local("Live with Ada", "Hello there", "hi", muted = true, speaker = true, phase = LiveCallPhase.LIVE, clock = "1:05"), + LiveCallRules.barModel(local, call, "t1", "Ada", 66_000), + "from liveSince on this phone's clock, not the computer's startedAt", + ) + } + + @Test + fun hangingUpSaysSoUntilTheComputerConfirms() { + val ending = LiveCallSnapshot(phase = LiveCallPhase.ENDING, botId = "b1", threadId = "t1", botName = "Ada", callId = "c9") + val bar = LiveCallRules.barModel(ending, call.copy(callId = "c9", status = LiveCallStatus.ENDING), "t1", "Ada", 0) + assertEquals("Hanging up…", (bar as LiveCallBarModel.Local).title) + assertEquals(LiveCallPhase.ENDING, bar.phase) + } + + @Test + fun aCallFromAnotherDeviceShowsAsRemoteOnItsChatOnly() { + assertEquals( + LiveCallBarModel.Remote("Live with Ada", "1:05", "From your computer", "c1"), + LiveCallRules.barModel(LiveCallSnapshot(), call, "t1", "Ada", 125_000), + ) + assertEquals(LiveCallBarModel.Hidden, LiveCallRules.barModel(LiveCallSnapshot(), call, "other", "Bo", 125_000)) + assertEquals( + LiveCallBarModel.Hidden, + LiveCallRules.barModel(LiveCallSnapshot(), call.copy(status = LiveCallStatus.ENDED), "t1", "Ada", 125_000), + ) + assertEquals(LiveCallBarModel.Hidden, LiveCallRules.barModel(LiveCallSnapshot(), null, "t1", "Ada", 125_000)) + } + + @Test + fun thePhoneButtonHidesWheneverACallCouldNotStart() { + val idle = LiveCallSnapshot() + assertTrue(LiveCallRules.offersCall(idle, null)) + assertTrue(LiveCallRules.offersCall(idle, call.copy(status = LiveCallStatus.ENDED)), "an ended call frees the line") + assertFalse(LiveCallRules.offersCall(idle, call), "another device is on the line: it has to hang up first") + assertFalse(LiveCallRules.offersCall(idle, call.copy(status = LiveCallStatus.ENDING)), "the line is still winding down") + assertFalse(LiveCallRules.offersCall(idle, call.copy(status = LiveCallStatus.UNKNOWN))) + listOf(LiveCallPhase.STARTING, LiveCallPhase.LIVE, LiveCallPhase.ENDING).forEach { phase -> + assertFalse(LiveCallRules.offersCall(LiveCallSnapshot(phase = phase, threadId = "t1"), null), "$phase: the bar has the controls") + } + assertTrue( + LiveCallRules.offersCall(LiveCallSnapshot(phase = LiveCallPhase.ENDED, threadId = "t1", notice = "Call ended."), null), + "after a call ends, a new one starts from the button too", + ) + } + + @Test + fun aCallThatIsEndingOnTheComputerShowsNoRemoteBar() { + assertEquals( + LiveCallBarModel.Hidden, + LiveCallRules.barModel(LiveCallSnapshot(), call.copy(status = LiveCallStatus.ENDING), "t1", "Ada", 125_000), + "a hang-up is briefly still ending on the computer after its own bar has gone", + ) + } + + @Test + fun aCallWithAStatusThisBuildDoesNotKnowStillShowsItsRemoteBar() { + assertEquals( + LiveCallBarModel.Remote("Live with Ada", "1:05", "From your computer", "c1"), + LiveCallRules.barModel(LiveCallSnapshot(), call.copy(status = LiveCallStatus.UNKNOWN), "t1", "Ada", 125_000), + ) + } + + @Test + fun thisPhonesOwnCallIsNeverRemoteWhileTheMacWindsItDown() { + // A hang-up or a dismiss leaves IDLE with the call's id; the Mac reports the call until OpenAI confirms the close. + val hungUp = LiveCallSnapshot(callId = "c1") + val ours = call.copy(client = "android") + assertEquals(LiveCallBarModel.Hidden, LiveCallRules.barModel(hungUp, ours, "t1", "Ada", 125_000)) + assertEquals( + LiveCallBarModel.Hidden, + LiveCallRules.barModel(hungUp, ours.copy(status = LiveCallStatus.ENDING), "t1", "Ada", 125_000), + "the Mac's `ending` echo of this phone's hang-up is not a call on another phone", + ) + assertEquals( + LiveCallBarModel.Remote("Live with Ada", "1:05", "From another phone", "c2"), + LiveCallRules.barModel(hungUp, ours.copy(callId = "c2"), "t1", "Ada", 125_000), + "a different call on the same chat is still another device's", + ) + } + + @Test + fun anEndedLocalCallKeepsTheBarWithItsNoticeAndStartingSaysConnecting() { + val ended = LiveCallSnapshot(phase = LiveCallPhase.ENDED, botId = "b1", threadId = "t1", botName = "Ada", notice = "Call dropped.") + val bar = LiveCallRules.barModel(ended, null, "t1", "Ada", 0) as LiveCallBarModel.Local + assertEquals("Call dropped.", bar.title) + assertEquals(LiveCallPhase.ENDED, bar.phase) + assertTrue(bar.canRetry) + val refused = ended.copy(notice = LiveCallRules.NEEDS_KEY_MESSAGE, canRetry = false) + assertFalse((LiveCallRules.barModel(refused, null, "t1", "Ada", 0) as LiveCallBarModel.Local).canRetry, "no Try again where it cannot help") + assertEquals(LiveCallBarModel.Hidden, LiveCallRules.barModel(ended, null, "other", "Bo", 0)) + val starting = LiveCallSnapshot(phase = LiveCallPhase.STARTING, botId = "b1", threadId = "t1", botName = "Ada") + assertEquals("Connecting…", (LiveCallRules.barModel(starting, null, "t1", "Ada", 0) as LiveCallBarModel.Local).title) + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallSettingsFormTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallSettingsFormTest.kt new file mode 100644 index 0000000000..00b47b32b5 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallSettingsFormTest.kt @@ -0,0 +1,99 @@ +package com.openmausbot.companion.ui + +import androidx.activity.ComponentActivity +import androidx.compose.ui.test.assertCountEquals +import androidx.compose.ui.test.assertIsDisplayed +import androidx.compose.ui.test.assertIsNotSelected +import androidx.compose.ui.test.assertIsSelected +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onAllNodesWithText +import androidx.compose.ui.test.onNodeWithText +import androidx.compose.ui.test.performClick +import androidx.compose.ui.test.performScrollTo +import com.openmausbot.companion.core.LiveSettings +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.GraphicsMode + +/** + * The gear's sheet: every row writes exactly the one field it owns, the key + * is never on offer, and the sheet says in plain words what a Live call sends + * to OpenAI and what turning off typed replies keeps back. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +@GraphicsMode(GraphicsMode.Mode.NATIVE) +class LiveCallSettingsFormTest { + @get:Rule val compose = createAndroidComposeRule() + private val changes = mutableListOf() + + private fun mount(settings: LiveSettings?, speaker: Boolean = true) { + compose.setContent { + CompanionTheme(darkTheme = false) { + LiveCallSettingsForm( + settings = settings, + speaker = speaker, + saving = false, + error = null, + onDone = { changes += "done" }, + onVoice = { changes += "voice:$it" }, + onSpeaker = { changes += "speaker:$it" }, + onReadTypedReplies = { changes += "typed:$it" }, + onIdleMinutes = { changes += "idle:$it" }, + ) + } + } + } + + @Test + fun everyRowWritesTheOneFieldItOwns() { + mount(LiveSettings(configured = true, voice = "marin", readTypedReplies = true, idleMinutes = 5)) + compose.onNodeWithText("Earpiece").performScrollTo().performClick() + compose.onNodeWithText("Cedar").performScrollTo().performClick() + compose.onNodeWithText("Read replies to typed messages").performScrollTo().performClick() + compose.onNodeWithText("10 minutes").performScrollTo().performClick() + assertEquals(listOf("speaker:false", "voice:cedar", "typed:false", "idle:10"), changes) + } + + @Test + fun theSheetSaysWhatReachesOpenAIAndAnOddIdleValueStaysSelectable() { + mount(LiveSettings(configured = true, voice = "", readTypedReplies = false, idleMinutes = 7)) + // Before any choice: what a call sends, and that the key stays on the computer. + compose.onNodeWithText(LiveCallRules.DISCLOSURE).assertIsDisplayed() + compose.onNodeWithText(LiveCallRules.TYPED_REPLIES_FOOTER).performScrollTo().assertIsDisplayed() + compose.onNodeWithText("7 minutes").performScrollTo().assertIsDisplayed() + compose.onNodeWithText("3 minutes").performScrollTo().assertIsDisplayed() + compose.onNodeWithText(LiveCallRules.VOICE_APPLIES_NEXT_CALL).performScrollTo().assertIsDisplayed() + compose.onNodeWithText("Done").performClick() + assertEquals(listOf("done"), changes) + } + + @Test + fun theDisclosureNamesEverythingACallSendsAndWhereTheKeyStays() { + val text = LiveCallRules.DISCLOSURE + listOf("voice", "recent messages", "the bot's answers", "details of any approval", "OpenAI", "key stays on your computer") + .forEach { assertTrue(it in text, "the disclosure leaves out \"$it\": $text") } + assertTrue("OpenAI" in LiveCallRules.TYPED_REPLIES_FOOTER) + } + + /** The route rule (`LiveCallAudioRouting`) gives a connected headset the call whatever is picked here. */ + @Test + fun soundOutputShowsThisPhonesChoiceAndSaysAHeadsetTakesTheCall() { + mount(LiveSettings(configured = true), speaker = false) + compose.onNodeWithText("Earpiece").performScrollTo().assertIsSelected() + compose.onNodeWithText("Speaker").assertIsNotSelected() + compose.onNodeWithText("A connected headset takes the call instead.").performScrollTo().assertIsDisplayed() + } + + @Test + fun nothingIsOfferedBeforeTheSettingsLoad() { + mount(null) + compose.onNodeWithText("Loading…").assertIsDisplayed() + compose.onAllNodesWithText("Cedar").assertCountEquals(0) + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallVoicePreviewTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallVoicePreviewTest.kt new file mode 100644 index 0000000000..5d97820589 --- /dev/null +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/LiveCallVoicePreviewTest.kt @@ -0,0 +1,179 @@ +package com.openmausbot.companion.ui + +import androidx.activity.ComponentActivity +import androidx.compose.runtime.CompositionLocalProvider +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.ui.MotionDurationScale +import androidx.compose.ui.semantics.SemanticsActions +import androidx.compose.ui.test.ExperimentalTestApi +import androidx.compose.ui.test.assertCountEquals +import androidx.compose.ui.test.assertIsNotEnabled +import androidx.compose.ui.test.hasText +import androidx.compose.ui.test.isEnabled +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onAllNodesWithText +import androidx.compose.ui.test.onNodeWithText +import androidx.compose.ui.test.performSemanticsAction +import com.openmausbot.companion.audio.LiveCallPhase +import com.openmausbot.companion.audio.LiveCallTransport +import com.openmausbot.companion.audio.MicrophoneAccess +import com.openmausbot.companion.core.Connection +import com.openmausbot.companion.core.Fleet +import com.openmausbot.companion.core.Frame +import com.openmausbot.companion.core.StreamFrame +import java.util.concurrent.ConcurrentLinkedQueue +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.test.StandardTestDispatcher +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.GraphicsMode + +/** + * The profile sheet's "Preview voice" while a Live call runs. A preview asks + * for transient audio focus, and losing focus ends the call as "another app + * took the audio", so the sheet keeps the preview off until the call is over + * (as the chat keeps dictation off). + * + * The sheet reads its voice settings in a `LaunchedEffect`. Compose's default + * test dispatcher is unconfined, so that effect would resume on OkHttp's + * thread, where its state writes intermittently never reach the screen (about + * one run in two, measured). A [StandardTestDispatcher] resumes it on the main + * thread, as production does. Under it `performScrollTo` on the sheet never + * settles, so the row is tapped through its click action instead. + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +@GraphicsMode(GraphicsMode.Mode.NATIVE) +@OptIn(ExperimentalCoroutinesApi::class, ExperimentalTestApi::class) +class LiveCallVoicePreviewTest { + @get:Rule + val compose = createAndroidComposeRule( + effectContext = StandardTestDispatcher() + object : MotionDurationScale { override val scaleFactor = 0f }, + ) + + private lateinit var server: MockWebServer + private lateinit var scene: WiringScene + private val requests = ConcurrentLinkedQueue() + private val fixture = bot().copy(threadId = "thread-bot-1") + + @Before + fun startServer() { + server = MockWebServer() + server.dispatcher = object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + requests.add(request) + return when { + // Already attached in the 201: nothing here is about going live. + request.method == "POST" && request.path == "/api/live/session" -> + json(201, """{"call":${call("live")},"transport":{"type":"webrtc","sdp":"v=0\r\nanswer\r\n"}}""") + request.method == "POST" && request.path == "/api/live/call/end" -> json(200, """{"call":${call("ended")}}""") + request.path == "/api/live/call" -> json(200, """{"call":null}""") + request.path == "/api/instances" -> json(200, """{"instances":[]}""") + // A voice this computer can speak: the preview row is there to tap. + request.path == "/api/config" -> json(200, """{"tts":{"configured":true,"voice":"shared-voice"}}""") + request.path == "/api/tts/voices" -> json(200, """{"voices":[]}""") + else -> MockResponse().setResponseCode(404) + } + } + } + server.start() + } + + @After + fun stopServer() { + if (::scene.isInitialized) scene.session.disconnect() + server.shutdown() + } + + @Test + fun `a call keeps the voice preview off, since its audio focus would end the call`() { + mount() + val liveCalls = scene.environment.liveCalls + val previewable = hasText("Preview voice") and isEnabled() + compose.waitUntil(5_000) { compose.onAllNodes(previewable).fetchSemanticsNodes().isNotEmpty() } + + compose.runOnIdle { liveCalls.start(fixture.id, fixture.threadId, fixture.name, MicrophoneAccess { it(true) }) } + compose.waitUntil(10_000) { compose.runOnIdle { liveCalls.state.value.phase } == LiveCallPhase.LIVE } + // The manager's transition can precede collectAsState and recomposition. + compose.waitUntil(5_000) { + compose.onAllNodesWithText(LiveCallRules.PREVIEW_DURING_CALL).fetchSemanticsNodes().isNotEmpty() + } + compose.onNodeWithText(LiveCallRules.PREVIEW_DURING_CALL).assertExists() + compose.onNodeWithText("Preview voice").assertIsNotEnabled() + // The tap, through the row's click action (see above on scrolling), + // which runs even on a disabled row: the sheet must refuse it itself. + compose.onNodeWithText("Preview voice").performSemanticsAction(SemanticsActions.OnClick) + // A preview starts by fetching its audio; give a fetch time to reach the server. + val fetched = runCatching { compose.waitUntil(1_000) { requests.any { it.path == "/api/tts/speak" } } }.isSuccess + assertFalse(fetched, "no preview is fetched, so none can take the audio") + assertEquals(LiveCallPhase.LIVE, liveCalls.state.value.phase) + + compose.runOnIdle { liveCalls.hangUp() } + compose.waitUntil(5_000) { compose.onAllNodes(previewable).fetchSemanticsNodes().isNotEmpty() } + compose.onAllNodesWithText(LiveCallRules.PREVIEW_DURING_CALL).assertCountEquals(0) + } + + private fun mount() { + scene = WiringScene( + connection = Connection(id = "live-fixture", name = "Fixture", host = "127.0.0.1", port = server.port), + fleet = Fleet(listOf(fixture), emptyList()), + liveTransports = { SilentTransport() }, + events = { + flow { + emit(StreamFrame(Frame.Hello(cursor = "fixture:1", resumed = false), seq = 1)) + awaitCancellation() + } + }, + ) + compose.setContent { + CompositionLocalProvider(LocalCompanion provides scene.environment) { + CompanionTheme(darkTheme = false) { + val state by scene.session.state.collectAsState() + if (state.bot(fixture.id) != null) AgentProfileSheet(fixture, onDismiss = {}, onOpenOverview = {}) + } + } + } + compose.runOnIdle { scene.session.connect() } + compose.waitUntil(5_000) { scene.session.state.value.bot(fixture.id) != null } + compose.waitForIdle() + } + + private fun call(status: String): String = + """{"callId":"c1","botId":"bot-1","threadId":"thread-bot-1","client":"android","voice":"marin","startedAt":${System.currentTimeMillis()},"status":"$status"}""" + + private fun json(code: Int, body: String): MockResponse = MockResponse() + .setResponseCode(code) + .setHeader("Content-Type", "application/json") + .setBody(body) + + /** Media is not what this test is about: an offer, an answer and an open channel, nothing else. */ + private class SilentTransport : LiveCallTransport { + private var listener: LiveCallTransport.Listener? = null + + override suspend fun offer(listener: LiveCallTransport.Listener): String { + this.listener = listener + return "v=0\r\noffer\r\n" + } + + override suspend fun accept(answerSdp: String) { + listener?.onChannelOpen() + } + override fun setMuted(muted: Boolean) = Unit + override fun sendClose() = Unit + override fun close() = Unit + } +} diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/VoiceNoteWiringTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/VoiceNoteWiringTest.kt index ea778434ef..761a39d6f2 100644 --- a/android/app/src/test/kotlin/com/openmausbot/companion/ui/VoiceNoteWiringTest.kt +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/VoiceNoteWiringTest.kt @@ -3,8 +3,10 @@ package com.openmausbot.companion.ui import androidx.activity.ComponentActivity import androidx.compose.foundation.layout.Column import androidx.compose.runtime.CompositionLocalProvider +import androidx.compose.ui.semantics.SemanticsActions import androidx.compose.ui.semantics.SemanticsProperties import androidx.compose.ui.semantics.getOrNull +import androidx.compose.ui.test.assertCountEquals import androidx.compose.ui.test.assertIsDisplayed import androidx.compose.ui.test.assertIsEnabled import androidx.compose.ui.test.assertIsNotEnabled @@ -14,6 +16,12 @@ import androidx.compose.ui.test.onAllNodesWithText import androidx.compose.ui.test.onNodeWithContentDescription import androidx.compose.ui.test.onNodeWithText import androidx.compose.ui.test.performClick +import androidx.compose.ui.test.performSemanticsAction +import com.openmausbot.companion.audio.LiveCallApi +import com.openmausbot.companion.audio.LiveCallLink +import com.openmausbot.companion.audio.LiveCallManager +import com.openmausbot.companion.audio.LiveCallTransport +import com.openmausbot.companion.audio.MicrophoneAccess import com.openmausbot.companion.audio.PreviewAudioFocus import com.openmausbot.companion.audio.VoiceNoteController import com.openmausbot.companion.audio.VoiceNoteEngine @@ -23,11 +31,17 @@ import com.openmausbot.companion.core.Connection import com.openmausbot.companion.core.CompanionJson import com.openmausbot.companion.core.Fleet import com.openmausbot.companion.core.Frame +import com.openmausbot.companion.core.LiveCallStart +import com.openmausbot.companion.core.LiveCallState +import com.openmausbot.companion.core.LiveCallStatus import com.openmausbot.companion.core.Message import com.openmausbot.companion.core.MessageImageAttachment import com.openmausbot.companion.core.StreamFrame import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.flow import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive @@ -44,12 +58,16 @@ import org.robolectric.RobolectricTestRunner import org.robolectric.annotation.Config import org.robolectric.annotation.GraphicsMode import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue /** * The wiring half of the voice-note bubble: a bot reply with a parked audio * attachment renders the play button, fetches the clip through the * authenticated file route on first play only, and pauses/resumes without a - * second request. + * second request. While this phone is on a Live call the play button is off, + * with the reason under it: a note would take the call's audio focus, and + * losing it ends the call. */ @RunWith(RobolectricTestRunner::class) @Config(sdk = [34]) @@ -274,11 +292,201 @@ class VoiceNoteWiringTest { assertEquals(0, compose.onAllNodesWithContentDescription("Pause voice note").fetchSemanticsNodes().size) } + @Test fun aLiveCallKeepsVoiceNotesFromPlayingAndSaysWhy() { + server.dispatcher = object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + // Only the clip's fetch counts; the session reads other routes on connect. + if (request.path?.endsWith("/file") != true) return MockResponse().setResponseCode(404) + requests.add(request) + return MockResponse().setHeader("Content-Type", "audio/mpeg").setBody(Buffer().write(ByteArray(64))) + } + } + server.start() + val fixture = bot().copy(threadId = "first", messages = emptyList()) + val player = VoiceNotePlayer( + controller = VoiceNoteController( + engineFactory = { StubEngine() }, + focus = GrantingFocus(), + ), + ) + val wiring = WiringScene( + connection = Connection(id = "voice-fixture", name = "Fixture", host = "127.0.0.1", port = server.port), + fleet = Fleet(listOf(fixture), emptyList()), + voiceNotes = player, + liveTransports = { SilentTransport() }, + liveApi = CallingComputer(), + ) { flow { emit(StreamFrame(Frame.Hello(cursor = "fixture:1", resumed = false), seq = 1)); awaitCancellation() } } + scene = wiring + val message = Message( + "reply", + Message.Role.BOT, + Message.Kind.TEXT, + 1.0, + text = "Heard you", + attachments = listOf(MessageImageAttachment("audio", "/attachments/note-call.mp3", "audio/mpeg", 4200.0)), + ) + compose.setContent { + CompositionLocalProvider(LocalCompanion provides wiring.environment) { + CompanionTheme { + MessageRow(Chat.BotChat(fixture), message) + } + } + } + compose.runOnIdle { wiring.session.connect() } + compose.waitUntil(5_000) { wiring.session.state.value.bot(fixture.id) != null } + compose.onNodeWithContentDescription("Play voice note").assertIsEnabled() + compose.onAllNodesWithText(LiveCallRules.VOICE_NOTE_DURING_CALL).assertCountEquals(0) + + val liveCalls = wiring.environment.liveCalls + compose.runOnIdle { liveCalls.start(fixture.id, fixture.threadId, fixture.name, MicrophoneAccess { it(true) }) } + compose.waitUntil(5_000) { compose.runOnIdle { liveCalls.state.value.holdsMedia } } + compose.onNodeWithContentDescription("Play voice note").assertIsNotEnabled() + compose.onNodeWithText(LiveCallRules.VOICE_NOTE_DURING_CALL).assertIsDisplayed() + + // A tap that reaches the click action anyway: the bubble refuses it itself. + compose.onNodeWithContentDescription("Play voice note").performSemanticsAction(SemanticsActions.OnClick) + compose.waitForIdle() + assertTrue(requests.isEmpty(), "nothing is fetched, so nothing can take the call's audio") + assertNull(player.playback.value) + + compose.runOnIdle { liveCalls.hangUp() } + compose.waitUntil(5_000) { compose.runOnIdle { !liveCalls.state.value.holdsMedia } } + compose.onNodeWithContentDescription("Play voice note").assertIsEnabled() + compose.onAllNodesWithText(LiveCallRules.VOICE_NOTE_DURING_CALL).assertCountEquals(0) + } + + /** + * The race the play button alone cannot stop: a note tapped before the + * call, whose download finishes after the call took the audio. The player + * refuses it (it would take the call's audio focus, which ends the call), + * the clip waits, ready, with the reason under it, and plays once the call + * is over, from the bytes it already has. + */ + @Test fun aNoteWhoseDownloadFinishesAfterACallStartedWaitsForTheCallToEnd() { + val release = CountDownLatch(1) + server.dispatcher = object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + if (request.path?.endsWith("/file") != true) return MockResponse().setResponseCode(404) + requests.add(request) + // Slow network: the clip arrives only once the call has begun. + release.await(10, TimeUnit.SECONDS) + return MockResponse().setHeader("Content-Type", "audio/mpeg").setBody(Buffer().write(ByteArray(64))) + } + } + server.start() + val fixture = bot().copy(threadId = "first", messages = emptyList()) + var calls: LiveCallManager? = null + val focus = CallAwareFocus { calls?.state?.value?.holdsMedia == true } + val player = VoiceNotePlayer(controller = VoiceNoteController(engineFactory = { StubEngine() }, focus = focus)) + val wiring = WiringScene( + connection = Connection(id = "voice-fixture", name = "Fixture", host = "127.0.0.1", port = server.port), + fleet = Fleet(listOf(fixture), emptyList()), + voiceNotes = player, + liveTransports = { SilentTransport() }, + liveApi = CallingComputer(), + ) { flow { emit(StreamFrame(Frame.Hello(cursor = "fixture:1", resumed = false), seq = 1)); awaitCancellation() } } + scene = wiring + val liveCalls = wiring.environment.liveCalls + calls = liveCalls + val message = Message( + "reply", + Message.Role.BOT, + Message.Kind.TEXT, + 1.0, + text = "Heard you", + attachments = listOf(MessageImageAttachment("audio", "/attachments/note-slow.mp3", "audio/mpeg", 4200.0)), + ) + compose.setContent { + CompositionLocalProvider(LocalCompanion provides wiring.environment) { + CompanionTheme { + MessageRow(Chat.BotChat(fixture), message) + } + } + } + compose.runOnIdle { wiring.session.connect() } + compose.waitUntil(5_000) { wiring.session.state.value.bot(fixture.id) != null } + + // Tapped before the call: the clip starts to download. + compose.waitUntil(10_000) { + if (requests.isEmpty() && compose.onAllNodesWithContentDescription("Play voice note").fetchSemanticsNodes().isNotEmpty()) { + compose.onAllNodesWithContentDescription("Play voice note")[0].performClick() + } + requests.isNotEmpty() + } + + // The call takes the audio while the clip is still on its way. + compose.runOnIdle { liveCalls.start(fixture.id, fixture.threadId, fixture.name, MicrophoneAccess { it(true) }) } + compose.waitUntil(5_000) { compose.runOnIdle { liveCalls.state.value.holdsMedia } } + release.countDown() + + // The clip lands: the player refuses it; nothing plays and nothing failed. + compose.waitUntil(10_000) { + // Queried on every poll: that drives the frame sync the download's + // coroutine resumes on (see the first test). + val failed = compose.onAllNodesWithText("Voice note unavailable").fetchSemanticsNodes().isNotEmpty() + focus.checks > 0 || player.playback.value != null || failed + } + compose.waitForIdle() + assertNull(player.playback.value, "the note did not take the call's audio") + assertEquals(0, focus.requests, "nothing asked for the audio focus") + compose.onAllNodesWithText("Voice note unavailable").assertCountEquals(0) + compose.onNodeWithText(LiveCallRules.VOICE_NOTE_DURING_CALL).assertIsDisplayed() + compose.onNodeWithContentDescription("Play voice note").assertIsNotEnabled() + + // Once the call is over the clip plays, without a second fetch. + compose.runOnIdle { liveCalls.hangUp() } + compose.waitUntil(5_000) { compose.runOnIdle { !liveCalls.state.value.holdsMedia } } + compose.waitUntil(5_000) { + if (player.playback.value?.playing != true && compose.onAllNodesWithContentDescription("Play voice note").fetchSemanticsNodes().isNotEmpty()) { + compose.onAllNodesWithContentDescription("Play voice note")[0].performClick() + } + player.playback.value?.playing == true + } + assertEquals(1, requests.size, "played from the clip it already had") + } + + /** The computer's side of a call: it answers at once, and nothing about it is under test here. */ + private class CallingComputer : LiveCallApi { + override val serverCall = MutableStateFlow(null) + override val link = MutableStateFlow(LiveCallLink(computerId = "voice-fixture", signedIn = true)) + + override suspend fun start(botId: String, threadId: String, sdp: String): LiveCallStart = LiveCallStart.Started( + LiveCallState("c1", botId, threadId, "android", "marin", 1.0, LiveCallStatus.CONNECTING), + answerSdp = "v=0\r\nanswer\r\n", + ) + + override suspend fun end(callId: String) = Unit + } + + private class SilentTransport : LiveCallTransport { + override suspend fun offer(listener: LiveCallTransport.Listener): String = "v=0\r\noffer\r\n" + override suspend fun accept(answerSdp: String) = Unit + override fun setMuted(muted: Boolean) = Unit + override fun sendClose() = Unit + override fun close() = Unit + } + private class GrantingFocus : PreviewAudioFocus { override fun request(onInterrupted: () -> Unit): Boolean = true override fun abandon() = Unit } + /** Grants the focus, and knows, as the app's focus gate does, whether this phone's Live call holds the audio. */ + private class CallAwareFocus(private val callHolds: () -> Boolean) : PreviewAudioFocus { + @Volatile var requests = 0 + @Volatile var checks = 0 + override fun request(onInterrupted: () -> Unit): Boolean { + requests += 1 + return true + } + override fun abandon() = Unit + override val heldByLiveCall: Boolean + get() { + checks += 1 + return callHolds() + } + } + private class StubEngine : VoiceNoteEngine { override var onCompletion: (() -> Unit)? = null override var onError: (() -> Unit)? = null diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/WiringScene.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/WiringScene.kt index d89f1753ff..21dd45954c 100644 --- a/android/app/src/test/kotlin/com/openmausbot/companion/ui/WiringScene.kt +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/WiringScene.kt @@ -2,6 +2,12 @@ package com.openmausbot.companion.ui import android.app.Application import android.content.Context +import com.openmausbot.companion.audio.LiveCallApi +import com.openmausbot.companion.audio.LiveCallAudioRoute +import com.openmausbot.companion.audio.LiveCallManager +import com.openmausbot.companion.audio.LiveCallPreferences +import com.openmausbot.companion.audio.LiveCallTransportFactory +import com.openmausbot.companion.audio.SessionLiveCallApi import com.openmausbot.companion.audio.VoicePreviewPlayer import com.openmausbot.companion.audio.VoiceNotePlayer import com.openmausbot.companion.avatar.AvatarImageStore @@ -60,6 +66,14 @@ internal class WiringScene( fleet: Fleet = Fleet(emptyList(), emptyList()), /** The transcript voice-note player; null builds a real one, tests inject a fake. */ voiceNotes: VoiceNotePlayer? = null, + /** A fake WebRTC transport for the call bar tests; null builds the production manager, which never creates media on its own. */ + liveTransports: LiveCallTransportFactory? = null, + /** The computer's side of a call, with [liveTransports]; null goes through the real session. */ + liveApi: LiveCallApi? = null, + /** The phone's clock as the call manager reads it, with [liveTransports]. */ + liveClock: () -> Long = System::currentTimeMillis, + /** Whether this phone already made a Live call (the first-call disclosure is behind it), with [liveTransports]. */ + liveDisclosureShown: Boolean = true, /** The body of the nth stream (1-based). Hangs by default, like a live SSE. */ private val events: (Int) -> Flow = { flow { awaitCancellation() } }, ) { @@ -90,6 +104,20 @@ internal class WiringScene( metadataFn = { throw APIError.Status(404) }, ) + /** The call manager, built before the players so they can ask it, as the app does. */ + private val liveCalls: LiveCallManager = if (liveTransports == null) { + LiveCallManager(context, session, scope) + } else { + LiveCallManager( + api = liveApi ?: SessionLiveCallApi(session), + scope = scope, + transports = liveTransports, + audio = SilentAudio, + preferences = MemoryPreferences(disclosureShown = liveDisclosureShown), + clock = liveClock, + ) + } + val environment = CompanionEnvironment( session = session, permissions = CompanionPermissions(sdkInt = 34, granted = { true }), @@ -107,8 +135,9 @@ internal class WiringScene( openSettings = {}, ), avatars = AvatarImageStore(fetch = { null }), - voicePreview = VoicePreviewPlayer(context), - voiceNotes = voiceNotes ?: VoiceNotePlayer(context), + voicePreview = VoicePreviewPlayer(context, liveCallHoldsAudio = { liveCalls.state.value.holdsMedia }), + voiceNotes = voiceNotes ?: VoiceNotePlayer(context, liveCallHoldsAudio = { liveCalls.state.value.holdsMedia }), + liveCalls = liveCalls, dictation = SpeechDictation( context = context, hasRecordAudio = { false }, @@ -129,6 +158,18 @@ internal class WiringScene( override fun discover(): Flow = emptyFlow() } + /** A fresh speaker choice per scene, so scenes cannot read each other's. */ + private class MemoryPreferences( + override var speaker: Boolean = true, + override var disclosureShown: Boolean = true, + ) : LiveCallPreferences + + private object SilentAudio : LiveCallAudioRoute { + override fun begin(speaker: Boolean, onFocusLost: () -> Unit) = Unit + override fun setSpeaker(speaker: Boolean) = Unit + override fun end() = Unit + } + private class FakeConnectionStore(private var saved: Connection?) : ConnectionStore { override suspend fun load(): Connection? = saved override suspend fun save(connection: Connection) { diff --git a/android/core/src/main/kotlin/com/openmausbot/companion/core/ChatPreferences.kt b/android/core/src/main/kotlin/com/openmausbot/companion/core/ChatPreferences.kt index a8364bb854..e1ffe7f962 100644 --- a/android/core/src/main/kotlin/com/openmausbot/companion/core/ChatPreferences.kt +++ b/android/core/src/main/kotlin/com/openmausbot/companion/core/ChatPreferences.kt @@ -156,6 +156,14 @@ internal fun previewText(message: Message): String = when (message.kind) { * Port of `isActivityReceipt` in `ChatPreferences.swift`. A routine-run card is * not one: it is the run's result, and it stays whatever the setting. */ +/** + * A status row the server writes while a turn runs ("notice: Qwen hit a rate + * limit and is retrying"). Port of `isStatusNotice` in `ChatPreferences.swift`: + * it tells the reader what the bot is doing, so it is never hidden or folded. + */ +fun isStatusNotice(message: Message): Boolean = + message.kind == Message.Kind.ACTIVITY && message.tool?.name?.startsWith("notice:") == true + fun isActivityReceipt(message: Message): Boolean = when (message.kind) { Message.Kind.ACTIVITY, Message.Kind.DIGEST, Message.Kind.COMPACTION -> true else -> false @@ -204,13 +212,13 @@ fun transcriptRows(messages: List, detail: ActivityDetail): List