Remove hardware limitation for wake word detection (#6625)

Author: TimoPtr

.github/workflows/pr.yml

@@ -295,23 +295,23 @@ jobs:
             arch: x86_64
             profile: "Nexus 5"
             target: "google_apis"
-            gradle_target: ":app:connectedFullDebugAndroidTest :app:connectedMinimalDebugAndroidTest :common:connectedDebugAndroidTest :microfrontend:connectedDebugAndroidTest"
+            gradle_target: ":app:connectedFullDebugAndroidTest :app:connectedMinimalDebugAndroidTest :common:connectedDebugAndroidTest :microwakeword:connectedDebugAndroidTest"
           - api-level: 36
             arch: x86_64
             profile: "pixel_7"
             target: "google_apis"
-            gradle_target: ":app:connectedFullDebugAndroidTest :app:connectedMinimalDebugAndroidTest :common:connectedDebugAndroidTest :microfrontend:connectedDebugAndroidTest"
+            gradle_target: ":app:connectedFullDebugAndroidTest :app:connectedMinimalDebugAndroidTest :common:connectedDebugAndroidTest :microwakeword:connectedDebugAndroidTest"
           - api-level: 33
             arch: x86_64
             profile: "automotive_1024p_landscape"
             target: "android-automotive"
-            gradle_target: ":automotive:connectedFullDebugAndroidTest :automotive:connectedMinimalDebugAndroidTest :common:connectedDebugAndroidTest :microfrontend:connectedDebugAndroidTest"
+            gradle_target: ":automotive:connectedFullDebugAndroidTest :automotive:connectedMinimalDebugAndroidTest :common:connectedDebugAndroidTest :microwakeword:connectedDebugAndroidTest"
           - api-level: "34"
             system-image-api-level: "34-ext9"
             arch: x86_64
             profile: "automotive_1024p_landscape"
             target: "android-automotive"
-            gradle_target: ":automotive:connectedFullDebugAndroidTest :automotive:connectedMinimalDebugAndroidTest :common:connectedDebugAndroidTest :microfrontend:connectedDebugAndroidTest"
+            gradle_target: ":automotive:connectedFullDebugAndroidTest :automotive:connectedMinimalDebugAndroidTest :common:connectedDebugAndroidTest :microwakeword:connectedDebugAndroidTest"
           - api-level: 26
             arch: x86
             profile: "wearos_square"
@@ -321,7 +321,7 @@ jobs:
             arch: x86_64
             profile: "wearos_small_round"
             target: "android-wear"
-            gradle_target: ":wear:connectedDebugAndroidTest :common:connectedDebugAndroidTest :microfrontend:connectedDebugAndroidTest"
+            gradle_target: ":wear:connectedDebugAndroidTest :common:connectedDebugAndroidTest :microwakeword:connectedDebugAndroidTest"
     steps:
       - name: Delete unnecessary tools 🔧
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1

app/build.gradle.kts

@@ -23,6 +23,18 @@ android {
         }
     }
 
+    buildTypes {
+        debug {
+            // Required for HWASan wrap.sh to be included uncompressed in the APK
+            // See https://developer.android.com/ndk/guides/hwasan
+            packaging {
+                jniLibs {
+                    useLegacyPackaging = true
+                }
+            }
+        }
+    }
+
     lint {
         // Until we fully migrate to Material3 this lint issue is too verbose https://github.com/home-assistant/android/issues/5420
         disable += listOf("UsingMaterialAndMaterial3Libraries")

app/gradle.lockfile

@@ -3,40 +3,22 @@ package io.homeassistant.companion.android.assist.wakeword
 import android.content.Context
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.platform.app.InstrumentationRegistry
-import io.homeassistant.companion.android.BuildConfig
+import io.homeassistant.companion.android.microwakeword.MicroWakeWord
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.channels.FileChannel
 import kotlinx.coroutines.test.runTest
 import org.junit.Assert.assertFalse
 import org.junit.Assert.assertTrue
-import org.junit.Assume.assumeTrue
 import org.junit.Test
 import org.junit.runner.RunWith
-import timber.log.Timber
 
 @RunWith(AndroidJUnit4::class)
 class MicroWakeWordModelTest {
 
     private val appContext: Context
         get() = InstrumentationRegistry.getInstrumentation().targetContext
 
-    /**
-     * Initializes TfLite and returns true if successful.
-     * Returns false if initialization fails (e.g., full flavor without GMS).
-     */
-    private suspend fun tryInitializeTfLite(): Boolean {
-        return try {
-            TfLiteInitializerImpl().initialize(appContext)
-            true
-        } catch (e: Exception) {
-            Timber.w(e, "TfLite initialization failed, skipping test")
-            if (BuildConfig.FLAVOR == "full") {
-                false
-            } else {
-                // In minimal the test should run since we use the embedded version of TfLite
-                throw e
-            }
-        }
-    }
-
     @Test
     fun loadsModelsFromAppAssets_verify_models_config_files() = runTest {
         val models = MicroWakeWordModelConfig.loadAvailableModels(appContext)
@@ -59,13 +41,18 @@ class MicroWakeWordModelTest {
 
     @Test
     fun microWakeWord_loadsAndProcessesAudio_withAllModels() = runTest {
-        assumeTrue("TfLite not available", tryInitializeTfLite())
         val models = MicroWakeWordModelConfig.loadAvailableModels(appContext)
 
         for (model in models) {
-            val detector = MicroWakeWord.create(appContext, model)
+            val modelBuffer = loadModelFile(appContext, model.modelAssetPath)
+            val detector = MicroWakeWord(
+                modelBuffer = modelBuffer,
+                featureStepSizeMs = model.micro.featureStepSize,
+                probabilityCutoff = model.micro.probabilityCutoff,
+                slidingWindowSize = model.micro.slidingWindowSize,
+            )
 
-            try {
+            detector.use { detector ->
                 // Process silent audio (160 samples = 10ms at 16kHz)
                 val silentAudio = ShortArray(160)
                 val detected = detector.processAudio(silentAudio)
@@ -75,20 +62,23 @@ class MicroWakeWordModelTest {
                     "Silent audio should not trigger '${model.wakeWord}' detection",
                     detected,
                 )
-            } finally {
-                detector.close()
             }
         }
     }
 
     @Test
     fun microWakeWord_canResetState_withoutCrashing() = runTest {
-        assumeTrue("TfLite not available", tryInitializeTfLite())
         val models = MicroWakeWordModelConfig.loadAvailableModels(appContext)
         val model = models.first()
 
-        val detector = MicroWakeWord.create(appContext, model)
-        try {
+        val modelBuffer = loadModelFile(appContext, model.modelAssetPath)
+        val detector = MicroWakeWord(
+            modelBuffer = modelBuffer,
+            featureStepSizeMs = model.micro.featureStepSize,
+            probabilityCutoff = model.micro.probabilityCutoff,
+            slidingWindowSize = model.micro.slidingWindowSize,
+        )
+        detector.use { detector ->
             // Process some audio
             detector.processAudio(ShortArray(160))
             detector.processAudio(ShortArray(160))
@@ -99,8 +89,15 @@ class MicroWakeWordModelTest {
             // Should be able to process more audio after reset
             val detected = detector.processAudio(ShortArray(160))
             assertFalse(detected)
-        } finally {
-            detector.close()
         }
     }
+
+    private fun loadModelFile(context: Context, assetPath: String): ByteBuffer {
+        val assetFd = context.assets.openFd(assetPath)
+        val inputStream = assetFd.createInputStream()
+        val fileChannel = inputStream.channel
+        val mapped = fileChannel.map(FileChannel.MapMode.READ_ONLY, assetFd.startOffset, assetFd.declaredLength)
+        mapped.order(ByteOrder.nativeOrder())
+        return mapped
+    }
 }

app/src/androidTest/kotlin/io/homeassistant/companion/android/assist/wakeword/MicroWakeWordModelTest.kt

@@ -0,0 +1,5 @@
+#!/system/bin/sh
+# Enables HWAddressSanitizer (HWASan) for debug builds on ARM64 devices running Android 14+.
+# HWASan detects memory errors (buffer overflows, use-after-free, double free, stack-use-after-return).
+# See https://developer.android.com/ndk/guides/hwasan
+LD_HWASAN=1 exec "$@"

app/src/debug/resources/lib/arm64-v8a/wrap.sh

@@ -45,6 +45,7 @@ class AssistAudioStrategyFactory @Inject constructor(
             wakeWordListenerFactory = wakeWordListenerFactory,
             assistConfigManager = assistConfigManager,
             wakeWordPhrase = wakeWordPhrase,
+            audioManager = context.getSystemService(),
             onListenerStopped = {
                 AssistVoiceInteractionService.resumeListening(context)
             },

app/src/full/kotlin/io/homeassistant/companion/android/assist/wakeword/TfLiteInitializerImpl.kt

@@ -64,7 +64,6 @@ class AssistVoiceInteractionService : VoiceInteractionService() {
             onWakeWordDetected = ::onWakeWordDetected,
             onListenerReady = ::onListenerReady,
             onListenerStopped = ::onListenerStopped,
-            onListenerFailed = ::onListenerFailed,
         )
     }
     private var isServiceReady = false
@@ -142,10 +141,6 @@ class AssistVoiceInteractionService : VoiceInteractionService() {
      */
     @SuppressLint("MissingPermission")
     private fun startListening() {
-        if (!assistConfigManager.isWakeWordSupported()) {
-            Timber.d("Wake word detection is not supported on this device")
-            return
-        }
         if (!hasRecordAudioPermission()) {
             Timber.w("RECORD_AUDIO permission not granted, cannot start listening")
             return
@@ -186,14 +181,6 @@ class AssistVoiceInteractionService : VoiceInteractionService() {
         stopForegroundCompat()
     }
 
-    private fun onListenerFailed() {
-        serviceScope.launch {
-            Timber.w("Wake word listener failed, disabling wake word to prevent issue")
-            @SuppressLint("MissingPermission")
-            assistConfigManager.setWakeWordEnabled(false)
-        }
-    }
-
     /**
      * Stop listening for wake word.
      */