From 4463351a11ce764727858a396506fa15e292dd25 Mon Sep 17 00:00:00 2001 From: Arthur Breton Date: Sat, 7 Nov 2026 04:35:17 +0800 Subject: [PATCH 1/3] fix(web): validate audio-studio, sherpa-onnx.rn and moonshine.rn web builds - sherpa-onnx.rn: web initTts names unfetched model/tokens/lexicon URLs and frees loaded files instead of crashing out of bounds; rejects non-VITS models; README documents VITS-only web TTS - sherpa-onnx.rn: web loader clears its 30 s backup timer once loading settles, so the timeout warning no longer fires after a good load - sherpa-voice: sample cards measure files by fetch on web, where getInfoAsync is missing (was 0 Bytes / 0:00); add the jest preset jest-expo needs so yarn test runs - playground: build:deps also builds sherpa-onnx.rn and moonshine.rn lib - agentic: WEB_FAKE_AUDIO_FILE for fake mic capture, hermetic recipe tests, four web validation recipes --- apps/playground/package.json | 2 +- apps/sherpa-voice/CHANGELOG.md | 4 + apps/sherpa-voice/package.json | 1 + apps/sherpa-voice/src/hooks/useAsr.ts | 7 +- .../sherpa-voice/src/hooks/useAudioTagging.ts | 8 +- apps/sherpa-voice/src/hooks/useSpeakerId.ts | 5 +- .../src/utils/__tests__/fileUtils.test.ts | 62 ++ apps/sherpa-voice/src/utils/fileUtils.ts | 15 + packages/sherpa-onnx.rn/CHANGELOG.md | 6 + packages/sherpa-onnx.rn/README.md | 2 + .../src/__tests__/webCombinedLoader.test.ts | 79 +++ .../src/__tests__/webTtsInit.test.ts | 106 +++ .../sherpa-onnx.rn/src/web/features/tts.ts | 39 +- .../wasm-src/sherpa-onnx-combined.js | 11 +- scripts/agentic/recipe/cli.test.mjs | 2 +- scripts/agentic/recipe/provider.test.mjs | 6 +- .../audio-studio-web-validation.recipe.json | 666 ++++++++++++++++++ ...onshine-web-live-transcription.recipe.json | 318 +++++++++ .../sherpa/web-asr-validation.recipe.json | 560 +++++++++++++++ .../sherpa/web-tts-validation.recipe.json | 233 ++++++ scripts/agentic/web-browser.mjs | 17 +- yarn.lock | 1 + 22 files changed, 2120 insertions(+), 30 deletions(-) create mode 100644 apps/sherpa-voice/src/utils/__tests__/fileUtils.test.ts create mode 100644 packages/sherpa-onnx.rn/src/__tests__/webCombinedLoader.test.ts create mode 100644 packages/sherpa-onnx.rn/src/__tests__/webTtsInit.test.ts create mode 100644 scripts/agentic/recipe/recipes/playground/audio-studio-web-validation.recipe.json create mode 100644 scripts/agentic/recipe/recipes/playground/moonshine-web-live-transcription.recipe.json create mode 100644 scripts/agentic/recipe/recipes/sherpa/web-asr-validation.recipe.json create mode 100644 scripts/agentic/recipe/recipes/sherpa/web-tts-validation.recipe.json diff --git a/apps/playground/package.json b/apps/playground/package.json index e5ae432fb..020cc9cc8 100644 --- a/apps/playground/package.json +++ b/apps/playground/package.json @@ -5,7 +5,7 @@ "scripts": { "eas-build-pre-install": "bash scripts/eas-build-pre-install.sh", "postinstall": "patch-package", - "build:deps": "cd ../../packages/audio-studio && yarn build && yarn build:plugin && cd ../playgroundapi && yarn build && cd ../react-native-essentia && yarn prepare", + "build:deps": "cd ../../packages/audio-studio && yarn build && yarn build:plugin && cd ../playgroundapi && yarn build && cd ../react-native-essentia && yarn prepare && cd ../sherpa-onnx.rn && yarn prepare && cd ../moonshine.rn && yarn prepare", "start": "NODE_ENV=development APP_VARIANT=development bash scripts/agentic/start-metro.sh && node scripts/agentic/start-interactive.mjs", "stop": "bash scripts/agentic/stop-metro.sh", "android": "APP_VARIANT=development PLATFORM=android HEADLESS=0 REBUILD=1 bash scripts/agentic/preflight.sh", diff --git a/apps/sherpa-voice/CHANGELOG.md b/apps/sherpa-voice/CHANGELOG.md index 31caba120..6a98dfc02 100644 --- a/apps/sherpa-voice/CHANGELOG.md +++ b/apps/sherpa-voice/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Web: the ASR, speaker ID and audio tagging sample cards show the file size and duration instead of `0 Bytes` and `0:00`. `getInfoAsync` has no web implementation, so web now fetches the file to measure it. +- `yarn test` runs again. `jest-expo` needs `@react-native/jest-preset` since React Native 0.88, so it is now a dev dependency. + ## [1.2.0] - 2026-11-02 ### Changed diff --git a/apps/sherpa-voice/package.json b/apps/sherpa-voice/package.json index d83c2c06f..5b2e36875 100644 --- a/apps/sherpa-voice/package.json +++ b/apps/sherpa-voice/package.json @@ -144,6 +144,7 @@ "@babel/plugin-proposal-decorators": "^7.25.9", "@config-plugins/detox": "^9.0.0", "@playwright/test": "^1.58.2", + "@react-native/jest-preset": "0.88.1", "@types/jest": "30.0.0", "@types/react": "~19.3.0", "@types/react-test-renderer": "^19", diff --git a/apps/sherpa-voice/src/hooks/useAsr.ts b/apps/sherpa-voice/src/hooks/useAsr.ts index 7e9b43e4e..3bfdb90bb 100644 --- a/apps/sherpa-voice/src/hooks/useAsr.ts +++ b/apps/sherpa-voice/src/hooks/useAsr.ts @@ -5,7 +5,7 @@ import { type AudioDataEvent, } from '@siteed/audio-studio' import { baseLogger } from '../config' -import { resolveModelDir } from '../utils/fileUtils' +import { getFileSize, resolveModelDir } from '../utils/fileUtils' import { SAMPLE_AUDIO_FILES } from '../utils/asrSamples' export { SAMPLE_AUDIO_FILES } from '../utils/asrSamples' @@ -14,7 +14,6 @@ import { makeWebProgressHandler, getWebModelBaseUrl } from '../utils/webModelUti import { getWebAsrBackend } from '../config/webFeatures' import { Asset } from 'expo-asset' import { createAudioPlayer } from 'expo-audio' -import * as FileSystem from 'expo-file-system/legacy' import { useLocalSearchParams } from 'expo-router' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useAsrModels, useAsrModelWithConfig } from './useModelWithConfig' @@ -239,7 +238,7 @@ export function useAsr() { setSelectedAudio(audioItem) setAudioMetadata({ isLoading: true }) try { - const info = await FileSystem.getInfoAsync(audioItem.localUri) + const size = await getFileSize(audioItem.localUri) let duration = 0 try { const tempPlayer = createAudioPlayer({ @@ -250,7 +249,7 @@ export function useAsr() { tempPlayer.remove() } catch (_) {} setAudioMetadata({ - size: info.exists ? info.size : 0, + size, duration, isLoading: false, }) diff --git a/apps/sherpa-voice/src/hooks/useAudioTagging.ts b/apps/sherpa-voice/src/hooks/useAudioTagging.ts index 6faf44084..20c7d58c0 100644 --- a/apps/sherpa-voice/src/hooks/useAudioTagging.ts +++ b/apps/sherpa-voice/src/hooks/useAudioTagging.ts @@ -8,6 +8,7 @@ import { createAudioPlayer } from 'expo-audio' import * as FileSystem from 'expo-file-system/legacy' import { useCallback, useEffect, useRef, useState } from 'react' import { Alert } from 'react-native' +import { getFileSize } from '../utils/fileUtils' import { makeWebProgressHandler, getWebModelBaseUrl } from '../utils/webModelUtils' import { useAudioTaggingModels, @@ -343,12 +344,7 @@ export function useAudioTagging() { if (audioItem.localUri) { try { - const fileInfo = await FileSystem.getInfoAsync( - audioItem.localUri - ) - const size = fileInfo.exists - ? (fileInfo as any).size || 0 - : 0 + const size = await getFileSize(audioItem.localUri) const tempPlayer = createAudioPlayer({ uri: audioItem.localUri, }) diff --git a/apps/sherpa-voice/src/hooks/useSpeakerId.ts b/apps/sherpa-voice/src/hooks/useSpeakerId.ts index 31ef8948a..cca91057d 100644 --- a/apps/sherpa-voice/src/hooks/useSpeakerId.ts +++ b/apps/sherpa-voice/src/hooks/useSpeakerId.ts @@ -6,9 +6,9 @@ import { } from '@siteed/sherpa-onnx.rn' import { Asset } from 'expo-asset' import { createAudioPlayer } from 'expo-audio' -import * as FileSystem from 'expo-file-system/legacy' import { useEffect, useState } from 'react' import { Alert } from 'react-native' +import { getFileSize } from '../utils/fileUtils' import { makeWebProgressHandler, getWebModelBaseUrl } from '../utils/webModelUtils' import { useSpeakerIdModelWithConfig, @@ -341,8 +341,7 @@ export function useSpeakerId() { setIdentifyResult(null) setAudioMetadata({ isLoading: true }) try { - const fileInfo = await FileSystem.getInfoAsync(audioItem.localUri) - const fileSize = fileInfo.exists ? (fileInfo as any).size || 0 : 0 + const fileSize = await getFileSize(audioItem.localUri) const tempPlayer = createAudioPlayer({ uri: audioItem.localUri }) await new Promise((r) => setTimeout(r, 500)) const durationMs = (tempPlayer.duration || 0) * 1000 diff --git a/apps/sherpa-voice/src/utils/__tests__/fileUtils.test.ts b/apps/sherpa-voice/src/utils/__tests__/fileUtils.test.ts new file mode 100644 index 000000000..677c6d5d9 --- /dev/null +++ b/apps/sherpa-voice/src/utils/__tests__/fileUtils.test.ts @@ -0,0 +1,62 @@ +/// +import { Platform } from 'react-native' +import * as FileSystem from 'expo-file-system/legacy' +import { getFileSize } from '../fileUtils' + +jest.mock('react-native', () => ({ Platform: { OS: 'ios' } })) +jest.mock('../../config', () => ({ + baseLogger: { extend: () => ({ info: jest.fn(), warn: jest.fn() }) }, +})) +jest.mock('expo-file-system/legacy', () => ({ getInfoAsync: jest.fn() })) + +const getInfoAsync = FileSystem.getInfoAsync as jest.Mock +const SAMPLE = '/assets/?unstable_path=.%2Fassets%2Fjfk.wav' + +describe('getFileSize', () => { + afterEach(() => { + jest.resetAllMocks() + Platform.OS = 'ios' + }) + + describe('on web', () => { + beforeEach(() => { + Platform.OS = 'web' + // expo-file-system has no web implementation of getInfoAsync. + getInfoAsync.mockRejectedValue( + new Error('ExponentFileSystem.getInfoAsync is not available on web') + ) + }) + + it('measures the fetched body instead of calling getInfoAsync', async () => { + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + blob: () => Promise.resolve({ size: 352044 }), + }) + + await expect(getFileSize(SAMPLE)).resolves.toBe(352044) + expect(global.fetch).toHaveBeenCalledWith(SAMPLE) + expect(getInfoAsync).not.toHaveBeenCalled() + }) + + it('returns 0 when the file is missing', async () => { + global.fetch = jest.fn().mockResolvedValue({ ok: false, status: 404 }) + + await expect(getFileSize(SAMPLE)).resolves.toBe(0) + }) + }) + + describe('on native', () => { + it('reads the size from getInfoAsync', async () => { + getInfoAsync.mockResolvedValue({ exists: true, size: 1234 }) + + await expect(getFileSize('file:///data/jfk.wav')).resolves.toBe(1234) + expect(getInfoAsync).toHaveBeenCalledWith('file:///data/jfk.wav') + }) + + it('returns 0 when the file is missing', async () => { + getInfoAsync.mockResolvedValue({ exists: false }) + + await expect(getFileSize('file:///data/missing.wav')).resolves.toBe(0) + }) + }) +}) diff --git a/apps/sherpa-voice/src/utils/fileUtils.ts b/apps/sherpa-voice/src/utils/fileUtils.ts index 3b7ae2c56..2d06e35b7 100644 --- a/apps/sherpa-voice/src/utils/fileUtils.ts +++ b/apps/sherpa-voice/src/utils/fileUtils.ts @@ -77,6 +77,21 @@ export async function fileExists(uri: string): Promise { return info.exists; } +/** + * Size of a file in bytes, or 0 when it is missing. + * On web: fetch the URL and measure the body (getInfoAsync is not available). + * On native: FileSystem.getInfoAsync. + */ +export async function getFileSize(uri: string): Promise { + if (Platform.OS === 'web') { + const response = await fetch(uri); + if (!response.ok) return 0; + return (await response.blob()).size; + } + const info = await FileSystem.getInfoAsync(uri); + return info.exists ? info.size : 0; +} + /** * Resolve the effective model directory path. * Sherpa-onnx tarballs extract into a `sherpa-onnx-*` subdirectory — descend diff --git a/packages/sherpa-onnx.rn/CHANGELOG.md b/packages/sherpa-onnx.rn/CHANGELOG.md index e0c720625..097a2d689 100644 --- a/packages/sherpa-onnx.rn/CHANGELOG.md +++ b/packages/sherpa-onnx.rn/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Web `initTts` fails with `Failed to load TTS model files: ` when `model.onnx`, `tokens.txt` or a configured `lexiconFile` cannot be fetched, and frees the files that did load. Before, a missing lexicon aborted engine creation with `RuntimeError: memory access out of bounds`, and a missing model or tokens file gave `Missing required files for VITS model configuration` without naming it. +- Web `initTts` rejects non-VITS models with `Web TTS supports VITS models only, not `. The WASM glue builds no engine config for matcha or kokoro, so those crashed during engine creation. +- The web loader no longer logs `Module loading timeout reached` 30 s after the modules loaded. It now clears its backup timer once loading settles, and a load that really times out warns once instead of twice. + ## [2.0.0] - 2026-11-02 ### Migration diff --git a/packages/sherpa-onnx.rn/README.md b/packages/sherpa-onnx.rn/README.md index 6d4dc9bab..c2c8ad777 100644 --- a/packages/sherpa-onnx.rn/README.md +++ b/packages/sherpa-onnx.rn/README.md @@ -78,6 +78,8 @@ if (Platform.OS === 'web') { Self-hosting (optional): copy files from `node_modules/@siteed/sherpa-onnx.rn/wasm/` and call `configureSherpaOnnx({ wasmBasePath: '/your/wasm/path/' })`. +Web TTS supports VITS models only; matcha and kokoro need iOS or Android. It fetches `model.onnx`, `tokens.txt` and `lexiconFile` from `modelBaseUrl`, so host the lexicon (for English, upstream `tts-models/lexicon-en-us.txt`) next to the voice files. The browser fetch needs the same origin or CORS headers; GitHub release download URLs send none. If a file is missing, `initTts` fails with `Failed to load TTS model files: `. + See **[Getting Started: Web](docs/GETTING_STARTED_WEB.md)** for the full guide including `onProgress`, loading indicators, and model configuration. ## Usage diff --git a/packages/sherpa-onnx.rn/src/__tests__/webCombinedLoader.test.ts b/packages/sherpa-onnx.rn/src/__tests__/webCombinedLoader.test.ts new file mode 100644 index 000000000..3cf363db2 --- /dev/null +++ b/packages/sherpa-onnx.rn/src/__tests__/webCombinedLoader.test.ts @@ -0,0 +1,79 @@ +const TIMEOUT_WARNING = 'Module loading timeout reached'; + +type FakeScript = { src?: string; onload?: () => void; onerror?: () => void }; +type LoaderGlobals = { + window?: unknown; + document?: unknown; + SherpaOnnx?: unknown; + Module?: unknown; + sherpaOnnxModulePaths?: string[]; + onModuleReady?: () => void; +}; + +const g = globalThis as unknown as LoaderGlobals; + +// Load the browser loader with a fake DOM whose